You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by be...@apache.org on 2023/11/21 12:59:21 UTC

(cassandra-accord) branch trunk updated: Improve validation and address various discovered faults

This is an automated email from the ASF dual-hosted git repository.

benedict pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git


The following commit(s) were added to refs/heads/trunk by this push:
     new 746dabe0 Improve validation and address various discovered faults
746dabe0 is described below

commit 746dabe0b43bf719badbd605e68a76037d01256d
Author: Benedict Elliott Smith <be...@apache.org>
AuthorDate: Tue Aug 8 09:09:40 2023 +0100

    Improve validation and address various discovered faults
    
    patch by Benedict; reviewed by Ariel Weisberg for CASSANDRA-19045
---
 accord-core/build.gradle                           |   9 +
 accord-core/src/main/java/accord/api/Agent.java    |   1 +
 .../src/main/java/accord/api/ProgressLog.java      |   5 +
 accord-core/src/main/java/accord/api/Result.java   |   8 -
 .../src/main/java/accord/coordinate/Barrier.java   |  12 +-
 .../main/java/accord/coordinate/CheckShards.java   |   2 +-
 .../java/accord/coordinate/CoordinateNoOp.java     |   6 +-
 .../src/main/java/accord/coordinate/FetchData.java |  37 +-
 .../src/main/java/accord/coordinate/FindRoute.java |   3 +-
 .../main/java/accord/coordinate/FindSomeRoute.java |   2 +-
 .../src/main/java/accord/coordinate/Infer.java     | 230 ++++++--
 .../main/java/accord/coordinate/MaybeRecover.java  |  30 +-
 .../java/accord/coordinate/ReadCoordinator.java    |  23 +-
 .../src/main/java/accord/coordinate/Recover.java   |   3 +
 .../java/accord/coordinate/RecoverWithRoute.java   |  53 +-
 .../accord/coordinate/RecoverWithSomeRoute.java    |   2 +-
 .../accord/coordinate/tracking/ReadTracker.java    |   5 +
 .../java/accord/impl/AbstractFetchCoordinator.java |   1 +
 .../java/accord/impl/AbstractSafeCommandStore.java |   8 +-
 .../main/java/accord/impl/CommandTimeseries.java   |  18 +-
 .../main/java/accord/impl/ErasedSafeCommand.java   |   4 +-
 .../java/accord/impl/InMemoryCommandStore.java     | 170 +++++-
 .../main/java/accord/impl/SafeCommandsForKey.java  |  11 +-
 .../main/java/accord/impl/SimpleProgressLog.java   | 124 ++--
 .../src/main/java/accord/local/Bootstrap.java      |   5 +-
 .../src/main/java/accord/local/Command.java        | 354 ++++++++----
 .../src/main/java/accord/local/CommandStore.java   | 208 +++++--
 .../src/main/java/accord/local/CommandStores.java  |  67 ++-
 .../src/main/java/accord/local/Commands.java       | 318 ++++++-----
 .../src/main/java/accord/local/DurableBefore.java  |  26 +-
 accord-core/src/main/java/accord/local/Node.java   |   3 +-
 .../main/java/accord/local/RedundantBefore.java    | 294 +++++++++-
 .../main/java/accord/local/RedundantStatus.java    | 114 +++-
 .../src/main/java/accord/local/SafeCommand.java    |   2 -
 .../main/java/accord/local/SafeCommandStore.java   |  88 +--
 .../src/main/java/accord/local/SaveStatus.java     |  77 ++-
 .../main/java/accord/local/SerializerSupport.java  |  12 +-
 accord-core/src/main/java/accord/local/Status.java | 471 ++++++++++++---
 .../main/java/accord/messages/AbstractExecute.java |  21 +-
 .../src/main/java/accord/messages/Accept.java      |   4 +-
 .../src/main/java/accord/messages/Apply.java       |   2 +
 .../accord/messages/ApplyThenWaitUntilApplied.java |   6 +
 .../java/accord/messages/BeginInvalidation.java    |   5 -
 .../main/java/accord/messages/BeginRecovery.java   |  38 +-
 .../src/main/java/accord/messages/CheckStatus.java | 629 +++++++++++++++++----
 .../src/main/java/accord/messages/Commit.java      |   2 +-
 .../src/main/java/accord/messages/GetDeps.java     |   3 +-
 .../src/main/java/accord/messages/MessageType.java |   2 +-
 .../src/main/java/accord/messages/PreAccept.java   |  45 +-
 .../src/main/java/accord/messages/Propagate.java   | 297 +++++++---
 .../src/main/java/accord/messages/ReadData.java    |   9 +-
 .../main/java/accord/primitives/AbstractKeys.java  |  13 +-
 .../java/accord/primitives/AbstractRanges.java     |  36 +-
 .../accord/primitives/AbstractUnseekableKeys.java  |  64 ++-
 .../src/main/java/accord/primitives/Deps.java      |   5 +
 .../main/java/accord/primitives/EpochSupplier.java |   5 +
 .../src/main/java/accord/primitives/KeyDeps.java   |  24 +
 .../src/main/java/accord/primitives/KeyRoute.java  |  16 +-
 .../src/main/java/accord/primitives/Keys.java      |  10 +
 .../main/java/accord/primitives/PartialDeps.java   |   7 +
 .../main/java/accord/primitives/Participants.java  |  43 +-
 .../main/java/accord/primitives/ProgressToken.java |   4 +-
 .../src/main/java/accord/primitives/Range.java     |  60 ++
 .../src/main/java/accord/primitives/RangeDeps.java | 219 +++++--
 .../main/java/accord/primitives/RangeRoute.java    |  11 +-
 .../src/main/java/accord/primitives/Ranges.java    |   6 +
 .../src/main/java/accord/primitives/Routable.java  |   5 +
 .../src/main/java/accord/primitives/Routables.java |   1 +
 .../src/main/java/accord/primitives/Route.java     |  32 +-
 .../src/main/java/accord/primitives/Txn.java       |  71 ++-
 .../src/main/java/accord/primitives/TxnId.java     |  19 +
 .../main/java/accord/primitives/Unseekables.java   |   2 +
 .../src/main/java/accord/utils/Invariants.java     |   4 +-
 .../src/main/java/accord/utils/RandomSource.java   | 270 ++++-----
 .../java/accord/utils/ReducingIntervalMap.java     |  44 +-
 .../main/java/accord/utils/ReducingRangeMap.java   | 157 ++++-
 .../java/accord/utils/SearchableRangeList.java     |  41 +-
 .../src/main/java/accord/utils/SimpleBitSet.java   |   6 +
 .../src/main/java/accord/utils/SortedArrays.java   |  12 +-
 .../src/main/java/accord/utils/random/Picker.java  | 122 ++++
 .../random/Suppliers.java}                         |  42 +-
 .../src/test/java/accord/burn/BurnTest.java        |  20 +-
 .../src/test/java/accord/burn/TopologyUpdates.java |  19 +-
 .../coordinate/tracking/TrackerReconciler.java     |   2 +-
 .../src/test/java/accord/impl/TestAgent.java       |   9 +-
 .../src/test/java/accord/impl/basic/Cluster.java   | 264 ++++++++-
 .../accord/impl/basic/DelayedCommandStores.java    |  23 +-
 .../src/test/java/accord/impl/basic/NodeSink.java  |  78 +--
 .../basic/SimulatedDelayedExecutorService.java     |  25 +-
 .../src/test/java/accord/impl/list/ListAgent.java  |  11 +-
 .../src/test/java/accord/impl/list/ListRead.java   |  45 +-
 .../test/java/accord/impl/list/ListRequest.java    |  15 +-
 .../src/test/java/accord/impl/list/ListWrite.java  |   9 +-
 .../java/accord/local/ImmutableCommandTest.java    |   1 +
 .../test/java/accord/primitives/RangeDepsTest.java |  23 +
 .../java/accord/topology/TopologyRandomizer.java   |  15 +
 .../test/java/accord/utils/RandomSourceTest.java   | 128 +++++
 .../main/java/accord/maelstrom/MaelstromAgent.java |   5 +
 98 files changed, 4426 insertions(+), 1486 deletions(-)

diff --git a/accord-core/build.gradle b/accord-core/build.gradle
index 877bac65..d87e5000 100644
--- a/accord-core/build.gradle
+++ b/accord-core/build.gradle
@@ -134,3 +134,12 @@ tasks {
     }
 }
 
+task allTestJar(type: Jar) {
+    dependsOn testJar
+    archiveBaseName = project.name + '-test-all'
+    duplicatesStrategy = DuplicatesStrategy.EXCLUDE
+    from { configurations.testCompileClasspath.collect { it.isDirectory() ? it : zipTree(it) } }
+    from { configurations.testRuntimeClasspath.collect { it.isDirectory() ? it : zipTree(it) } }
+    from { sourceSets.test.output }
+    with jar
+}
\ No newline at end of file
diff --git a/accord-core/src/main/java/accord/api/Agent.java b/accord-core/src/main/java/accord/api/Agent.java
index 4a4460d1..f229ab82 100644
--- a/accord-core/src/main/java/accord/api/Agent.java
+++ b/accord-core/src/main/java/accord/api/Agent.java
@@ -63,6 +63,7 @@ public interface Agent extends UncaughtExceptionListener
      * outside of Accord.
      */
     default void onLocalBarrier(@Nonnull Seekables<?, ?> keysOrRanges, @Nonnull Timestamp executeAt) {}
+    void onStale(Timestamp staleSince, Ranges ranges);
 
     @Override
     void onUncaughtException(Throwable t);
diff --git a/accord-core/src/main/java/accord/api/ProgressLog.java b/accord-core/src/main/java/accord/api/ProgressLog.java
index 47244c17..d3572312 100644
--- a/accord-core/src/main/java/accord/api/ProgressLog.java
+++ b/accord-core/src/main/java/accord/api/ProgressLog.java
@@ -114,6 +114,11 @@ public interface ProgressLog
      */
     void accepted(Command command, ProgressShard shard);
 
+    /**
+     * The transaction's executeAt is known. This may be enough to satisfy some waiting transactions.
+     */
+    void precommitted(Command command);
+
     /**
      * Has committed
      *
diff --git a/accord-core/src/main/java/accord/api/Result.java b/accord-core/src/main/java/accord/api/Result.java
index 0509c476..15fbe217 100644
--- a/accord-core/src/main/java/accord/api/Result.java
+++ b/accord-core/src/main/java/accord/api/Result.java
@@ -26,14 +26,6 @@ import accord.primitives.ProgressToken;
  */
 public interface Result extends Outcome
 {
-    Result APPLIED = new Result() { };
-    
-    Result INVALIDATED = new Result()
-    {
-        @Override
-        public ProgressToken asProgressToken() { return ProgressToken.INVALIDATED; }
-    };
-
     @Override
     default ProgressToken asProgressToken() { return ProgressToken.APPLIED; }
 }
diff --git a/accord-core/src/main/java/accord/coordinate/Barrier.java b/accord-core/src/main/java/accord/coordinate/Barrier.java
index 2939c479..84885476 100644
--- a/accord-core/src/main/java/accord/coordinate/Barrier.java
+++ b/accord-core/src/main/java/accord/coordinate/Barrier.java
@@ -18,6 +18,7 @@
 
 package accord.coordinate;
 
+import java.util.Objects;
 import java.util.concurrent.ExecutionException;
 import javax.annotation.Nonnull;
 
@@ -34,7 +35,6 @@ import accord.local.PreLoadContext;
 import accord.local.SafeCommand;
 import accord.local.SafeCommandStore;
 import accord.local.SafeCommandStore.TestDep;
-import accord.local.SafeCommandStore.TestKind;
 import accord.local.SafeCommandStore.TestTimestamp;
 import accord.local.Status;
 import accord.primitives.Routable.Domain;
@@ -45,6 +45,7 @@ import accord.utils.MapReduceConsume;
 import accord.utils.async.AsyncResults;
 
 import static accord.local.PreLoadContext.contextFor;
+import static accord.primitives.Txn.Kind.Kinds.Any;
 import static accord.utils.Invariants.checkArgument;
 import static accord.utils.Invariants.checkState;
 import static accord.utils.async.AsyncChains.getUninterruptibly;
@@ -252,27 +253,28 @@ public class Barrier<S extends Seekables<?, ?>> extends AsyncResults.AbstractRes
         @Override
         public BarrierTxn apply(SafeCommandStore safeStore)
         {
-            BarrierTxn found = safeStore.mapReduceWithTerminate(
+            BarrierTxn found = safeStore.mapReduce(
                     seekables,
                     safeStore.ranges().allAfter(minEpoch),
                     // Barriers are trying to establish that committed transactions are applied before the barrier (or in this case just minEpoch)
                     // so all existing transaction types should ensure that at this point. An earlier txnid may have an executeAt that is after
                     // this barrier or the transaction we listen on and that is fine
-                    TestKind.Any,
+                    Any,
                     TestTimestamp.EXECUTES_AFTER,
                     TxnId.minForEpoch(minEpoch),
                     TestDep.ANY_DEPS,
                     null,
                     Status.Committed,
                     Status.Applied,
-                    (keyOrRange, txnId, executeAt, status, barrierTxn) -> {
+                    (p1, keyOrRange, txnId, executeAt, status, barrierTxn) -> {
                         if (keyOrRange.domain() == Domain.Key)
                             return new BarrierTxn(txnId, executeAt, status, keyOrRange.asKey());
                         return null;
                     },
                     null,
+                    null,
                     // Take the first one we find, and call it good enough to wait on
-                    barrierTxn -> barrierTxn != null);
+                    Objects::nonNull);
             // It's not applied so add a listener to find out when it is applied
             if (found != null && !found.status.equals(Status.Applied))
             {
diff --git a/accord-core/src/main/java/accord/coordinate/CheckShards.java b/accord-core/src/main/java/accord/coordinate/CheckShards.java
index d9bb0f05..656bab71 100644
--- a/accord-core/src/main/java/accord/coordinate/CheckShards.java
+++ b/accord-core/src/main/java/accord/coordinate/CheckShards.java
@@ -111,7 +111,7 @@ public abstract class CheckShards<U extends Unseekables<?>> extends ReadCoordina
     @Override
     protected void finishOnExhaustion()
     {
-        if (merged != null && merged.isTruncated()) finishOnFailure(new Truncated(txnId, null), false);
+        if (merged != null && merged.isTruncatedResponse()) finishOnFailure(new Truncated(txnId, null), false);
         else super.finishOnExhaustion();
     }
 }
diff --git a/accord-core/src/main/java/accord/coordinate/CoordinateNoOp.java b/accord-core/src/main/java/accord/coordinate/CoordinateNoOp.java
index 84c3e465..bc25373e 100644
--- a/accord-core/src/main/java/accord/coordinate/CoordinateNoOp.java
+++ b/accord-core/src/main/java/accord/coordinate/CoordinateNoOp.java
@@ -76,14 +76,16 @@ public class CoordinateNoOp extends CoordinatePreAccept<Timestamp>
         }
         else
         {
-            new Propose<Timestamp>(node, topologies, Ballot.ZERO, txnId, txn, route, executeAt, Deps.NONE, this)
+            Deps preacceptDeps = Deps.merge(successes, ok -> ok.deps);
+            new Propose<Timestamp>(node, topologies, Ballot.ZERO, txnId, txn, route, executeAt, preacceptDeps, this)
             {
                 @Override
                 void onAccepted()
                 {
                     Writes writes = txn.execute(txnId, txnId, null);
                     Result result = txn.result(txnId, executeAt, null);
-                    Apply.sendMaximal(node, txnId, route, txn, executeAt, deps, writes, result);
+                    Deps acceptDeps = Deps.merge(this.acceptOks, ok -> ok.deps);
+                    Apply.sendMaximal(node, txnId, route, txn, executeAt, acceptDeps, writes, result);
                     accept(executeAt, null);
                 }
             }.start();
diff --git a/accord-core/src/main/java/accord/coordinate/FetchData.java b/accord-core/src/main/java/accord/coordinate/FetchData.java
index 0188a0a7..1ece4675 100644
--- a/accord-core/src/main/java/accord/coordinate/FetchData.java
+++ b/accord-core/src/main/java/accord/coordinate/FetchData.java
@@ -31,7 +31,6 @@ import accord.utils.Invariants;
 import javax.annotation.Nullable;
 
 import static accord.coordinate.Infer.InvalidateAndCallback.locallyInvalidateAndCallback;
-import static accord.messages.CheckStatus.WithQuorum.NoQuorum;
 import static accord.primitives.Route.castToRoute;
 import static accord.primitives.Route.isRoute;
 
@@ -44,26 +43,27 @@ public class FetchData extends CheckShards<Route<?>>
 {
     public static Object fetch(Known fetch, Node node, TxnId txnId, Unseekables<?> someUnseekables, BiConsumer<? super Known, Throwable> callback)
     {
-        return fetch(fetch, node, txnId, someUnseekables, null, callback);
+        return fetch(fetch, node, txnId, someUnseekables, null, null, callback);
     }
 
-    public static Object fetch(Known fetch, Node node, TxnId txnId, Unseekables<?> someUnseekables, @Nullable Timestamp executeAt, BiConsumer<? super Known, Throwable> callback)
+    public static Object fetch(Known fetch, Node node, TxnId txnId, Unseekables<?> someUnseekables, @Nullable EpochSupplier forLocalEpoch, @Nullable Timestamp executeAt, BiConsumer<? super Known, Throwable> callback)
     {
-        if (someUnseekables.kind().isRoute()) return fetch(fetch, node, txnId, castToRoute(someUnseekables), executeAt, callback);
-        else return fetchViaSomeRoute(fetch, node, txnId, someUnseekables, executeAt, callback);
+        if (someUnseekables.kind().isRoute()) return fetch(fetch, node, txnId, castToRoute(someUnseekables), forLocalEpoch, executeAt, callback);
+        else return fetchViaSomeRoute(fetch, node, txnId, someUnseekables, forLocalEpoch, executeAt, callback);
     }
 
-    public static Object fetch(Known fetch, Node node, TxnId txnId, Route<?> route, @Nullable Timestamp executeAt, BiConsumer<? super Known, Throwable> callback)
+    public static Object fetch(Known fetch, Node node, TxnId txnId, Route<?> route, @Nullable EpochSupplier forLocalEpoch, @Nullable Timestamp executeAt, BiConsumer<? super Known, Throwable> callback)
     {
         long srcEpoch = fetch.fetchEpoch(txnId, executeAt);
         if (!node.topology().hasEpoch(srcEpoch))
-            return node.topology().awaitEpoch(srcEpoch).map(ignore -> fetch(fetch, node, txnId, route, executeAt, callback)).beginAsResult();
+            return node.topology().awaitEpoch(srcEpoch).map(ignore -> fetch(fetch, node, txnId, route, forLocalEpoch, executeAt, callback)).beginAsResult();
 
         Invariants.checkArgument(node.topology().hasEpoch(srcEpoch), "Unknown epoch %d, latest known is %d", srcEpoch, node.epoch());
-        Ranges ranges = node.topology().localRangesForEpochs(txnId.epoch(), srcEpoch);
+        long toLocalEpoch = Math.max(srcEpoch, forLocalEpoch == null ? 0 : forLocalEpoch.epoch());
+        Ranges ranges = node.topology().localRangesForEpochs(txnId.epoch(), toLocalEpoch);
         if (!route.covers(ranges))
         {
-            return fetchWithIncompleteRoute(fetch, node, txnId, route, executeAt, callback);
+            return fetchWithIncompleteRoute(fetch, node, txnId, route, forLocalEpoch, executeAt, callback);
         }
         else
         {
@@ -72,7 +72,7 @@ public class FetchData extends CheckShards<Route<?>>
     }
 
     @SuppressWarnings({"unchecked", "rawtypes"})
-    private static Object fetchViaSomeRoute(Known fetch, Node node, TxnId txnId, Unseekables<?> someUnseekables, @Nullable Timestamp executeAt, BiConsumer<? super Known, Throwable> callback)
+    private static Object fetchViaSomeRoute(Known fetch, Node node, TxnId txnId, Unseekables<?> someUnseekables, @Nullable EpochSupplier forLocalEpoch, @Nullable Timestamp executeAt, BiConsumer<? super Known, Throwable> callback)
     {
         return FindSomeRoute.findSomeRoute(node, txnId, someUnseekables, (foundRoute, fail) -> {
             if (fail != null) callback.accept(null, fail);
@@ -91,7 +91,7 @@ public class FetchData extends CheckShards<Route<?>>
                 Route<?> route = foundRoute.route;
                 if (isRoute(someUnseekables))
                     route = Route.merge(route, (Route)someUnseekables);
-                fetch(fetch, node, txnId, route, executeAt, callback);
+                fetch(fetch, node, txnId, route, forLocalEpoch, executeAt, callback);
             }
         });
     }
@@ -118,25 +118,27 @@ public class FetchData extends CheckShards<Route<?>>
             case Erased:
             case WasApply:
             case Apply:
+                // TODO (required): we may now be stale
                 callback.accept(found, null);
         }
     }
 
-    private static Object fetchWithIncompleteRoute(Known fetch, Node node, TxnId txnId, Route<?> someRoute, @Nullable Timestamp executeAt, BiConsumer<? super Known, Throwable> callback)
+    private static Object fetchWithIncompleteRoute(Known fetch, Node node, TxnId txnId, Route<?> someRoute, @Nullable EpochSupplier forLocalEpoch, @Nullable Timestamp executeAt, BiConsumer<? super Known, Throwable> callback)
     {
         long srcEpoch = fetch.fetchEpoch(txnId, executeAt);
         Invariants.checkArgument(node.topology().hasEpoch(srcEpoch), "Unknown epoch %d, latest known is %d", srcEpoch, node.epoch());
         return FindRoute.findRoute(node, txnId, someRoute.withHomeKey(), (foundRoute, fail) -> {
             if (fail != null) callback.accept(null, fail);
-            else if (foundRoute == null) fetchViaSomeRoute(fetch, node, txnId, someRoute, executeAt, callback);
-            else fetch(fetch, node, txnId, foundRoute.route, foundRoute.executeAt, callback);
+            else if (foundRoute == null) fetchViaSomeRoute(fetch, node, txnId, someRoute, forLocalEpoch, executeAt, callback);
+            else fetch(fetch, node, txnId, foundRoute.route, forLocalEpoch, foundRoute.executeAt, callback);
         });
     }
 
-    public static Object fetch(Known fetch, Node node, TxnId txnId, FullRoute<?> route, @Nullable Timestamp executeAt, BiConsumer<? super Known, Throwable> callback)
+    public static Object fetch(Known fetch, Node node, TxnId txnId, FullRoute<?> route, @Nullable EpochSupplier forLocalEpoch, @Nullable Timestamp executeAt, BiConsumer<? super Known, Throwable> callback)
     {
         return node.awaitEpoch(executeAt).map(ignore -> {
-            Ranges ranges = node.topology().localRangesForEpochs(txnId.epoch(), fetch.fetchEpoch(txnId, executeAt));
+            long toEpoch = Math.max(fetch.fetchEpoch(txnId, executeAt), forLocalEpoch == null ? 0 : forLocalEpoch.epoch());
+            Ranges ranges = node.topology().localRangesForEpochs(txnId.epoch(), toEpoch);
             return fetchInternal(ranges, fetch, node, txnId, route.sliceStrict(ranges), executeAt, callback);
         }).beginAsResult();
     }
@@ -200,7 +202,7 @@ public class FetchData extends CheckShards<Route<?>>
 
     protected boolean isSufficient(Route<?> scope, CheckStatus.CheckStatusOk ok)
     {
-        return target.isSatisfiedBy(((CheckStatusOkFull)ok).sufficientFor(scope.participants(), NoQuorum));
+        return target.isSatisfiedBy(ok.knownFor(scope.participants()));
     }
 
     @Override
@@ -216,6 +218,7 @@ public class FetchData extends CheckShards<Route<?>>
             if (success == ReadCoordinator.Success.Success)
                 Invariants.checkState(isSufficient(merged), "Status %s is not sufficient", merged);
 
+            // TODO (expected): should we automatically trigger a new fetch if we find executeAt but did not request enough information? would be more rob ust
             Propagate.propagate(node, txnId, sourceEpoch, success.withQuorum, route(), target, (CheckStatusOkFull) merged, callback);
         }
     }
diff --git a/accord-core/src/main/java/accord/coordinate/FindRoute.java b/accord-core/src/main/java/accord/coordinate/FindRoute.java
index f85ba888..6997c836 100644
--- a/accord-core/src/main/java/accord/coordinate/FindRoute.java
+++ b/accord-core/src/main/java/accord/coordinate/FindRoute.java
@@ -21,7 +21,6 @@ package accord.coordinate;
 import java.util.function.BiConsumer;
 
 import accord.local.Node;
-import accord.local.Status;
 import accord.messages.CheckStatus.CheckStatusOk;
 import accord.messages.CheckStatus.IncludeInfo;
 import accord.primitives.*;
@@ -47,7 +46,7 @@ public class FindRoute extends CheckShards<Route<?>>
         public Result(CheckStatusOk ok)
         {
             this.route = Route.castToFullRoute(ok.route);
-            this.executeAt = ok.saveStatus.status.compareTo(Status.PreCommitted) >= 0 ? ok.executeAt : null;
+            this.executeAt = ok.maxKnown().executeAt.isDecidedAndKnownToExecute() ? ok.executeAt : null;
         }
     }
 
diff --git a/accord-core/src/main/java/accord/coordinate/FindSomeRoute.java b/accord-core/src/main/java/accord/coordinate/FindSomeRoute.java
index 2babeac7..19060ed9 100644
--- a/accord-core/src/main/java/accord/coordinate/FindSomeRoute.java
+++ b/accord-core/src/main/java/accord/coordinate/FindSomeRoute.java
@@ -75,6 +75,6 @@ public class FindSomeRoute extends CheckShards<Unseekables<?>>
     {
         if (failure != null) callback.accept(null, failure);
         else if (merged == null) callback.accept(new Result(null, Nothing, success.withQuorum), null);
-        else callback.accept(new Result(merged.route, merged.inferredOrKnown(success.withQuorum), success.withQuorum), null);
+        else callback.accept(new Result(merged.route, merged.finish(this.route, success.withQuorum).knownFor(this.route), success.withQuorum), null);
     }
 }
diff --git a/accord-core/src/main/java/accord/coordinate/Infer.java b/accord-core/src/main/java/accord/coordinate/Infer.java
index dc42d834..ef04b8f3 100644
--- a/accord-core/src/main/java/accord/coordinate/Infer.java
+++ b/accord-core/src/main/java/accord/coordinate/Infer.java
@@ -26,11 +26,11 @@ import accord.local.Command;
 import accord.local.Commands;
 import accord.local.Node;
 import accord.local.PreLoadContext;
+import accord.local.RedundantStatus;
 import accord.local.SafeCommand;
 import accord.local.SafeCommandStore;
-import accord.local.SaveStatus;
 import accord.local.Status;
-import accord.messages.CheckStatus;
+import accord.local.Status.Known;
 import accord.primitives.Participants;
 import accord.primitives.Ranges;
 import accord.primitives.Route;
@@ -40,17 +40,164 @@ import accord.primitives.Unseekables;
 import accord.utils.Invariants;
 import accord.utils.MapReduceConsume;
 
+import static accord.coordinate.Infer.InvalidIf.IfPreempted;
+import static accord.coordinate.Infer.InvalidIf.IfQuorum;
+import static accord.coordinate.Infer.InvalidIf.NotKnown;
+import static accord.coordinate.Infer.InvalidIfNot.IfUnknown;
+import static accord.coordinate.Infer.InvalidIfNot.IfUndecided;
 import static accord.local.PreLoadContext.contextFor;
 import static accord.local.Status.Durability.Majority;
-import static accord.local.Status.PreAccepted;
 import static accord.local.Status.PreApplied;
 import static accord.local.Status.PreCommitted;
-import static accord.messages.CheckStatus.WithQuorum.HasQuorum;
 import static accord.primitives.Route.castToRoute;
 import static accord.primitives.Route.isRoute;
 
 public class Infer
 {
+    public enum InvalidIfNot
+    {
+        /**
+         * There is no information to suggest the command is invalid
+         */
+        NotKnownToBeInvalid(NotKnown, NotKnown),
+
+        /**
+         * If the command has not been preaccepted on a majority of any shard and
+         * the command's original coordinator had been preempted prior to all responses we rely upon
+         * (so we are not racing with it)
+         */
+        IfUnknownAndPreempted(IfPreempted, NotKnown),
+
+        /**
+         * If the command has not had its execution timestamp agreed on any shard and
+         * the command's original coordinator had been preempted prior to all responses we rely upon
+         * (so we are not racing with it)
+         */
+        IfUndecidedAndPreempted(IfPreempted, IfPreempted),
+
+        /**
+         * If the command has not been preaccepted on a majority of any shard
+         */
+        IfUnknown(IfQuorum, NotKnown),
+
+        /**
+         * If the command has not had its execution timestamp agreed on any shard and
+         * the command's original coordinator had been preempted prior to all responses we rely upon
+         * (so we are not racing with it)
+         */
+        IfUnknownOrIfUndecidedAndPreempted(IfQuorum, IfPreempted),
+
+        /**
+         * If the command has not had its execution timestamp agreed on any shard
+         */
+        IfUndecided(IfQuorum, IfQuorum);
+
+        final InvalidIf unknown, undecided;
+        private static final InvalidIfNot[] LOOKUP;
+        private static final int invalidIfs = InvalidIf.values().length;
+
+        static
+        {
+            LOOKUP = new InvalidIfNot[invalidIfs * invalidIfs];
+            InvalidIfNot[] invalidIfNot = InvalidIfNot.values();
+            for (InvalidIfNot ifNot : invalidIfNot)
+                LOOKUP[ifNot.unknown.ordinal() * invalidIfs + ifNot.undecided.ordinal()] = ifNot;
+        }
+
+        InvalidIfNot(InvalidIf unknown, InvalidIf undecided)
+        {
+            this.unknown = unknown;
+            this.undecided = undecided;
+        }
+
+        public static boolean isMax(InvalidIfNot that)
+        {
+            return that == IfUndecided;
+        }
+
+        public InvalidIfNot atLeast(InvalidIfNot that)
+        {
+            return lookup(atLeast(this.unknown, that.unknown), atLeast(this.undecided, that.undecided));
+        }
+
+        public InvalidIfNot reduce(InvalidIfNot that)
+        {
+            return lookup(reduce(this.unknown, that.unknown), reduce(this.undecided, that.undecided));
+        }
+
+        private InvalidIfNot lookup(InvalidIf unknown, InvalidIf undecided)
+        {
+            return LOOKUP[unknown.ordinal() * invalidIfs + undecided.ordinal()];
+        }
+
+        private static InvalidIf atLeast(InvalidIf a, InvalidIf b)
+        {
+            if (a == b) return a;
+            return IfPreempted;
+        }
+
+        private static InvalidIf reduce(InvalidIf a, InvalidIf b)
+        {
+            return a.compareTo(b) <= 0 ? a : b;
+        }
+
+        public boolean inferInvalidWithQuorum(IsPreempted isPreempted, Known known)
+        {
+            return inferInvalidWithQuorum(undecided, isPreempted, !known.isDecided())
+                   || inferInvalidWithQuorum(unknown, isPreempted, !known.hasDefinitionBeenKnown());
+        }
+
+        private static boolean inferInvalidWithQuorum(InvalidIf invalidIf, IsPreempted isPreempted, boolean hasCondition)
+        {
+            if (!hasCondition)
+                return false;
+
+            switch (invalidIf)
+            {
+                default: throw new AssertionError("Unhandled InvalidIf: " + invalidIf);
+                case NotKnown: break;
+                case IfQuorum: return true;
+                case IfPreempted:
+                    if (isPreempted == IsPreempted.Preempted)
+                        return true;
+            }
+
+            return false;
+        }
+    }
+
+    enum InvalidIf
+    {
+        NotKnown,
+
+        /**
+         * We did not have a quorum of responses with the associated lower bound, so we require that the command has been preempted at a quorum
+         */
+        IfPreempted,
+
+        /**
+         * If we obtain a quorum of responses with the associated lower bound, we can infer the command is invalidated if it has not been witnessed at the lower bound
+         */
+        IfQuorum
+    }
+
+    // only valid with a quorum of responses
+    public enum IsPreempted
+    {
+        NotPreempted, MaybePreempted, Preempted;
+
+        public IsPreempted merge(IsPreempted that)
+        {
+            if (this == that) return this;
+            return MaybePreempted;
+        }
+
+        public IsPreempted validForBoth(IsPreempted that)
+        {
+            return this.compareTo(that) <= 0 ? this : that;
+        }
+    }
+
     private static abstract class CleanupAndCallback<T> implements MapReduceConsume<SafeCommandStore, Void>
     {
         final Node node;
@@ -121,36 +268,39 @@ public class Infer
         }
     }
 
-    static class EraseNonParticipatingAndCallback<T> extends CleanupAndCallback<T>
+    /**
+     * Erase if it is safe to do so, i.e. if Infer.safeToCleanup permits it.
+     */
+    static class SafeEraseAndCallback<T> extends CleanupAndCallback<T>
     {
-        private EraseNonParticipatingAndCallback(Node node, TxnId txnId, Unseekables<?> someUnseekables, T param, BiConsumer<T, Throwable> callback)
+        private SafeEraseAndCallback(Node node, TxnId txnId, Unseekables<?> someUnseekables, T param, BiConsumer<T, Throwable> callback)
         {
             super(node, txnId, someUnseekables, param, callback);
         }
 
-        public static <T> void eraseNonParticipatingAndCallback(Node node, TxnId txnId, Unseekables<?> someUnseekables, T param, BiConsumer<T, Throwable> callback)
+        public static <T> void safeEraseAndCallback(Node node, TxnId txnId, Unseekables<?> someUnseekables, T param, BiConsumer<T, Throwable> callback)
         {
             if (!Route.isRoute(someUnseekables)) callback.accept(param, null);
-            else new EraseNonParticipatingAndCallback<>(node, txnId, someUnseekables, param, callback).start();
+            else new SafeEraseAndCallback<>(node, txnId, someUnseekables, param, callback).start();
         }
 
         @Override
         Void apply(SafeCommandStore safeStore, SafeCommand safeCommand)
         {
-            // we're applying an invalidation, so the record will not be cleaned up until the whole range is truncated
             Command command = safeCommand.current();
+            // TODO (required): introduce a special form of Erased where we do not imply the phase is "Cleanup"
             if (!command.hasBeen(PreApplied) && safeToCleanup(safeStore, command, Route.castToRoute(someUnseekables), null))
                 Commands.setErased(safeStore, safeCommand);
             return null;
         }
     }
 
-    public static Status invalidIfNotAtLeast(SafeCommandStore safeStore, TxnId txnId, Unseekables<?> query)
+    public static InvalidIfNot invalidIfNotAtLeast(SafeCommandStore safeStore, TxnId txnId, Unseekables<?> query)
     {
         if (safeStore.commandStore().globalDurability(txnId).compareTo(Majority) >= 0)
         {
             Unseekables<?> preacceptsWith = isRoute(query) ? castToRoute(query).withHomeKey() : query;
-            return safeStore.commandStore().isRejectedIfNotPreAccepted(txnId, preacceptsWith) ? PreAccepted : PreCommitted;
+            return safeStore.commandStore().isRejectedIfNotPreAccepted(txnId, preacceptsWith) ? IfUnknown : IfUndecided;
         }
 
         // TODO (expected, consider): should we force this to be a Route or a Participants?
@@ -159,49 +309,51 @@ public class Infer
             Participants<?> participants = castToRoute(query).participants();
             // TODO (desired): limit to local participants to avoid O(n2) work across cluster
             if (safeStore.commandStore().durableBefore().isSomeShardDurable(txnId, participants, Majority))
-                return PreCommitted;
+                return IfUndecided;
         }
 
         if (safeStore.commandStore().durableBefore().isUniversal(txnId, safeStore.ranges().allAt(txnId.epoch())))
-            return PreCommitted;
+            return IfUndecided;
 
-        return Status.NotDefined;
+        return InvalidIfNot.NotKnownToBeInvalid;
     }
 
-    public static boolean inferInvalidated(CheckStatus.WithQuorum withQuorum, Status invalidIfNotAtLeast, SaveStatus saveStatus, SaveStatus maxSaveStatus)
+    public enum YesNoMaybe
     {
-        if (saveStatus == SaveStatus.Invalidated)
-            return true;
-
-        if (withQuorum != HasQuorum)
-            return false;
-
-        // should not be possible to reach a quorum without finding the definition unless cleanup is in progress
-        Invariants.checkState(saveStatus != SaveStatus.Accepted || maxSaveStatus.phase == Status.Phase.Cleanup);
-
-        if (saveStatus.status.compareTo(invalidIfNotAtLeast) < 0)
-            return true;
-
-        return invalidIfNotAtLeast.compareTo(PreAccepted) >= 0 && saveStatus == SaveStatus.AcceptedInvalidate;
+        No, Maybe, Yes
     }
 
     public static boolean safeToCleanup(SafeCommandStore safeStore, Command command, Route<?> fetchedWith, @Nullable Timestamp executeAt)
     {
         Invariants.checkArgument(fetchedWith != null || command.route() != null);
         TxnId txnId = command.txnId();
-        if (command.is(Status.NotDefined))
-            return !command.saveStatus().isUninitialised();
+        if (command.route() == null || !fetchedWith.covers(safeStore.ranges().allAt(txnId.epoch())))
+            return false;
 
-        if (command.route() != null || fetchedWith.covers(safeStore.ranges().allAt(txnId.epoch())))
-        {
-            Route<?> route = command.route();
-            if (route == null) route = fetchedWith;
+        Route<?> route = command.route();
+        if (route == null) route = fetchedWith;
+
+        // TODO (required): is it safe to cleanup without an executeAt?
+        executeAt = command.executeAtIfKnown(Timestamp.nonNullOrMax(executeAt, txnId));
+        Ranges coordinateRanges = safeStore.ranges().coordinates(txnId);
+        Ranges acceptRanges = executeAt.epoch() == txnId.epoch() ? coordinateRanges : safeStore.ranges().allBetween(txnId, executeAt);
+        if (!route.participatesIn(coordinateRanges) && !route.participatesIn(acceptRanges))
+            return true;
 
-            executeAt = command.executeAtIfKnown(Timestamp.nonNullOrMax(executeAt, txnId));
-            Ranges coordinateRanges = safeStore.ranges().coordinates(txnId);
-            Ranges acceptRanges = executeAt.epoch() == txnId.epoch() ? coordinateRanges : safeStore.ranges().allBetween(txnId, executeAt);
-            return !route.participatesIn(coordinateRanges) && !route.participatesIn(acceptRanges);
+        RedundantStatus status = safeStore.commandStore().redundantBefore().status(txnId, executeAt, route.participants());
+        switch (status)
+        {
+            default: throw new AssertionError("Unhandled RedundantStatus: " + status);
+            case NOT_OWNED:
+            case LIVE:
+            case REDUNDANT_PRE_BOOTSTRAP_OR_STALE:
+            case PARTIALLY_PRE_BOOTSTRAP_OR_STALE:
+                return false;
+            case LOCALLY_REDUNDANT:
+            case SHARD_REDUNDANT:
+                Invariants.checkState(!command.hasBeen(PreCommitted));
+            case PRE_BOOTSTRAP_OR_STALE:
+                return true;
         }
-        return false;
     }
 }
diff --git a/accord-core/src/main/java/accord/coordinate/MaybeRecover.java b/accord-core/src/main/java/accord/coordinate/MaybeRecover.java
index ab4ea919..48c3754f 100644
--- a/accord-core/src/main/java/accord/coordinate/MaybeRecover.java
+++ b/accord-core/src/main/java/accord/coordinate/MaybeRecover.java
@@ -21,7 +21,6 @@ package accord.coordinate;
 import java.util.function.BiConsumer;
 
 import accord.local.Status.Known;
-import accord.messages.CheckStatus.WithQuorum;
 import accord.primitives.*;
 import accord.utils.Invariants;
 
@@ -29,9 +28,8 @@ import accord.local.Node;
 import accord.messages.CheckStatus.CheckStatusOk;
 import accord.messages.CheckStatus.IncludeInfo;
 
-import static accord.coordinate.Infer.EraseNonParticipatingAndCallback.eraseNonParticipatingAndCallback;
+import static accord.coordinate.Infer.SafeEraseAndCallback.safeEraseAndCallback;
 import static accord.coordinate.Infer.InvalidateAndCallback.locallyInvalidateAndCallback;
-import static accord.utils.Functions.reduceNonNull;
 
 /**
  * A result of null indicates the transaction is globally persistent
@@ -79,45 +77,47 @@ public class MaybeRecover extends CheckShards<Route<?>>
         else
         {
             Invariants.checkState(merged != null);
-            Known known = merged.saveStatus.known;
-            Route<?> someRoute = reduceNonNull(Route::union, (Route)this.route, merged.route);
+            CheckStatusOk full = merged.finish(this.route, success.withQuorum);
+            Known known = full.maxKnown();
+            Route<?> someRoute = full.route;
 
             switch (known.outcome)
             {
                 default: throw new AssertionError();
                 case Unknown:
-                    if (known.canProposeInvalidation() && !Route.isFullRoute(merged.route))
+                    if (known.canProposeInvalidation() && !Route.isFullRoute(full.route))
                     {
                         // for correctness reasons, we have not necessarily preempted the initial pre-accept round and
                         // may have raced with it, so we must attempt to recover anything we see pre-accepted.
                         Invalidate.invalidate(node, txnId, someRoute, callback);
                         break;
                     }
+                    // fall through otherwise to recovery
 
                 case Apply:
                     // we have included the home key, and one that witnessed the definition has responded, so it should also know the full route
-                    if (hasMadeProgress(merged))
+                    if (hasMadeProgress(full))
                     {
-                        callback.accept(merged.toProgressToken(), null);
+                        callback.accept(full.toProgressToken(), null);
                     }
                     else
                     {
-                        Invariants.checkState(Route.isFullRoute(merged.route), "Require a full route but given %s", merged.route);
-                        node.recover(txnId, Route.castToFullRoute(merged.route)).addCallback(callback);
+                        Invariants.checkState(Route.isFullRoute(someRoute), "Require a full route but given %s", full.route);
+                        node.recover(txnId, Route.castToFullRoute(someRoute)).addCallback(callback);
                     }
                     break;
 
                 case WasApply:
-                    callback.accept(merged.toProgressToken(), null);
+                    callback.accept(full.toProgressToken(), null);
                     break;
 
                 case Invalidated:
-                    locallyInvalidateAndCallback(node, txnId, someRoute, merged.toProgressToken(), callback);
+                    locallyInvalidateAndCallback(node, txnId, someRoute, full.toProgressToken(), callback);
+                    break;
 
                 case Erased:
-                    WithQuorum withQuorum = success.withQuorum;
-                    if (!merged.inferInvalidated(withQuorum)) eraseNonParticipatingAndCallback(node, txnId, someRoute, merged.toProgressToken(), callback);
-                    else locallyInvalidateAndCallback(node, txnId, someRoute, merged.toProgressToken(), callback);
+                    Invariants.checkState(!full.knownFor(route.participants()).isInvalidated());
+                    safeEraseAndCallback(node, txnId, someRoute, full.toProgressToken(), callback);
             }
         }
     }
diff --git a/accord-core/src/main/java/accord/coordinate/ReadCoordinator.java b/accord-core/src/main/java/accord/coordinate/ReadCoordinator.java
index 4cb55e83..d8f1bfb0 100644
--- a/accord-core/src/main/java/accord/coordinate/ReadCoordinator.java
+++ b/accord-core/src/main/java/accord/coordinate/ReadCoordinator.java
@@ -144,11 +144,7 @@ public abstract class ReadCoordinator<Reply extends accord.messages.Reply> exten
                 break;
 
             case ApprovePartial:
-                Ranges unavailable = unavailable(reply);
-                RequestStatus result = recordPartialReadSuccess(from, unavailable);
-                if (result == RequestStatus.Failed && failure == null)
-                    failure = new RangeUnavailable(unavailable, txnId);
-                handle(result);
+                handle(recordPartialReadSuccess(from, unavailable(reply)));
                 break;
         }
     }
@@ -177,11 +173,9 @@ public abstract class ReadCoordinator<Reply extends accord.messages.Reply> exten
     @Override
     public void onCallbackFailure(Id from, Throwable failure)
     {
+        node.agent().onUncaughtException(failure);
         if (isDone)
-        {
-            node.agent().onUncaughtException(failure);
             return;
-        }
 
         if (this.failure != null)
             failure.addSuppressed(this.failure);
@@ -218,7 +212,18 @@ public abstract class ReadCoordinator<Reply extends accord.messages.Reply> exten
     {
         Invariants.checkState(!isDone);
         if (failure == null)
-            failure = new Exhausted(txnId, null);
+        {
+            Ranges unavailable = Ranges.EMPTY;
+            Ranges exhausted = Ranges.EMPTY;
+            for (ReadShardTracker tracker : trackers)
+            {
+                if (tracker.hasSucceeded())
+                    continue;
+                if (tracker.unavailable() != null) unavailable = unavailable.with(tracker.unavailable());
+                else exhausted = exhausted.with(Ranges.of(tracker.shard.range));
+            }
+            failure = new Exhausted(txnId, null, (unavailable.isEmpty() ? "" : "unavailable: " + unavailable + (exhausted.isEmpty() ? "" : "; ")) + (exhausted.isEmpty() ? "" : "no response: " + exhausted));
+        }
         finishOnFailure();
     }
 
diff --git a/accord-core/src/main/java/accord/coordinate/Recover.java b/accord-core/src/main/java/accord/coordinate/Recover.java
index df56b791..add3e88a 100644
--- a/accord-core/src/main/java/accord/coordinate/Recover.java
+++ b/accord-core/src/main/java/accord/coordinate/Recover.java
@@ -139,6 +139,9 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
 
     private Recover(Node node, Ballot ballot, TxnId txnId, Txn txn, FullRoute<?> route, BiConsumer<Outcome, Throwable> callback, Topologies topologies)
     {
+        // TODO (required, correctness): we may have to contact all epochs to ensure we spot any future transaction that might not have taken us as dependency?
+        //    or we need an exclusive sync point covering us and closing out the old epoch before recovering;
+        //    or we need to manage dependencies for ranges we don't own in future epochs; this might be simplest
         this.node = node;
         this.ballot = ballot;
         this.txnId = txnId;
diff --git a/accord-core/src/main/java/accord/coordinate/RecoverWithRoute.java b/accord-core/src/main/java/accord/coordinate/RecoverWithRoute.java
index dfcc3f0c..32a0797c 100644
--- a/accord-core/src/main/java/accord/coordinate/RecoverWithRoute.java
+++ b/accord-core/src/main/java/accord/coordinate/RecoverWithRoute.java
@@ -33,19 +33,19 @@ import accord.primitives.Ballot;
 import accord.primitives.Deps;
 import accord.primitives.FullRoute;
 import accord.primitives.PartialRoute;
+import accord.primitives.Participants;
 import accord.primitives.Ranges;
 import accord.primitives.Route;
 import accord.primitives.Txn;
 import accord.primitives.TxnId;
-import accord.primitives.Unseekables;
 import accord.topology.Topologies;
 import accord.utils.Invariants;
 import javax.annotation.Nullable;
 
+import static accord.local.Status.Durability.Majority;
+import static accord.local.Status.KnownDeps.DepsKnown;
 import static accord.local.Status.KnownExecuteAt.ExecuteAtKnown;
 import static accord.local.Status.Outcome.Apply;
-import static accord.messages.CheckStatus.WithQuorum.HasQuorum;
-import static accord.messages.CheckStatus.WithQuorum.NoQuorum;
 import static accord.primitives.ProgressToken.APPLIED;
 import static accord.primitives.ProgressToken.INVALIDATED;
 import static accord.primitives.ProgressToken.TRUNCATED;
@@ -120,7 +120,7 @@ public class RecoverWithRoute extends CheckShards<FullRoute<?>>
     protected boolean isSufficient(Route<?> route, CheckStatusOk ok)
     {
         CheckStatusOkFull full = (CheckStatusOkFull)ok;
-        Known sufficientTo = full.sufficientFor(route.participants(), NoQuorum);
+        Known sufficientTo = full.knownFor(route.participants());
         if (!sufficientTo.isDefinitionKnown())
             return false;
 
@@ -140,8 +140,8 @@ public class RecoverWithRoute extends CheckShards<FullRoute<?>>
             return;
         }
 
-        CheckStatusOkFull full = (CheckStatusOkFull) this.merged;
-        Known known = full.sufficientFor(route.participants(), success == Success.Quorum ? HasQuorum : NoQuorum);
+        CheckStatusOkFull full = ((CheckStatusOkFull) this.merged).finish(route, success.withQuorum);
+        Known known = full.knownFor(route.participants());
 
         switch (known.outcome)
         {
@@ -152,38 +152,61 @@ public class RecoverWithRoute extends CheckShards<FullRoute<?>>
                     Txn txn = full.partialTxn.reconstitute(route);
                     Recover.recover(node, txnId, txn, route, callback);
                 }
-                else
+                else if (!known.definition.isOrWasKnown())
                 {
+                    // TODO (required): this logic should be put in Infer, alongside any similar inferences in Recover
                     if (witnessedByInvalidation != null && witnessedByInvalidation.compareTo(Status.PreAccepted) > 0)
                         throw new IllegalStateException("We previously invalidated, finding a status that should be recoverable");
                     Invalidate.invalidate(node, txnId, route, witnessedByInvalidation != null, callback);
                 }
+                else
+                {
+                    callback.accept(full.toProgressToken(), null);
+                }
                 break;
 
             case WasApply:
             case Apply:
-                if (!known.isDefinitionKnown() || known.executeAt != ExecuteAtKnown || known.outcome != Apply)
+                if (!known.isDefinitionKnown())
                 {
-                    if (!full.truncated.isEmpty() && known.executeAt == ExecuteAtKnown)
+                    // TODO (expected): if we determine new durability, propagate it
+                    CheckStatusOkFull propagate;
+                    if (full.isTruncatedResponse())
                     {
                         // we might have only part of the full transaction, and a shard may have truncated;
                         // in this case we want to skip straight to apply, but only for the shards that haven't truncated
-                        Unseekables<?> sendTo = route.subtract(full.truncated);
+                        Participants<?> sendTo = route.participants().subtract(full.truncatedResponse());
                         if (!sendTo.isEmpty())
                         {
-                            Invariants.checkState(full.committedDeps.covering.containsAll(sendTo));
-                            Invariants.checkState(full.partialTxn.covering().containsAll(sendTo));
-                            Persist.persistPartialMaximal(node, txnId, sendTo, route, full.partialTxn, full.executeAt, full.committedDeps, full.writes, full.result);
+                            known = full.knownFor(sendTo);
+                            // we might not know the Apply outcome because we might have raced with truncation on one shard, and the original apply on another,
+                            // so that we know to WasApply, but not
+                            if (known.executeAt == ExecuteAtKnown && known.deps == DepsKnown && known.outcome == Apply)
+                            {
+                                Invariants.checkState(full.committedDeps.covering.containsAll(sendTo));
+                                Invariants.checkState(full.partialTxn.covering().containsAll(sendTo));
+                                Persist.persistPartialMaximal(node, txnId, sendTo, route, full.partialTxn, full.executeAt, full.committedDeps, full.writes, full.result);
+                            }
+                            propagate = full;
                         }
+                        else
+                        {
+                            // TODO (expected): tighten up / centralise this implication, perhaps in Infer
+                            propagate = full.merge(Majority);
+                        }
+                    }
+                    else
+                    {
+                        propagate = full;
                     }
 
-                    Propagate.propagate(node, txnId, sourceEpoch, success.withQuorum, route, null, full, (s, f) -> callback.accept(f == null ? full.toProgressToken() : null, f));
+                    Propagate.propagate(node, txnId, sourceEpoch, success.withQuorum, route, null, propagate, (s, f) -> callback.accept(f == null ? propagate.toProgressToken() : null, f));
                     break;
                 }
 
                 // TODO (required): might not be able to fully recover transaction - may only have enough for local shard
                 Txn txn = full.partialTxn.reconstitute(route);
-                if (known.deps.hasDecidedDeps())
+                if (known.executeAt.isDecidedAndKnownToExecute() && known.deps.hasDecidedDeps() && known.outcome == Apply)
                 {
                     Deps deps = full.committedDeps.reconstitute(route());
                     node.withEpoch(full.executeAt.epoch(), () -> {
diff --git a/accord-core/src/main/java/accord/coordinate/RecoverWithSomeRoute.java b/accord-core/src/main/java/accord/coordinate/RecoverWithSomeRoute.java
index d893ae80..a5d4883b 100644
--- a/accord-core/src/main/java/accord/coordinate/RecoverWithSomeRoute.java
+++ b/accord-core/src/main/java/accord/coordinate/RecoverWithSomeRoute.java
@@ -89,7 +89,7 @@ public class RecoverWithSomeRoute extends CheckShards<Route<?>> implements BiCon
                     callback.accept(null, new IllegalStateException());
                     return;
                 case Quorum:
-                    if (merged != null && merged.isTruncated())
+                    if (merged != null && merged.isTruncatedResponse())
                     {
                         callback.accept(ProgressToken.TRUNCATED, null);
                     }
diff --git a/accord-core/src/main/java/accord/coordinate/tracking/ReadTracker.java b/accord-core/src/main/java/accord/coordinate/tracking/ReadTracker.java
index bc8cbf18..55639458 100644
--- a/accord-core/src/main/java/accord/coordinate/tracking/ReadTracker.java
+++ b/accord-core/src/main/java/accord/coordinate/tracking/ReadTracker.java
@@ -193,6 +193,11 @@ public class ReadTracker extends AbstractTracker<ReadTracker.ReadShardTracker>
         {
             return hasData;
         }
+
+        public Ranges unavailable()
+        {
+            return unavailable;
+        }
     }
 
     // TODO (required): abstract the candidate selection process so the implementation may prioritise based on distance/health etc
diff --git a/accord-core/src/main/java/accord/impl/AbstractFetchCoordinator.java b/accord-core/src/main/java/accord/impl/AbstractFetchCoordinator.java
index 0f03927c..36bd6502 100644
--- a/accord-core/src/main/java/accord/impl/AbstractFetchCoordinator.java
+++ b/accord-core/src/main/java/accord/impl/AbstractFetchCoordinator.java
@@ -155,6 +155,7 @@ public abstract class AbstractFetchCoordinator extends FetchCoordinator
                             default: throw new AssertionError("Unhandled enum: " + reply);
                             case Invalid:
                             case Redundant:
+                                // TODO (expected): stop fetch sync points from garbage collecting too quickly
                                 throw new AssertionError(String.format("Unexpected reply: %s", reply));
                         }
                     }
diff --git a/accord-core/src/main/java/accord/impl/AbstractSafeCommandStore.java b/accord-core/src/main/java/accord/impl/AbstractSafeCommandStore.java
index 955e686b..aa053409 100644
--- a/accord-core/src/main/java/accord/impl/AbstractSafeCommandStore.java
+++ b/accord-core/src/main/java/accord/impl/AbstractSafeCommandStore.java
@@ -38,6 +38,7 @@ import accord.primitives.RoutableKey;
 import accord.primitives.Seekable;
 import accord.primitives.Seekables;
 import accord.primitives.TxnId;
+import accord.utils.Invariants;
 
 public abstract class AbstractSafeCommandStore<CommandType extends SafeCommand, CommandsForKeyType extends SafeCommandsForKey> extends SafeCommandStore
 {
@@ -119,8 +120,8 @@ public abstract class AbstractSafeCommandStore<CommandType extends SafeCommand,
         else
         {
             RedundantBefore.Entry entry = commandStore().redundantBefore().get(key.toUnseekable());
-            if (entry != null && cfk.current().hasRedundant(entry.redundantBefore))
-                cfk.set(cfk.current().withoutRedundant(entry.redundantBefore));
+            if (entry != null && cfk.current().hasRedundant(entry.shardRedundantBefore()))
+                cfk.set(cfk.current().withoutRedundant(entry.shardRedundantBefore()));
         }
         return cfk;
     }
@@ -128,8 +129,7 @@ public abstract class AbstractSafeCommandStore<CommandType extends SafeCommand,
     public CommandsForKeyType commandsForKey(RoutableKey key)
     {
         CommandsForKeyType cfk = getCommandsForKeyInternal(key);
-        if (cfk == null)
-            throw new IllegalStateException(String.format("%s was not specified in PreLoadContext", key));
+        Invariants.checkState(cfk != null, "%s was not specified in PreLoadContext", key);
         if (cfk.isEmpty())
             cfk.initialize(cfkLoader(key));
         return cfk;
diff --git a/accord-core/src/main/java/accord/impl/CommandTimeseries.java b/accord-core/src/main/java/accord/impl/CommandTimeseries.java
index 66d26eed..abc319b8 100644
--- a/accord-core/src/main/java/accord/impl/CommandTimeseries.java
+++ b/accord-core/src/main/java/accord/impl/CommandTimeseries.java
@@ -26,7 +26,6 @@ import java.util.function.Predicate;
 import java.util.stream.Stream;
 import javax.annotation.Nullable;
 
-import com.google.common.base.Predicates;
 import com.google.common.collect.ImmutableSortedMap;
 
 import accord.api.Key;
@@ -36,6 +35,7 @@ import accord.local.SaveStatus;
 import accord.local.Status;
 import accord.primitives.Seekable;
 import accord.primitives.Timestamp;
+import accord.primitives.Txn.Kind.Kinds;
 import accord.primitives.TxnId;
 
 import static accord.local.SafeCommandStore.TestDep.ANY_DEPS;
@@ -117,20 +117,10 @@ public class CommandTimeseries<D>
      * <p>
      * TODO (expected, efficiency): TestDep should be asynchronous; data should not be kept memory-resident as only used for recovery
      */
-    public <T> T mapReduce(SafeCommandStore.TestKind testKind, TestTimestamp testTimestamp, Timestamp timestamp,
+    public <P1, T> T mapReduce(Kinds testKind, TestTimestamp testTimestamp, Timestamp timestamp,
                            SafeCommandStore.TestDep testDep, @Nullable TxnId depId,
                            @Nullable Status minStatus, @Nullable Status maxStatus,
-                           SafeCommandStore.CommandFunction<T, T> map, T initialValue, T terminalValue)
-    {
-        return mapReduceWithTerminate(testKind, testTimestamp, timestamp,
-                         testDep, depId,
-                         minStatus, maxStatus,
-                         map, initialValue, Predicates.equalTo(terminalValue));
-    }
-    public <T> T mapReduceWithTerminate(SafeCommandStore.TestKind testKind, TestTimestamp testTimestamp, Timestamp timestamp,
-                           SafeCommandStore.TestDep testDep, @Nullable TxnId depId,
-                           @Nullable Status minStatus, @Nullable Status maxStatus,
-                           SafeCommandStore.CommandFunction<T, T> map, T initialValue, Predicate<T> terminatePredicate)
+                           SafeCommandStore.CommandFunction<P1, T, T> map, P1 p1, T initialValue, Predicate<? super T> terminatePredicate)
     {
 
         for (D data : (testTimestamp == TestTimestamp.BEFORE ? commands.headMap(timestamp, false) : commands.tailMap(timestamp, false)).values())
@@ -147,7 +137,7 @@ public class CommandTimeseries<D>
             if (testDep != ANY_DEPS && (!status.known.deps.hasProposedOrDecidedDeps() || (deps.contains(depId) != (testDep == WITH))))
                 continue;
             Timestamp executeAt = loader.executeAt(data);
-            initialValue = map.apply(keyOrRange, txnId, executeAt, status.status, initialValue);
+            initialValue = map.apply(p1, keyOrRange, txnId, executeAt, status.status, initialValue);
             if (terminatePredicate.test(initialValue))
                 break;
         }
diff --git a/accord-core/src/main/java/accord/impl/ErasedSafeCommand.java b/accord-core/src/main/java/accord/impl/ErasedSafeCommand.java
index e9eea9b8..48cde3be 100644
--- a/accord-core/src/main/java/accord/impl/ErasedSafeCommand.java
+++ b/accord-core/src/main/java/accord/impl/ErasedSafeCommand.java
@@ -26,7 +26,7 @@ import accord.local.SaveStatus;
 import accord.primitives.TxnId;
 
 import static accord.local.Listeners.Immutable.EMPTY;
-import static accord.local.Status.Durability.DurableOrInvalidated;
+import static accord.local.Status.Durability.UniversalOrInvalidated;
 
 public class ErasedSafeCommand extends SafeCommand
 {
@@ -35,7 +35,7 @@ public class ErasedSafeCommand extends SafeCommand
     public ErasedSafeCommand(TxnId txnId)
     {
         super(txnId);
-        this.erased = new Command.Truncated(txnId, SaveStatus.Erased, DurableOrInvalidated, null, null, EMPTY, null, null);
+        this.erased = new Command.Truncated(txnId, SaveStatus.Erased, UniversalOrInvalidated, null, null, EMPTY, null, null);
     }
 
     @Override
diff --git a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
index fe47f27f..1a970c91 100644
--- a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
+++ b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.NavigableMap;
@@ -63,6 +64,7 @@ import accord.local.Status;
 import accord.primitives.AbstractKeys;
 import accord.primitives.Deps;
 import accord.primitives.PartialDeps;
+import accord.primitives.Participants;
 import accord.primitives.Range;
 import accord.primitives.Ranges;
 import accord.primitives.Routable;
@@ -71,6 +73,7 @@ import accord.primitives.Routables;
 import accord.primitives.Seekable;
 import accord.primitives.Seekables;
 import accord.primitives.Timestamp;
+import accord.primitives.Txn.Kind.Kinds;
 import accord.primitives.TxnId;
 import accord.utils.Invariants;
 import accord.utils.async.AsyncChain;
@@ -80,6 +83,8 @@ import static accord.local.Command.NotDefined.uninitialised;
 
 import static accord.local.SafeCommandStore.TestDep.ANY_DEPS;
 import static accord.local.SafeCommandStore.TestDep.WITH;
+import static accord.local.SaveStatus.Applying;
+import static accord.local.SaveStatus.ReadyToExecute;
 import static accord.local.Status.Committed;
 import static accord.local.Status.Truncated;
 import static accord.primitives.Routables.Slice.Minimal;
@@ -87,8 +92,10 @@ import static accord.primitives.Routables.Slice.Minimal;
 public abstract class InMemoryCommandStore extends CommandStore
 {
     private static final Logger logger = LoggerFactory.getLogger(InMemoryCommandStore.class);
+    private static final boolean CHECK_DEPENDENCY_INVARIANTS = false;
 
     final NavigableMap<TxnId, GlobalCommand> commands = new TreeMap<>();
+    final NavigableMap<Timestamp, GlobalCommand> commandsByExecuteAt = new TreeMap<>();
     private final NavigableMap<RoutableKey, GlobalCommandsForKey> commandsForKey = new TreeMap<>();
     // TODO (find library, efficiency): this is obviously super inefficient, need some range map
 
@@ -103,6 +110,11 @@ public abstract class InMemoryCommandStore extends CommandStore
         super(id, time, agent, store, progressLogFactory, epochUpdateHolder);
     }
 
+    protected boolean canExposeUnloaded()
+    {
+        return true;
+    }
+
     @Override
     public Agent agent()
     {
@@ -126,7 +138,80 @@ public abstract class InMemoryCommandStore extends CommandStore
 
     public GlobalCommand command(TxnId txnId)
     {
-        return commands.computeIfAbsent(txnId, GlobalCommand::new);
+        return commands.computeIfAbsent(txnId, this::newGlobalCommand);
+    }
+
+    public void onInitialise(GlobalCommand newGlobalCommand)
+    {
+        if (CHECK_DEPENDENCY_INVARIANTS)
+        {
+
+            newGlobalCommand.addListener(new Command.TransientListener()
+            {
+                @Override
+                public void onChange(SafeCommandStore safeStore, SafeCommand safeCommand)
+                {
+                    Command cur = safeCommand.current();
+                    if (cur.saveStatus() == ReadyToExecute || cur.saveStatus() == Applying) // TODO (desired): only run the check once
+                    {
+                        long epoch = cur.executeAt().epoch();
+                        Ranges ranges = safeStore.ranges().allAt(epoch);
+                        Participants<?> participants = cur.route().participants(ranges, Minimal);
+                        // TODO (required): look forwards also, but we only need to look at ?=ReadyToExecute transactions as they have already run their backwards checks
+                        Iterator<GlobalCommand> iter = commandsByExecuteAt.descendingMap().tailMap(cur.executeAt(), false).values().iterator();
+                        while (iter.hasNext())
+                        {
+                            GlobalCommand prevGlobal = iter.next();
+                            Command prev = prevGlobal.value();
+                            Timestamp prevExecuteAt = prev.executeAtIfKnown();
+                            if (prevExecuteAt == null)
+                            {
+                                iter.remove();
+                                continue;
+                            }
+
+                            if (prevExecuteAt.epoch() < epoch)
+                            {
+                                epoch = prevExecuteAt.epoch();
+                                ranges = ranges.slice(safeStore.ranges().allAt(epoch), Minimal);
+                                participants = participants.slice(ranges, Minimal);
+                            }
+
+                            if (participants.isEmpty())
+                                break;
+
+                            Participants participantsOfPrev = prev.route().participants(ranges, Minimal);
+                            Participants intersectingParticipants = participants.intersect(participantsOfPrev);
+                            if (intersectingParticipants.isEmpty())
+                                continue;
+
+                            if (!cur.txnId().rw().witnesses().test(prev.txnId().rw()) && !cur.partialDeps().contains(prev.txnId()))
+                                continue;
+
+                            Participants<?> depParticipants = cur.partialDeps().participants(prev.txnId());
+                            if (!depParticipants.containsAll(intersectingParticipants))
+                                Invariants.illegalState(cur.txnId() + " does not maintain dependency invariants with immediately preceding transaction " + prev.txnId() + "; intersecting participants: " + intersectingParticipants + "; dependency participants: " + depParticipants);
+
+                            if (prev.txnId().isWrite())
+                                participants = participants.subtract(intersectingParticipants);
+                        };
+                    }
+                }
+
+                @Override
+                public PreLoadContext listenerPreLoadContext(TxnId caller)
+                {
+                    return PreLoadContext.contextFor(caller);
+                }
+            });
+        }
+    }
+
+    private GlobalCommand newGlobalCommand(TxnId txnId)
+    {
+        GlobalCommand globalCommand = new GlobalCommand(txnId);
+        onInitialise(globalCommand);
+        return globalCommand;
     }
 
     public InMemorySafeCommand lazyReference(TxnId txnId)
@@ -170,8 +255,9 @@ public abstract class InMemoryCommandStore extends CommandStore
                 });
                 return mutable;
             case Range:
+                // TODO (required): confirm we slice to owned ranges on C*
                 rangeCommands.computeIfAbsent(command.txnId(), ignore -> new RangeCommand(commands.get(command.txnId())))
-                        .update((Ranges)keysOrRanges);
+                        .update(((Ranges)keysOrRanges).slice(slice, Minimal));
         }
         return attrs;
     }
@@ -190,13 +276,14 @@ public abstract class InMemoryCommandStore extends CommandStore
                 });
                 return mutable;
             case Range:
+                // TODO (required): confirm we slice to owned ranges on C*
                 rangeCommands.computeIfAbsent(command.txnId(), ignore -> new RangeCommand(commands.get(command.txnId())))
-                        .update(Ranges.of((Range)keyOrRange));
+                        .update(Ranges.of((Range)keyOrRange).slice(slice, Minimal));
         }
         return attrs;
     }
 
-    private <O> O mapReduceForKey(InMemorySafeStore safeStore, Routables<?> keysOrRanges, Ranges slice, BiFunction<CommandsForKey, O, O> map, O accumulate, Predicate<O> terminate)
+    private <O> O mapReduceForKey(InMemorySafeStore safeStore, Routables<?> keysOrRanges, Ranges slice, BiFunction<CommandsForKey, O, O> map, O accumulate, Predicate<? super O> terminate)
     {
         switch (keysOrRanges.domain()) {
             default:
@@ -279,11 +366,20 @@ public abstract class InMemoryCommandStore extends CommandStore
 
         // TODO (now): apply on retrieval
         historicalRangeCommands.entrySet().removeIf(next -> next.getKey().compareTo(syncId) < 0 && next.getValue().intersects(ranges));
-        rangeCommands.entrySet().removeIf(next -> {
-            if (!(next.getKey().compareTo(syncId) < 0 && next.getValue().ranges.intersects(ranges)))
+        rangeCommands.entrySet().removeIf(tx -> {
+            if (tx.getKey().compareTo(syncId) >= 0)
                 return false;
-            maxRedundant = Timestamp.nonNullOrMax(maxRedundant, next.getValue().command.value().executeAt());
-            return true;
+            Ranges newRanges = tx.getValue().ranges.subtract(ranges);
+            if (!newRanges.isEmpty())
+            {
+                tx.getValue().ranges = newRanges;
+                return false;
+            }
+            else
+            {
+                maxRedundant = Timestamp.nonNullOrMax(maxRedundant, tx.getValue().command.value().executeAt());
+                return true;
+            }
         });
         ranges.forEach(r -> {
             commandsForKey.subMap(r.start(), r.startInclusive(), r.end(), r.endInclusive()).values().forEach(forKey -> {
@@ -431,7 +527,7 @@ public abstract class InMemoryCommandStore extends CommandStore
         private Command loadForCFK(CFKEntry entry)
         {
             GlobalCommand globalCommand = ifPresent(entry);
-            if (globalCommand != null)
+            if (globalCommand != null && globalCommand.value() != null)
                 return globalCommand.value();
             if (entry.uninitialised)
                 return uninitialised(entry);
@@ -610,6 +706,11 @@ public abstract class InMemoryCommandStore extends CommandStore
         @Override
         protected InMemorySafeCommand getIfLoaded(TxnId txnId)
         {
+            InMemorySafeCommand safeCommand = commands.get(txnId);
+            if (safeCommand != null)
+                return safeCommand;
+            if (!commandStore.canExposeUnloaded())
+                return null;
             GlobalCommand global = commandStore.ifPresent(txnId);
             return global != null ? global.createSafeReference() : null;
         }
@@ -617,6 +718,11 @@ public abstract class InMemoryCommandStore extends CommandStore
         @Override
         protected InMemorySafeCommandsForKey getIfLoaded(RoutableKey key)
         {
+            InMemorySafeCommandsForKey safeCommandsForKey = commandsForKey.get((Key) key);
+            if (safeCommandsForKey != null)
+                return safeCommandsForKey;
+            if (!commandStore.canExposeUnloaded())
+                return null;
             GlobalCommandsForKey global = commandStore.ifPresent((Key) key);
             return global != null ? global.createSafeReference() : null;
         }
@@ -661,7 +767,8 @@ public abstract class InMemoryCommandStore extends CommandStore
                 if (command.ranges.intersects(sliced))
                     timestamp = Timestamp.nonNullOrMax(timestamp, command.command.value().executeAt());
             }
-            return Timestamp.nonNullOrMax(timestamp, commandStore.maxRedundant);
+            // TODO (required): check we have this on C*-side; accept TxnId instead of Ranges when updating API, to avoid slicing
+            return Timestamp.nonNullOrMax(timestamp, commandStore.redundantBefore().minConflict(keysOrRanges.slice(slice)));
         }
 
         @Override
@@ -691,7 +798,7 @@ public abstract class InMemoryCommandStore extends CommandStore
             });
             TreeMap<TxnId, RangeCommand> rangeCommands = commandStore.rangeCommands();
             TreeMap<TxnId, Ranges> historicalRangeCommands = commandStore.historicalRangeCommands();
-            deps.rangeDeps.forEachUniqueTxnId(allRanges, txnId -> {
+            deps.rangeDeps.forEachUniqueTxnId(allRanges, null, (ignore, txnId) -> {
 
                 if (rangeCommands.containsKey(txnId))
                     return;
@@ -712,14 +819,9 @@ public abstract class InMemoryCommandStore extends CommandStore
             return commandStore.time;
         }
 
+        // TODO (expected): instead of accepting a slice, accept the min/max epoch and let implementation handle it
         @Override
-        public <T> T mapReduce(Seekables<?, ?> keysOrRanges, Ranges slice, TestKind testKind, TestTimestamp testTimestamp, Timestamp timestamp, TestDep testDep, @Nullable TxnId depId, @Nullable Status minStatus, @Nullable Status maxStatus, CommandFunction<T, T> map, T accumulate, T terminalValue)
-        {
-            return mapReduceWithTerminate(keysOrRanges, slice, testKind, testTimestamp, timestamp, testDep, depId, minStatus, maxStatus, map, accumulate, Predicate.isEqual(terminalValue));
-        }
-
-        @Override
-        public <T> T mapReduceWithTerminate(Seekables<?, ?> keysOrRanges, Ranges slice, TestKind testKind, TestTimestamp testTimestamp, Timestamp timestamp, TestDep testDep, @Nullable TxnId depId, @Nullable Status minStatus, @Nullable Status maxStatus, CommandFunction<T, T> map, T accumulate, Predicate<T> terminate)
+        public <P1, T> T mapReduce(Seekables<?, ?> keysOrRanges, Ranges slice, Kinds testKind, TestTimestamp testTimestamp, Timestamp timestamp, TestDep testDep, @Nullable TxnId depId, @Nullable Status minStatus, @Nullable Status maxStatus, CommandFunction<P1, T, T> map, P1 p1, T accumulate, Predicate<? super T> terminate)
         {
             accumulate = commandStore.mapReduceForKey(this, keysOrRanges, slice, (forKey, prev) -> {
                 CommandTimeseries<?> timeseries;
@@ -746,7 +848,7 @@ public abstract class InMemoryCommandStore extends CommandStore
                     case MAY_EXECUTE_BEFORE:
                         remapTestTimestamp = CommandTimeseries.TestTimestamp.BEFORE;
                 }
-                return timeseries.mapReduceWithTerminate(testKind, remapTestTimestamp, timestamp, testDep, depId, minStatus, maxStatus, map, prev, terminate);
+                return timeseries.mapReduce(testKind, remapTestTimestamp, timestamp, testDep, depId, minStatus, maxStatus, map, p1, prev, terminate);
             }, accumulate, terminate);
 
             if (terminate.test(accumulate))
@@ -758,7 +860,7 @@ public abstract class InMemoryCommandStore extends CommandStore
             commandStore.rangeCommands.forEach(((txnId, rangeCommand) -> {
                 Command command = rangeCommand.command.value();
                 // TODO (now): probably this isn't safe - want to ensure we take dependency on any relevant syncId
-                if (command.is(Truncated))
+                if (command.saveStatus().compareTo(SaveStatus.Erased) >= 0)
                     return;
 
                 Invariants.nonNull(command);
@@ -794,7 +896,18 @@ public abstract class InMemoryCommandStore extends CommandStore
                     if (!command.known().deps.hasProposedOrDecidedDeps())
                         return;
 
-                    if ((testDep == WITH) == !command.partialDeps().contains(depId))
+                    // TODO (required): ensure C* matches this behaviour
+                    // We are looking for transactions A that have (or have not) B as a dependency.
+                    // If B covers ranges [1..3] and A covers [2..3], but the command store only covers ranges [1..2],
+                    // we could have A adopt B as a dependency on [3..3] only, and have that A intersects B on this
+                    // command store, but also that there is no dependency relation between them on the overlapping
+                    // key range [2..2].
+
+                    // This can lead to problems on recovery, where we believe a transaction is a dependency
+                    // and so it is safe to execute, when in fact it is only a dependency on a different shard
+                    // (and that other shard, perhaps, does not know that it is a dependency - and so it is not durably known)
+                    // TODO (required): consider this some more
+                    if ((testDep == WITH) == !command.partialDeps().intersects(depId, rangeCommand.ranges))
                         return;
                 }
 
@@ -847,7 +960,9 @@ public abstract class InMemoryCommandStore extends CommandStore
                 for (Map.Entry<TxnId, TimestampAndStatus> command : e.getValue())
                 {
                     T initial = accumulate;
-                    accumulate = map.apply(e.getKey(), command.getKey(), command.getValue().timestamp, command.getValue().status, initial);
+                    accumulate = map.apply(p1, e.getKey(), command.getKey(), command.getValue().timestamp, command.getValue().status, initial);
+                    if (terminate.test(accumulate))
+                        return accumulate;
                 }
             }
 
@@ -875,6 +990,17 @@ public abstract class InMemoryCommandStore extends CommandStore
         @Override
         protected void invalidateSafeState()
         {
+            commands.values().forEach(c -> {
+                if (c != null && c.current() != null)
+                {
+                    Timestamp executeAt = c.current().executeAtIfKnown();
+                    if (executeAt == null)
+                        return;
+
+                    if (c.current().hasBeen(Truncated)) commandStore.commandsByExecuteAt.remove(executeAt);
+                    else commandStore.commandsByExecuteAt.put(executeAt, commandStore.command(c.txnId()));
+                }
+            });
             commands.values().forEach(SafeState::invalidate);
             commandsForKey.values().forEach(SafeState::invalidate);
         }
diff --git a/accord-core/src/main/java/accord/impl/SafeCommandsForKey.java b/accord-core/src/main/java/accord/impl/SafeCommandsForKey.java
index 06b5a542..1e5e41b0 100644
--- a/accord-core/src/main/java/accord/impl/SafeCommandsForKey.java
+++ b/accord-core/src/main/java/accord/impl/SafeCommandsForKey.java
@@ -22,6 +22,7 @@ import accord.api.Key;
 import accord.api.VisibleForImplementation;
 import accord.impl.CommandTimeseries.CommandLoader;
 import accord.local.Command;
+import accord.local.SafeCommandStore;
 import accord.primitives.Timestamp;
 import accord.primitives.TxnId;
 import accord.utils.Invariants;
@@ -165,14 +166,18 @@ public abstract class SafeCommandsForKey implements SafeState<CommandsForKey>
     }
 
     @VisibleForImplementation
-    public <D> CommandsForKey updateLastExecutionTimestamps(Timestamp executeAt, boolean isForWriteTxn)
+    public <D> CommandsForKey updateLastExecutionTimestamps(SafeCommandStore safeStore, Timestamp executeAt, boolean isForWriteTxn)
     {
         CommandsForKey current = current();
 
         Timestamp lastWrite = current.lastWriteTimestamp();
 
         if (executeAt.compareTo(lastWrite) < 0)
+        {
+            if (!safeStore.commandStore().safeToReadAt(executeAt).contains(key.toUnseekable()))
+                return current;
             throw new IllegalArgumentException(String.format("%s is less than the most recent write timestamp %s", executeAt, lastWrite));
+        }
 
         Timestamp lastExecuted = current.lastExecutedTimestamp();
         int cmp = executeAt.compareTo(lastExecuted);
@@ -180,7 +185,11 @@ public abstract class SafeCommandsForKey implements SafeState<CommandsForKey>
         if (cmp == 0 || (!isForWriteTxn && cmp < 0))
             return current;
         if (cmp < 0)
+        {
+            if (!safeStore.commandStore().safeToReadAt(executeAt).contains(key.toUnseekable()))
+                return current;
             throw new IllegalArgumentException(String.format("%s is less than the most recent executed timestamp %s", executeAt, lastExecuted));
+        }
 
         long micros = executeAt.hlc();
         long lastMicros = current.lastExecutedHlc();
diff --git a/accord-core/src/main/java/accord/impl/SimpleProgressLog.java b/accord-core/src/main/java/accord/impl/SimpleProgressLog.java
index ae395921..d7477088 100644
--- a/accord-core/src/main/java/accord/impl/SimpleProgressLog.java
+++ b/accord-core/src/main/java/accord/impl/SimpleProgressLog.java
@@ -32,7 +32,6 @@ import org.slf4j.LoggerFactory;
 
 import accord.api.ProgressLog;
 import accord.coordinate.FetchData;
-import accord.coordinate.Invalidate;
 import accord.coordinate.Outcome;
 import accord.local.Command;
 import accord.local.CommandStore;
@@ -45,8 +44,10 @@ import accord.local.SaveStatus;
 import accord.local.SaveStatus.LocalExecution;
 import accord.local.Status;
 import accord.local.Status.Known;
+import accord.primitives.EpochSupplier;
 import accord.primitives.Participants;
 import accord.primitives.ProgressToken;
+import accord.primitives.Ranges;
 import accord.primitives.Route;
 import accord.primitives.Timestamp;
 import accord.primitives.TxnId;
@@ -70,9 +71,8 @@ import static accord.local.PreLoadContext.contextFor;
 import static accord.local.PreLoadContext.empty;
 import static accord.local.SaveStatus.LocalExecution.NotReady;
 import static accord.local.SaveStatus.LocalExecution.WaitingToApply;
-import static accord.local.Status.Durability.Majority;
+import static accord.local.Status.Durability.MajorityOrInvalidated;
 import static accord.local.Status.PreApplied;
-import static accord.local.Status.PreCommitted;
 
 // TODO (desired, consider): consider propagating invalidations in the same way as we do applied
 // TODO (expected): report long-lived recurring transactions / operations
@@ -240,6 +240,7 @@ public class SimpleProgressLog implements ProgressLog.Factory
                             throw new IllegalStateException("Unexpected status: " + status);
 
                         case ReadyToExecute:
+                            // TODO (expected): we should have already exited this loop if this conditions is true
                             if (command.durability().isDurableOrInvalidated())
                             {
                                 // should not reach here with an invalidated state
@@ -251,47 +252,35 @@ public class SimpleProgressLog implements ProgressLog.Factory
                         case Uncommitted:
                         {
                             Invariants.checkState(token.status != Status.Invalidated, "ProgressToken is invalidated, but we have not cleared the ProgressLog");
-                            // TODO (expected): this should be encapsulated in Recover/FetchData
-                            if (!command.durability().isDurable() && token.durability.isDurableOrInvalidated() && (token.durability.isDurable() || command.hasBeen(PreCommitted)))
-                                command = Commands.setDurability(safeStore, safeCommand, Majority);
-
-                            if (command.durability().isDurable() || token.durability.isDurableOrInvalidated())
+                            // TODO (expected): we should have already exited this loop if either of these conditions are true
+                            if (command.durability().isDurableOrInvalidated() || token.durability.isDurableOrInvalidated())
                             {
-                                // not guaranteed to know the execution epoch, as might have received durability info obliquely
-                                Timestamp executeAt = command.executeAtIfKnown();
-                                long epoch = executeAt == null ? txnId.epoch() : executeAt.epoch();
-                                Route<?> route = command.route();
-
-                                node.withEpoch(epoch, () -> debugInvestigating = FetchData.fetch(PreApplied.minKnown, node, txnId, route, executeAt, (success, fail) -> {
-                                    commandStore.execute(empty(), ignore -> {
-                                        // should have found enough information to apply the result, but in case we did not reset progress
-                                        if (progress() == Investigating)
-                                            setProgress(Expected);
-                                    }).begin(commandStore.agent());
-                                }));
+                                if (!command.durability().isDurableOrInvalidated())
+                                    Commands.setDurability(safeStore, safeCommand, MajorityOrInvalidated);
+
+                                durableGlobal();
+                                return;
                             }
-                            else
-                            {
-                                Route<?> route = Invariants.nonNull(command.route()).withHomeKey();
-                                node.withEpoch(txnId.epoch(), () -> {
-                                    AsyncResult<? extends Outcome> recover = node.maybeRecover(txnId, route, token);
-                                    recover.addCallback((success, fail) -> {
-                                        // TODO (expected): callback should be on safeStore, and should provide safeStore as a parameter
-                                        commandStore.execute(contextFor(txnId), safeStore0 -> {
-                                            if (status.isAtMostReadyToExecute() && progress() == Investigating)
-                                            {
-                                                setProgress(Expected);
-                                                if (fail != null)
-                                                    return;
-
-                                                updateMax(success.asProgressToken());
-                                            }
-                                        });
-                                    });
 
-                                    debugInvestigating = recover;
+                            Route<?> route = Invariants.nonNull(command.route()).withHomeKey();
+                            node.withEpoch(txnId.epoch(), () -> {
+                                AsyncResult<? extends Outcome> recover = node.maybeRecover(txnId, route, token);
+                                recover.addCallback((success, fail) -> {
+                                    // TODO (expected): callback should be on safeStore, and should provide safeStore as a parameter
+                                    commandStore.execute(contextFor(txnId), safeStore0 -> {
+                                        if (status.isAtMostReadyToExecute() && progress() == Investigating)
+                                        {
+                                            setProgress(Expected);
+                                            if (fail != null)
+                                                return;
+
+                                            updateMax(success.asProgressToken());
+                                        }
+                                    });
                                 });
-                            }
+
+                                debugInvestigating = recover;
+                            });
                         }
                     }
                 }
@@ -352,10 +341,10 @@ public class SimpleProgressLog implements ProgressLog.Factory
                     setProgress(Investigating);
                     // first make sure we have enough information to obtain the command locally
                     Timestamp executeAt = command.executeAtIfKnown();
-                    Participants<?> maxParticipants = maxParticipants(command);
                     // we want to fetch a route if we have it, so that we can go to our neighbouring shards for info
                     // (rather than the home shard, which may have GC'd its state if the result is durable)
                     Unseekables<?> fetchKeys = maxContact(command);
+                    EpochSupplier forLocalEpoch = safeStore.ranges().latestEpochWithNewParticipants(txnId.epoch(), fetchKeys);
 
                     BiConsumer<Known, Throwable> callback = (success, fail) -> {
                         // TODO (expected): this should be invoked on this commandStore; also do not need to load txn unless in DEBUG mode
@@ -364,17 +353,12 @@ public class SimpleProgressLog implements ProgressLog.Factory
                                 return;
 
                             setProgress(Expected);
-                            if (fail == null && blockedUntil.isSatisfiedBy(success))
-                            {
-                                Command test = safeStore0.ifInitialised(txnId).current();
-                                Invariants.checkState(test.has(success), "Command %s was expected to have known %s, but had %s", test, success, test.known());
-                                record(success);
-                            }
+                            Invariants.checkState(fail != null || !blockedUntil.isSatisfiedBy(success.propagates()));
                         }).begin(commandStore.agent());
                     };
 
                     node.withEpoch(blockedUntil.fetchEpoch(txnId, executeAt), () -> {
-                        debugInvestigating = FetchData.fetch(blockedUntil.requires, node, txnId, fetchKeys, executeAt, callback);
+                        debugInvestigating = FetchData.fetch(blockedUntil.requires, node, txnId, fetchKeys, forLocalEpoch, executeAt, callback);
                     });
                 }
 
@@ -393,24 +377,6 @@ public class SimpleProgressLog implements ProgressLog.Factory
                     return Participants.merge(route == null ? null : route.participants(), (Participants) participants);
                 }
 
-                private void invalidate(Node node, TxnId txnId, Participants<?> participants)
-                {
-                    setProgress(Investigating);
-                    // TODO (now): we should avoid invalidating with home key, as this simplifies erasing of invalidated transactions
-                    // TODO (required): exponential back-off or time-slicing
-                    debugInvestigating = Invalidate.invalidate(node, txnId, participants, (success, fail) -> {
-                        commandStore.execute(contextFor(txnId), safeStore -> {
-                            if (progress() != Investigating)
-                                return;
-
-                            setProgress(Expected);
-                            SafeCommand safeCommand = safeStore.ifInitialised(txnId);
-                            Invariants.checkState(safeCommand != null, "No initialised command for %s but have active progress log", txnId);
-                            record(safeCommand.current().known());
-                        }).begin(commandStore.agent());
-                    });
-                }
-
                 @Override
                 public String toString()
                 {
@@ -473,7 +439,7 @@ public class SimpleProgressLog implements ProgressLog.Factory
                 blockingState.recordBlocking(waitingFor, route, participants);
             }
 
-            CoordinateState local()
+            CoordinateState coordinate()
             {
                 if (coordinateState == null)
                     coordinateState = new CoordinateState();
@@ -482,12 +448,12 @@ public class SimpleProgressLog implements ProgressLog.Factory
 
             void ensureAtLeast(Command command, CoordinateStatus newStatus, Progress newProgress)
             {
-                local().ensureAtLeast(command, newStatus, newProgress);
+                coordinate().ensureAtLeast(command, newStatus, newProgress);
             }
 
             void ensureAtLeast(CoordinateStatus newStatus, Progress newProgress)
             {
-                local().ensureAtLeast(newStatus, newProgress);
+                coordinate().ensureAtLeast(newStatus, newProgress);
             }
 
             void touchNonHomeUnsafe()
@@ -595,6 +561,14 @@ public class SimpleProgressLog implements ProgressLog.Factory
             ensureSafeOrAtLeast(command, shard, Uncommitted, Expected);
         }
 
+        @Override
+        public void precommitted(Command command)
+        {
+            State state = stateMap.get(command.txnId());
+            if (state != null && state.blockingState != null)
+                state.blockingState.record(SaveStatus.PreCommitted.known);
+        }
+
         @Override
         public void committed(Command command, ProgressShard shard)
         {
@@ -644,10 +618,14 @@ public class SimpleProgressLog implements ProgressLog.Factory
             // TODO (expected): we should delete the in-memory state once we reach Done, and guard against backward progress with Command state
             //   however, we need to be careful:
             //     - we might participate in the execution epoch so need to be sure we have received a route covering both
-            if (!command.status().hasBeen(PreApplied) && command.route() != null && command.route().hasParticipants())
+            if (command.route() == null)
+                return;
+
+            Ranges coordinateRanges = commandStore.unsafeRangesForEpoch().allAt(command.txnId().epoch());
+            if (!command.status().hasBeen(PreApplied) && command.route().participatesIn(coordinateRanges))
                 state.recordBlocking(command.txnId(), WaitingToApply, command.route(), null);
-            if (state.coordinateState != null)
-                state.coordinateState.durableGlobal();
+            if (coordinateRanges.contains(command.route().homeKey()))
+                state.coordinate().durableGlobal();
         }
 
         @Override
@@ -658,6 +636,8 @@ public class SimpleProgressLog implements ProgressLog.Factory
 
             // ensure we have a record to work with later; otherwise may think has been truncated
             blockedBy.initialise();
+            if (blockedBy.current().has(blockedUntil.requires))
+                return;
 
             // TODO (consider): consider triggering a preemption of existing coordinator (if any) in some circumstances;
             //                  today, an LWT can pre-empt more efficiently (i.e. instantly) a failed operation whereas Accord will
diff --git a/accord-core/src/main/java/accord/local/Bootstrap.java b/accord-core/src/main/java/accord/local/Bootstrap.java
index 41ce28e1..0652fd3c 100644
--- a/accord-core/src/main/java/accord/local/Bootstrap.java
+++ b/accord-core/src/main/java/accord/local/Bootstrap.java
@@ -362,7 +362,7 @@ class Bootstrap
                 }
             }
 
-            store.markSafeToRead(state.safeToReadAt, newDone);
+            store.markSafeToRead(globalSyncId, state.safeToReadAt, newDone);
             fetchedAndSafeToRead = fetchedAndSafeToRead.with(newDone);
             maybeComplete();
         }
@@ -399,6 +399,9 @@ class Bootstrap
                 if (!fetchedAndSafeToRead.containsAll(fetchCompleted ? fetched : valid))
                     return;
 
+                // normalise fetched and fetchedAndSafeToRead against remaining valid ranges before completion
+                fetched = fetched.slice(valid, Minimal);
+                fetchedAndSafeToRead = fetchedAndSafeToRead.slice(valid, Minimal);
                 retry = valid.subtract(fetchedAndSafeToRead);
                 completed = true;
             }
diff --git a/accord-core/src/main/java/accord/local/Command.java b/accord-core/src/main/java/accord/local/Command.java
index 49ebb5c4..3c680a8e 100644
--- a/accord-core/src/main/java/accord/local/Command.java
+++ b/accord-core/src/main/java/accord/local/Command.java
@@ -22,34 +22,36 @@ import java.util.Objects;
 
 import com.google.common.collect.ImmutableSortedSet;
 
-import accord.api.Data;
 import accord.api.Result;
 import accord.api.RoutingKey;
 import accord.api.VisibleForImplementation;
+import accord.local.Status.Durability;
+import accord.local.Status.Known;
 import accord.primitives.Ballot;
 import accord.primitives.Deps;
+import accord.primitives.FullRoute;
 import accord.primitives.Keys;
 import accord.primitives.PartialDeps;
 import accord.primitives.PartialTxn;
-import accord.primitives.Ranges;
 import accord.primitives.Route;
 import accord.primitives.Timestamp;
 import accord.primitives.Txn;
 import accord.primitives.TxnId;
-import accord.primitives.Unseekables;
 import accord.primitives.Writes;
 import accord.utils.ImmutableBitSet;
 import accord.utils.IndexedQuadConsumer;
 import accord.utils.Invariants;
 import accord.utils.SimpleBitSet;
-import accord.utils.async.AsyncChain;
 import javax.annotation.Nullable;
 
+import static accord.local.Command.AbstractCommand.validate;
 import static accord.local.Listeners.Immutable.EMPTY;
 import static accord.local.SaveStatus.Uninitialised;
-import static accord.local.Status.Durability.DurableOrInvalidated;
 import static accord.local.Status.Durability.Local;
 import static accord.local.Status.Durability.NotDurable;
+import static accord.local.Status.Durability.ShardUniversal;
+import static accord.local.Status.Durability.UniversalOrInvalidated;
+import static accord.local.Status.Invalidated;
 import static accord.local.Status.KnownExecuteAt.ExecuteAtKnown;
 import static accord.utils.SortedArrays.forEachIntersection;
 import static accord.utils.Utils.ensureImmutable;
@@ -130,9 +132,9 @@ public abstract class Command implements CommonAttributes
         return command instanceof PreLoadContext ? (PreLoadContext) command : PreLoadContext.contextFor(command.txnId(), command.partialTxn().keys());
     }
 
-    private static Status.Durability durability(Status.Durability durability, SaveStatus status)
+    private static Durability durability(Durability durability, SaveStatus status)
     {
-        if (status.compareTo(SaveStatus.PreApplied) >= 0 && durability == NotDurable)
+        if (status.compareTo(SaveStatus.PreApplied) >= 0 && !status.hasBeen(Invalidated) && durability == NotDurable)
             return Local; // not necessary anywhere, but helps for logical consistency
         return durability;
     }
@@ -218,16 +220,16 @@ public abstract class Command implements CommonAttributes
      */
     public abstract boolean isEqualOrFuller(Command other);
 
-    private abstract static class AbstractCommand extends Command
+    abstract static class AbstractCommand extends Command
     {
         private final TxnId txnId;
         private final SaveStatus status;
-        private final Status.Durability durability;
+        private final Durability durability;
         private final Route<?> route;
         private final Ballot promised;
         private final Listeners.Immutable listeners;
 
-        private AbstractCommand(TxnId txnId, SaveStatus status, Status.Durability durability, Route<?> route, Ballot promised, Listeners.Immutable listeners)
+        private AbstractCommand(TxnId txnId, SaveStatus status, Durability durability, Route<?> route, Ballot promised, Listeners.Immutable listeners)
         {
             this.txnId = txnId;
             this.status = validateCommandClass(status, getClass());
@@ -299,7 +301,7 @@ public abstract class Command implements CommonAttributes
         }
 
         @Override
-        public Status.Durability durability()
+        public Durability durability()
         {
             return Command.durability(durability, saveStatus());
         }
@@ -315,6 +317,87 @@ public abstract class Command implements CommonAttributes
         {
             return status;
         }
+
+        public static <T extends AbstractCommand> T validate(T validate)
+        {
+            Known known = validate.known();
+            switch (known.route)
+            {
+                default: throw new AssertionError("Unhandled KnownRoute: " + known.route);
+                case Maybe: break;
+                case Full: Invariants.checkState(Route.isFullRoute(validate.route())); break;
+                case Covering: Invariants.checkState(Route.isRoute(validate.route())); break;
+            }
+            {
+                PartialTxn partialTxn = validate.partialTxn();
+                switch (known.definition)
+                {
+                    default: throw new AssertionError("Unhandled Definition: " + known.definition);
+                    case DefinitionErased:
+                    case DefinitionUnknown:
+                    case NoOp:
+                        Invariants.checkState(partialTxn == null);
+                        break;
+                    case DefinitionKnown:
+                        Invariants.checkState(partialTxn != null);
+                        break;
+                }
+            }
+            {
+                Timestamp executeAt = validate.executeAt();
+                switch (known.executeAt)
+                {
+                    default: throw new AssertionError("Unhandled KnownExecuteAt: " + known.executeAt);
+                    case ExecuteAtErased:
+                    case ExecuteAtUnknown: break;
+                    case ExecuteAtProposed:
+                    case ExecuteAtKnown:
+                        Invariants.checkState(executeAt != null && !executeAt.equals(Timestamp.NONE));
+                        break;
+                    case NoExecuteAt:
+                        Invariants.checkState(executeAt.equals(Timestamp.NONE));
+                        break;
+                }
+            }
+            {
+                PartialDeps deps = validate.partialDeps();
+                switch (known.deps)
+                {
+                    default: throw new AssertionError("Unhandled KnownDeps: " + known.deps);
+                    case DepsUnknown:
+                    case DepsErased:
+                    case NoDeps:
+                        Invariants.checkState(deps == null);
+                        break;
+                    case DepsProposed:
+                    case DepsKnown:
+                        Invariants.checkState(deps != null);
+                        break;
+                }
+            }
+            {
+                Writes writes = validate.writes();
+                Result result = validate.result();
+                switch (known.outcome)
+                {
+                    default: throw new AssertionError("Unhandled Outcome: " + known.outcome);
+                    case Apply:
+                        Invariants.checkState(writes != null);
+                        Invariants.checkState(result != null);
+                        break;
+                    case Invalidated:
+                        Invariants.checkState(validate.durability().isMaybeInvalidated());
+                    case Unknown:
+                        Invariants.checkState(validate.durability() != Local);
+                    case Erased:
+                    case WasApply:
+                        Invariants.checkState(writes == null);
+                        Invariants.checkState(result == null);
+                        break;
+                }
+            }
+            return validate;
+        }
     }
 
     @Override
@@ -361,7 +444,7 @@ public abstract class Command implements CommonAttributes
     public abstract TxnId txnId();
     public abstract Ballot promised();
     @Override
-    public abstract Status.Durability durability();
+    public abstract Durability durability();
     @Override
     public abstract Listeners.Immutable<DurableAndIdempotentListener> durableListeners();
     public abstract SaveStatus saveStatus();
@@ -440,12 +523,18 @@ public abstract class Command implements CommonAttributes
         return executeAt == null || executeAt.equals(Timestamp.NONE) ? txnId() : executeAt;
     }
 
+    public final Timestamp executeAtIfKnownOrTxnId()
+    {
+        Timestamp executeAt = executeAtIfKnown();
+        return executeAt == null || executeAt.equals(Timestamp.NONE) ? txnId() : executeAt;
+    }
+
     public final Status status()
     {
         return saveStatus().status;
     }
 
-    public final Status.Known known()
+    public final Known known()
     {
         return saveStatus().known;
     }
@@ -455,7 +544,7 @@ public abstract class Command implements CommonAttributes
         return status().compareTo(status) >= 0;
     }
 
-    public boolean has(Status.Known known)
+    public boolean has(Known known)
     {
         return known.isSatisfiedBy(saveStatus().known);
     }
@@ -465,16 +554,6 @@ public abstract class Command implements CommonAttributes
         return saveStatus().execution.compareTo(execution) >= 0;
     }
 
-    public boolean has(Status.Definition definition)
-    {
-        return known().definition.compareTo(definition) >= 0;
-    }
-
-    public boolean has(Status.Outcome outcome)
-    {
-        return known().outcome.compareTo(outcome) >= 0;
-    }
-
     public boolean is(Status status)
     {
         return status() == status;
@@ -549,7 +628,7 @@ public abstract class Command implements CommonAttributes
 
     public static final class NotDefined extends AbstractCommand
     {
-        NotDefined(TxnId txnId, SaveStatus status, Status.Durability durability, Route<?> route, Ballot promised, Listeners.Immutable listeners)
+        NotDefined(TxnId txnId, SaveStatus status, Durability durability, Route<?> route, Ballot promised, Listeners.Immutable listeners)
         {
             super(txnId, status, durability, route, promised, listeners);
         }
@@ -562,17 +641,17 @@ public abstract class Command implements CommonAttributes
         @Override
         public Command updateAttributes(CommonAttributes attrs, Ballot promised)
         {
-            return new NotDefined(attrs, initialise(saveStatus()), promised);
+            return validate(new NotDefined(attrs, initialise(saveStatus()), promised));
         }
 
         public static NotDefined notDefined(CommonAttributes common, Ballot promised)
         {
-            return new NotDefined(common, SaveStatus.NotDefined, promised);
+            return validate(new NotDefined(common, SaveStatus.NotDefined, promised));
         }
 
         public static NotDefined uninitialised(TxnId txnId)
         {
-            return new NotDefined(txnId, Uninitialised, NotDurable, null, Ballot.ZERO, null);
+            return validate(new NotDefined(txnId, Uninitialised, NotDurable, null, Ballot.ZERO, null));
         }
 
         @Override
@@ -626,7 +705,7 @@ public abstract class Command implements CommonAttributes
             this.result = result;
         }
 
-        public Truncated(TxnId txnId, SaveStatus saveStatus, Status.Durability durability, Route<?> route, @Nullable Timestamp executeAt, Listeners.Immutable listeners, @Nullable Writes writes, @Nullable Result result)
+        public Truncated(TxnId txnId, SaveStatus saveStatus, Durability durability, Route<?> route, @Nullable Timestamp executeAt, Listeners.Immutable listeners, @Nullable Writes writes, @Nullable Result result)
         {
             super(txnId, saveStatus, durability, route, Ballot.MAX, listeners);
             this.executeAt = executeAt;
@@ -660,23 +739,32 @@ public abstract class Command implements CommonAttributes
 
         public static Truncated erased(Command command)
         {
-            return erased(command.txnId(), command.durability(), command.route());
+            Durability durability = Durability.mergeAtLeast(command.durability(), UniversalOrInvalidated);
+            return erased(command.txnId(), durability, command.route());
         }
 
         public static Truncated erased(TxnId txnId, Status.Durability durability, Route<?> route)
         {
-            return new Truncated(txnId, SaveStatus.Erased, durability, route, null, EMPTY, null, null);
+            return validate(new Truncated(txnId, SaveStatus.Erased, durability, route, null, EMPTY, null, null));
         }
 
         public static Truncated truncatedApply(Command command)
         {
-            Invariants.checkArgument(command.known().executeAt.hasDecidedExecuteAt());
-            return new Truncated(command.txnId(), SaveStatus.TruncatedApply, command.durability(), command.route(), command.executeAtIfKnown(), EMPTY, null, null);
+            return truncatedApply(command, null);
+        }
+
+        public static Truncated truncatedApply(Command command, @Nullable FullRoute<?> route)
+        {
+            Invariants.checkArgument(command.known().executeAt.isDecidedAndKnownToExecute());
+            if (route == null) route = Route.castToNonNullFullRoute(command.route());
+            Durability durability = Durability.mergeAtLeast(command.durability(), ShardUniversal);
+            return validate(new Truncated(command.txnId(), SaveStatus.TruncatedApply, durability, route, command.executeAt(), EMPTY, null, null));
         }
 
         public static Truncated truncatedApplyWithOutcome(Executed command)
         {
-            return new Truncated(command.txnId(), SaveStatus.TruncatedApplyWithOutcome, command.durability(), command.route(), command.executeAt(), EMPTY, command.writes, command.result);
+            Durability durability = Durability.mergeAtLeast(command.durability(), ShardUniversal);
+            return validate(new Truncated(command.txnId(), SaveStatus.TruncatedApplyWithOutcome, durability, command.route(), command.executeAt(), EMPTY, command.writes, command.result));
         }
 
         public static Truncated truncatedApply(CommonAttributes common, SaveStatus saveStatus, Timestamp executeAt, Writes writes, Result result)
@@ -684,7 +772,8 @@ public abstract class Command implements CommonAttributes
             Invariants.checkArgument(executeAt != null);
             Invariants.checkArgument(saveStatus == SaveStatus.TruncatedApply || saveStatus == SaveStatus.TruncatedApplyWithDeps || saveStatus == SaveStatus.TruncatedApplyWithOutcome);
             // TODO review Is this correctly handling all three versions of truncatedApplyStatus? Should writes be null some of the time?
-            return new Truncated(common.txnId(), saveStatus, common.durability(), common.route(), executeAt, EMPTY, writes, result);
+            Durability durability = Durability.mergeAtLeast(common.durability(), ShardUniversal);
+            return validate(new Truncated(common.txnId(), saveStatus, durability, common.route(), executeAt, EMPTY, writes, result));
         }
 
         public static Truncated invalidated(Command command)
@@ -696,7 +785,9 @@ public abstract class Command implements CommonAttributes
 
         public static Truncated invalidated(TxnId txnId, Listeners.Immutable durableListeners)
         {
-            return new Truncated(txnId, SaveStatus.Invalidated, DurableOrInvalidated, null, Timestamp.NONE, durableListeners, null, Result.INVALIDATED);
+            // TODO (expected): migrate to using null for executeAt when invalidated
+            // TODO (expected): is UniversalOrInvalidated correct here? Should we have a lower implication pure Invalidated?
+            return validate(new Truncated(txnId, SaveStatus.Invalidated, UniversalOrInvalidated, null, Timestamp.NONE, durableListeners, null, null));
         }
 
         @Override
@@ -739,7 +830,7 @@ public abstract class Command implements CommonAttributes
         public Command updateAttributes(CommonAttributes attrs, Ballot promised)
         {
             // TODO (now): invoke listeners precisely once when we adopt this state, then we can simply return `this`
-            return new Truncated(txnId(), saveStatus(), attrs.durability(), attrs.route(), executeAt, attrs.durableListeners(), writes, result);
+            return validate(new Truncated(txnId(), saveStatus(), attrs.durability(), attrs.route(), executeAt, attrs.durableListeners(), writes, result));
         }
     }
 
@@ -749,18 +840,23 @@ public abstract class Command implements CommonAttributes
         private final PartialTxn partialTxn;
         private final @Nullable PartialDeps partialDeps;
 
-        private PreAccepted(CommonAttributes common, SaveStatus status, Timestamp executeAt, Ballot promised)
+        private PreAccepted(CommonAttributes common, SaveStatus status, Ballot promised, Timestamp executeAt)
+        {
+            this(common, status, promised, executeAt, common.partialTxn(), common.partialDeps());
+        }
+
+        private PreAccepted(CommonAttributes common, SaveStatus status, Ballot promised, Timestamp executeAt, PartialTxn partialTxn, PartialDeps partialDeps)
         {
             super(common, status, promised);
             this.executeAt = executeAt;
-            this.partialTxn = common.partialTxn();
-            this.partialDeps = common.partialDeps();
+            this.partialTxn = partialTxn;
+            this.partialDeps = partialDeps;
         }
 
         @Override
         public Command updateAttributes(CommonAttributes attrs, Ballot promised)
         {
-            return new PreAccepted(attrs, saveStatus(), executeAt(), promised);
+            return new PreAccepted(attrs, saveStatus(), promised, executeAt());
         }
 
         @Override
@@ -790,7 +886,7 @@ public abstract class Command implements CommonAttributes
 
         public static PreAccepted preAccepted(CommonAttributes common, Timestamp executeAt, Ballot promised)
         {
-            return new PreAccepted(common, SaveStatus.PreAccepted, executeAt, promised);
+            return validate(new PreAccepted(common, SaveStatus.PreAccepted, promised, executeAt));
         }
 
         public static PreAccepted preAccepted(PreAccepted command, CommonAttributes common, Ballot promised)
@@ -830,16 +926,22 @@ public abstract class Command implements CommonAttributes
     {
         private final Ballot accepted;
 
-        Accepted(CommonAttributes common, SaveStatus status, Timestamp executeAt, Ballot promised, Ballot accepted)
+        Accepted(CommonAttributes common, SaveStatus status, Ballot promised, Timestamp executeAt, Ballot accepted)
         {
-            super(common, status, executeAt, promised);
+            super(common, status, promised, executeAt);
+            this.accepted = accepted;
+        }
+
+        Accepted(CommonAttributes common, SaveStatus status, Ballot promised, Timestamp executeAt, PartialTxn partialTxn, PartialDeps partialDeps, Ballot accepted)
+        {
+            super(common, status, promised, executeAt, partialTxn, partialDeps);
             this.accepted = accepted;
         }
 
         @Override
         public Command updateAttributes(CommonAttributes attrs, Ballot promised)
         {
-            return new Accepted(attrs, saveStatus(), executeAt(), promised, accepted());
+            return validate(new Accepted(attrs, saveStatus(), promised, executeAt(), accepted()));
         }
 
         @Override
@@ -864,14 +966,14 @@ public abstract class Command implements CommonAttributes
 
         static Accepted accepted(CommonAttributes common, SaveStatus status, Timestamp executeAt, Ballot promised, Ballot accepted)
         {
-            return new Accepted(common, status, executeAt, promised, accepted);
+            return validate(new Accepted(common, status, promised, executeAt, accepted));
         }
 
         static Accepted accepted(Accepted command, CommonAttributes common, SaveStatus status, Ballot promised)
         {
             checkPromised(command, promised);
             checkSameClass(command, Accepted.class, "Cannot update");
-            return new Accepted(common, status, command.executeAt(), promised, command.accepted());
+            return validate(new Accepted(common, status, promised, command.executeAt(), command.accepted()));
         }
 
         static Accepted accepted(Accepted command, CommonAttributes common, Ballot promised)
@@ -892,16 +994,16 @@ public abstract class Command implements CommonAttributes
 
         private Committed(CommonAttributes common, SaveStatus status, Timestamp executeAt, Ballot promised, Ballot accepted, WaitingOn waitingOn)
         {
-            super(common, status, executeAt, promised, accepted);
+            super(common, status, promised, executeAt, accepted);
             this.waitingOn = waitingOn;
             Invariants.checkState(common.route().kind().isFullRoute(), "Expected a full route but given %s", common.route().kind());
-            Invariants.checkState(waitingOn.deps.equals(common.partialDeps()), "Deps do not match; expected %s == %s", waitingOn.deps, common.partialDeps());
+            Invariants.checkState(waitingOn == WaitingOn.EMPTY || waitingOn.deps.equals(common.partialDeps()), "Deps do not match; expected %s == %s", waitingOn.deps, common.partialDeps());
         }
 
         @Override
         public Command updateAttributes(CommonAttributes attrs, Ballot promised)
         {
-            return new Committed(attrs, saveStatus(), executeAt(), promised, accepted(), waitingOn());
+            return validate(new Committed(attrs, saveStatus(), executeAt(), promised, accepted(), waitingOn()));
         }
 
         @Override
@@ -929,7 +1031,7 @@ public abstract class Command implements CommonAttributes
         {
             checkPromised(command, promised);
             checkSameClass(command, Committed.class, "Cannot update");
-            return new Committed(common, status, command.executeAt(), promised, command.accepted(), waitingOn);
+            return validate(new Committed(common, status, command.executeAt(), promised, command.accepted(), waitingOn));
         }
 
         static Committed committed(Committed command, CommonAttributes common, Ballot promised)
@@ -949,12 +1051,7 @@ public abstract class Command implements CommonAttributes
 
         static Committed committed(CommonAttributes common, SaveStatus status, Timestamp executeAt, Ballot promised, Ballot accepted, WaitingOn waitingOn)
         {
-            return new Committed(common, status, executeAt, promised, accepted, waitingOn);
-        }
-
-        public AsyncChain<Data> read(SafeCommandStore safeStore)
-        {
-            return partialTxn().read(safeStore, executeAt());
+            return validate(new Committed(common, status, executeAt, promised, accepted, waitingOn));
         }
 
         public WaitingOn waitingOn()
@@ -994,7 +1091,7 @@ public abstract class Command implements CommonAttributes
         @Override
         public Command updateAttributes(CommonAttributes attrs, Ballot promised)
         {
-            return new Executed(attrs, saveStatus(), executeAt(), promised, accepted(), waitingOn(), writes, result);
+            return validate(new Executed(attrs, saveStatus(), executeAt(), promised, accepted(), waitingOn(), writes, result));
         }
 
         @Override
@@ -1022,7 +1119,7 @@ public abstract class Command implements CommonAttributes
         public static Executed executed(Executed command, CommonAttributes common, SaveStatus status, Ballot promised, WaitingOn waitingOn)
         {
             checkSameClass(command, Executed.class, "Cannot update");
-            return new Executed(common, status, command.executeAt(), promised, command.accepted(), waitingOn, command.writes(), command.result());
+            return validate(new Executed(common, status, command.executeAt(), promised, command.accepted(), waitingOn, command.writes(), command.result()));
         }
 
         public static Executed executed(Executed command, CommonAttributes common, SaveStatus status)
@@ -1042,7 +1139,7 @@ public abstract class Command implements CommonAttributes
 
         public static Executed executed(CommonAttributes common, SaveStatus status, Timestamp executeAt, Ballot promised, Ballot accepted, WaitingOn waitingOn, Writes writes, Result result)
         {
-            return new Executed(common, status, executeAt, promised, accepted, waitingOn, writes, result);
+            return validate(new Executed(common, status, executeAt, promised, accepted, waitingOn, writes, result));
         }
 
         public Writes writes()
@@ -1225,18 +1322,10 @@ public abstract class Command implements CommonAttributes
                 this(committed.waitingOn);
             }
 
-            public Update(Ranges ranges, Unseekables<?> participants, Deps deps)
+            public Update(Deps deps)
             {
                 this.deps = deps;
                 this.waitingOnCommit = new SimpleBitSet(deps.txnIdCount(), false);
-                deps.keyDeps.forEach(ranges, 0, deps.keyDeps.txnIdCount(), this, null, (u, v, i) -> {
-                    u.waitingOnCommit.set(i);
-                });
-                // we select range deps on actual participants rather than covered ranges,
-                // since we may otherwise adopt false dependencies for range txns
-                deps.rangeDeps.forEach(participants, this, (u, i) -> {
-                    u.waitingOnCommit.set(u.deps.keyDeps.txnIdCount() + i);
-                });
                 this.waitingOnApply = new SimpleBitSet(deps.txnIdCount(), false);
                 this.appliedOrInvalidated = new SimpleBitSet(deps.txnIdCount(), false);
             }
@@ -1282,89 +1371,127 @@ public abstract class Command implements CommonAttributes
                 return true;
             }
 
-            public boolean removeWaitingOnApply(TxnId txnId)
+            void initialiseWaitingOnCommit(int index)
             {
-                int index = deps.indexOf(txnId);
-                if (!waitingOnApply.get(index))
-                    return false;
+                waitingOnCommit.set(index);
+            }
 
-                waitingOnApply = ensureMutable(waitingOnApply);
-                waitingOnApply.unset(index);
-                return true;
+            public boolean isEmpty()
+            {
+                return waitingOnApply.isEmpty() && waitingOnCommit.isEmpty();
             }
 
-            public boolean removeWaitingOn(TxnId txnId)
+            public TxnId minWaitingOnTxnId()
             {
-                return removeWaitingOnCommit(txnId) || removeWaitingOnApply(txnId);
+                return WaitingOn.minWaitingOnTxnId(deps, waitingOnCommit, waitingOnApply);
             }
 
-            public boolean removeInvalidatedAppliedOrTruncated(TxnId txnId)
+            public boolean isWaitingOn(int txnIdx)
             {
-                int index = this.deps.indexOf(txnId);
-                return setAppliedOrInvalidated(index);
+                return waitingOnApply.get(txnIdx) || waitingOnCommit.get(txnIdx);
             }
 
-            public boolean isEmpty()
+            public boolean isWaitingOnRangeIdx(int txnIdx)
             {
-                return waitingOnApply.isEmpty() && waitingOnCommit.isEmpty();
+                txnIdx += deps.keyDeps.txnIdCount();
+                return waitingOnApply.get(txnIdx) || waitingOnCommit.get(txnIdx);
             }
 
-            public boolean removeApplied(TxnId txnId, WaitingOn propagate)
+            boolean removeWaitingOn(int i, boolean isAppliedOrInvalidated)
             {
-                int index = this.deps.indexOf(txnId);
-                if (!setAppliedOrInvalidated(index))
+                if (!removeWaitingOn(i))
                     return false;
 
-                if (!propagate.appliedOrInvalidated.isEmpty())
+                if (isAppliedOrInvalidated)
                 {
-                    forEachIntersection(propagate.deps.keyDeps.txnIds(), deps.keyDeps.txnIds(),
-                                        (from, to, ignore, i1, i2) -> {
-                                            if (from.get(i1))
-                                                to.setAppliedOrInvalidated(i2);
-                                        }, propagate.appliedOrInvalidated, this, null);
-
-                    forEachIntersection(propagate.deps.rangeDeps.txnIds(), deps.rangeDeps.txnIds(),
-                                        (from, to, ignore, i1, i2) -> {
-                                            if (from.isAppliedOrInvalidatedRangeIdx(i1))
-                                                to.setAppliedOrInvalidatedRangeIdx(i2);
-                                        }, propagate, this, null);
+                    appliedOrInvalidated = ensureMutable(appliedOrInvalidated);
+                    appliedOrInvalidated.set(i);
                 }
-
                 return true;
             }
 
-            public TxnId minWaitingOnTxnId()
+            public boolean removeWaitingOn(TxnId txnId)
             {
-                return WaitingOn.minWaitingOnTxnId(deps, waitingOnCommit, waitingOnApply);
+                int index = this.deps.indexOf(txnId);
+                return removeWaitingOn(index);
             }
 
-            boolean setAppliedOrInvalidatedRangeIdx(int i)
+            boolean removeWaitingOnRangeIdx(int i)
             {
-                return setAppliedOrInvalidated(i + deps.keyDeps.txnIdCount());
+                return removeWaitingOn(i + deps.keyDeps.txnIdCount());
             }
 
-            boolean setAppliedOrInvalidated(int i)
+            boolean removeWaitingOn(int i)
             {
-                if (appliedOrInvalidated.get(i))
-                    return false;
-
                 if (waitingOnCommit.get(i))
                 {
                     waitingOnCommit = ensureMutable(waitingOnCommit);
                     waitingOnCommit.unset(i);
+                    return true;
                 }
                 else if (waitingOnApply.get(i))
                 {
                     waitingOnApply = ensureMutable(waitingOnApply);
                     waitingOnApply.unset(i);
+                    return true;
                 }
                 else
                 {
                     return false;
                 }
+            }
+
+            /**
+             * Warning: DO NOT invoke this when you really mean removeWaitingOn.
+             * This propagates the applied/invalidated status to dependent transactions, which may
+             * adopt a different set of dependency relations on this transaction. If the transaction
+             * is e.g. partially stale, pre-bootstrap etc this could lead to an erroneous propagation
+             * unless the transaction is truly (and fully) applied or invalidated locally.
+             */
+            public boolean setAppliedOrInvalidated(TxnId txnId)
+            {
+                int index = this.deps.indexOf(txnId);
+                return setAppliedOrInvalidated(index);
+            }
+
+            boolean setAppliedOrInvalidatedRangeIdx(int i)
+            {
+                return setAppliedOrInvalidated(i + deps.keyDeps.txnIdCount());
+            }
+
+            public boolean setAppliedOrInvalidated(int i)
+            {
+                if (appliedOrInvalidated.get(i))
+                    return false;
+
+                if (!removeWaitingOn(i))
+                    return false;
 
                 appliedOrInvalidated = ensureMutable(appliedOrInvalidated);
                 appliedOrInvalidated.set(i);
+                return true;
+            }
+
+            public boolean setAppliedAndPropagate(TxnId txnId, WaitingOn propagate)
+            {
+                int index = this.deps.indexOf(txnId);
+                if (!setAppliedOrInvalidated(index))
+                    return false;
+
+                if (!propagate.appliedOrInvalidated.isEmpty())
+                {
+                    forEachIntersection(propagate.deps.keyDeps.txnIds(), deps.keyDeps.txnIds(),
+                                        (from, to, ignore, i1, i2) -> {
+                                            if (from.get(i1))
+                                                to.setAppliedOrInvalidated(i2);
+                                        }, propagate.appliedOrInvalidated, this, null);
+
+                    forEachIntersection(propagate.deps.rangeDeps.txnIds(), deps.rangeDeps.txnIds(),
+                                        (from, to, ignore, i1, i2) -> {
+                                            if (from.isAppliedOrInvalidatedRangeIdx(i1))
+                                                to.setAppliedOrInvalidatedRangeIdx(i2);
+                                        }, propagate, this, null);
+                }
 
                 return true;
             }
@@ -1422,8 +1549,9 @@ public abstract class Command implements CommonAttributes
         }
         else if (command.status() == Status.AcceptedInvalidate && command.executeAt() == null)
         {
+            // TODO (now): reconsider this special-casing
             Command.Accepted accepted = command.asAccepted();
-            return Command.Accepted.accepted(attrs, accepted.saveStatus(), executeAt, ballot, accepted.accepted());
+            return Command.Accepted.accepted(attrs, SaveStatus.enrich(accepted.saveStatus(), SaveStatus.PreAccepted.known), executeAt, ballot, accepted.accepted());
         }
         else
         {
@@ -1435,29 +1563,29 @@ public abstract class Command implements CommonAttributes
     static Command.Accepted markDefined(Command command, CommonAttributes attributes, Ballot promised)
     {
         if (Command.isSameClass(command, Command.Accepted.class))
-            return Command.Accepted.accepted(command.asAccepted(), attributes, SaveStatus.enrich(command.saveStatus(), Status.Known.DefinitionOnly), promised);
+            return Command.Accepted.accepted(command.asAccepted(), attributes, SaveStatus.enrich(command.saveStatus(), Known.DefinitionOnly), promised);
         return (Command.Accepted) command.updateAttributes(attributes, promised);
     }
 
     static Command.Accepted accept(Command command, CommonAttributes attrs, Timestamp executeAt, Ballot ballot)
     {
-        return new Command.Accepted(attrs, SaveStatus.get(Status.Accepted, command.known()), executeAt, ballot, ballot);
+        return validate(new Command.Accepted(attrs, SaveStatus.get(Status.Accepted, command.known()), ballot, executeAt, ballot));
     }
 
     static Command.Accepted acceptInvalidated(Command command, Ballot ballot)
     {
         Timestamp executeAt = command.isDefined() ? command.asWitnessed().executeAt() : null;
-        return new Command.Accepted(command, SaveStatus.get(Status.AcceptedInvalidate, command.known()), executeAt, ballot, ballot);
+        return validate(new Command.Accepted(command, SaveStatus.get(Status.AcceptedInvalidate, command.known()), ballot, executeAt, command.partialTxn(), null, ballot));
     }
 
     static Command.Committed commit(Command command, CommonAttributes attrs, Timestamp executeAt, Command.WaitingOn waitingOn)
     {
-        return Command.Committed.committed(attrs, SaveStatus.get(Status.Committed, command.known()), executeAt, command.promised(), command.accepted(), waitingOn);
+        return validate(Command.Committed.committed(attrs, SaveStatus.get(Status.Committed, command.known()), executeAt, command.promised(), command.accepted(), waitingOn));
     }
 
     static Command precommit(CommonAttributes attrs, Command command, Timestamp executeAt)
     {
-        return new Command.Accepted(attrs, SaveStatus.get(Status.PreCommitted, command.known()), executeAt, command.promised(), command.accepted());
+        return validate(new Command.Accepted(attrs, SaveStatus.get(Status.PreCommitted, command.known()), command.promised(), executeAt, command.accepted()));
     }
 
     static Command.Committed readyToExecute(Command.Committed command)
diff --git a/accord-core/src/main/java/accord/local/CommandStore.java b/accord-core/src/main/java/accord/local/CommandStore.java
index ae453818..3ba61142 100644
--- a/accord-core/src/main/java/accord/local/CommandStore.java
+++ b/accord-core/src/main/java/accord/local/CommandStore.java
@@ -18,8 +18,29 @@
 
 package accord.local;
 
+import accord.api.ProgressLog;
+import accord.api.DataStore;
+import accord.coordinate.CollectDeps;
+import accord.local.Command.WaitingOn;
+
+import javax.annotation.Nullable;
+import accord.api.Agent;
+
+import accord.local.CommandStores.RangesForEpoch;
+import accord.primitives.Range;
+import accord.primitives.Txn.Kind.Kinds;
+import accord.utils.RelationMultiMap.SortedRelationList;
+import accord.utils.async.AsyncChain;
+
+import accord.api.ConfigurationService.EpochReady;
+import accord.utils.DeterministicIdentitySet;
+import accord.utils.Invariants;
+import accord.utils.ReducingRangeMap;
+import accord.utils.async.AsyncResult;
+
 import java.util.AbstractMap.SimpleImmutableEntry;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.NavigableMap;
 import java.util.Objects;
@@ -30,16 +51,9 @@ import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.Supplier;
-import javax.annotation.Nullable;
 
 import com.google.common.collect.ImmutableSortedMap;
 
-import accord.api.Agent;
-import accord.api.ConfigurationService.EpochReady;
-import accord.api.DataStore;
-import accord.api.ProgressLog;
-import accord.coordinate.CollectDeps;
-import accord.local.Command.WaitingOn;
 import accord.primitives.FullRoute;
 import accord.primitives.KeyDeps;
 import accord.primitives.Participants;
@@ -49,14 +63,7 @@ import accord.primitives.Seekables;
 import accord.primitives.Timestamp;
 import accord.primitives.TxnId;
 import accord.primitives.Unseekables;
-import accord.utils.DeterministicIdentitySet;
-import accord.utils.Invariants;
-import accord.utils.ReducingRangeMap;
-import accord.utils.RelationMultiMap.SortedRelationList;
-import accord.utils.async.AsyncChain;
-import accord.utils.async.AsyncResult;
 import accord.utils.async.AsyncResults;
-import accord.local.CommandStores.RangesForEpoch;
 
 import static accord.api.ConfigurationService.EpochReady.DONE;
 import static accord.local.PreLoadContext.contextFor;
@@ -98,13 +105,13 @@ public abstract class CommandStore implements AgentExecutor
         // TODO (desired): can better encapsulate by accepting only the newRangesForEpoch and deriving the add/remove ranges
         public void add(long epoch, RangesForEpoch newRangesForEpoch, Ranges addRanges)
         {
-            RedundantBefore addRedundantBefore = RedundantBefore.create(addRanges, epoch, Long.MAX_VALUE, TxnId.NONE, TxnId.minForEpoch(epoch));
+            RedundantBefore addRedundantBefore = RedundantBefore.create(addRanges, epoch, Long.MAX_VALUE, TxnId.NONE, TxnId.NONE, TxnId.minForEpoch(epoch));
             update(newRangesForEpoch, addRedundantBefore);
         }
 
         public void remove(long epoch, RangesForEpoch newRangesForEpoch, Ranges removeRanges)
         {
-            RedundantBefore addRedundantBefore = RedundantBefore.create(removeRanges, Long.MIN_VALUE, epoch, TxnId.NONE, TxnId.NONE);
+            RedundantBefore addRedundantBefore = RedundantBefore.create(removeRanges, Long.MIN_VALUE, epoch, TxnId.NONE, TxnId.NONE, TxnId.NONE);
             update(newRangesForEpoch, addRedundantBefore);
         }
 
@@ -145,6 +152,20 @@ public abstract class CommandStore implements AgentExecutor
     protected RangesForEpoch rangesForEpoch;
 
     // TODO (desired): merge with redundantBefore?
+    /**
+     * safeToRead is related to RedundantBefore, but a distinct concept.
+     * While bootstrappedAt defines the txnId bounds we expect to maintain data for locally,
+     * safeToRead defines executeAt bounds we can safely participate in transaction execution for.
+     * safeToRead is defined by the no-op transaction we execute after a bootstrap is initiated,
+     * and creates a global bound before which we know we have complete data from our bootstrap.
+     *
+     * There's a smearing period during bootstrap where some keys may be ahead of others, for instance,
+     * since we do not create a precise instant in the transaction log for bootstrap so as to avoid impeding execution.
+     *
+     * We also update safeToRead when we go stale, to remove ranges we may have bootstrapped but that are now known to
+     * be incomplete. In this case we permit transactions to execute in any order for the unsafe key ranges.
+     * But they may still be ordered for other key ranges they participate in.
+     */
     private NavigableMap<Timestamp, Ranges> safeToRead = ImmutableSortedMap.of(Timestamp.NONE, Ranges.EMPTY);
     private final Set<Bootstrap> bootstraps = Collections.synchronizedSet(new DeterministicIdentitySet<>());
     // TODO (required): we must synchronise this on joining, prior to marking ourselves ready to coordinate
@@ -201,12 +222,12 @@ public abstract class CommandStore implements AgentExecutor
 
     private static Timestamp maxApplied(SafeCommandStore safeStore, Seekables<?, ?> keysOrRanges, Ranges slice)
     {
-        return safeStore.mapReduce(keysOrRanges, slice, SafeCommandStore.TestKind.Ws,
+        return safeStore.mapReduce(keysOrRanges, slice, Kinds.Ws,
                                    SafeCommandStore.TestTimestamp.STARTED_AFTER, Timestamp.NONE,
                                    SafeCommandStore.TestDep.ANY_DEPS, null,
-                                   Status.Applied, Status.Applied,
-                                   (key, txnId, executeAt, status, max) -> Timestamp.max(max, executeAt),
-                                   Timestamp.NONE, Timestamp.MAX);
+                                   Status.PreApplied, Status.Truncated,
+                                   (p1, key, txnId, executeAt, status, max) -> Timestamp.max(max, executeAt),
+                                   null, Timestamp.NONE, Timestamp.MAX::equals);
     }
 
     public AsyncChain<Timestamp> maxAppliedFor(Seekables<?, ?> keysOrRanges, Ranges slice)
@@ -262,16 +283,6 @@ public abstract class CommandStore implements AgentExecutor
         this.safeToRead = newSafeToRead;
     }
 
-    public NavigableMap<TxnId, Ranges> bootstrapBeganAt()
-    {
-        return bootstrapBeganAt;
-    }
-
-    public NavigableMap<Timestamp, Ranges> safeToRead()
-    {
-        return safeToRead;
-    }
-
     public final void markExclusiveSyncPoint(SafeCommandStore safeStore, TxnId txnId, Ranges ranges)
     {
         // TODO (desired): narrow ranges to those that are owned
@@ -281,6 +292,14 @@ public abstract class CommandStore implements AgentExecutor
         setRejectBefore(newRejectBefore);
     }
 
+    public final void markExclusiveSyncPointApplied(SafeCommandStore safeStore, TxnId txnId, Ranges ranges)
+    {
+        // TODO (desired): narrow ranges to those that are owned
+        Invariants.checkArgument(txnId.rw() == ExclusiveSyncPoint);
+        RedundantBefore newRedundantBefore = RedundantBefore.merge(redundantBefore, RedundantBefore.create(ranges, txnId, TxnId.NONE, TxnId.NONE));
+        setRedundantBefore(newRedundantBefore);
+    }
+
     final Timestamp preaccept(TxnId txnId, Seekables<?, ?> keys, SafeCommandStore safeStore, boolean permitFastPath)
     {
         NodeTimeService time = safeStore.time();
@@ -463,7 +482,7 @@ public abstract class CommandStore implements AgentExecutor
     final void markBootstrapping(SafeCommandStore safeStore, TxnId globalSyncId, Ranges ranges)
     {
         setBootstrapBeganAt(bootstrap(globalSyncId, ranges, bootstrapBeganAt));
-        RedundantBefore addRedundantBefore = RedundantBefore.create(ranges, Long.MIN_VALUE, Long.MAX_VALUE, TxnId.NONE, globalSyncId);
+        RedundantBefore addRedundantBefore = RedundantBefore.create(ranges, Long.MIN_VALUE, Long.MAX_VALUE, TxnId.NONE, TxnId.NONE, globalSyncId);
         setRedundantBefore(RedundantBefore.merge(redundantBefore, addRedundantBefore));
         DurableBefore addDurableBefore = DurableBefore.create(ranges, TxnId.NONE, TxnId.NONE);
         setDurableBefore(DurableBefore.merge(durableBefore, addDurableBefore));
@@ -472,14 +491,36 @@ public abstract class CommandStore implements AgentExecutor
     // TODO (expected): we can immediately truncate dependencies locally once an exclusiveSyncPoint applies, we don't need to wait for the whole shard
     public void markShardDurable(SafeCommandStore safeStore, TxnId globalSyncId, Ranges ranges)
     {
-        // TODO (now): IMPORTANT we should insert additional entries for each epoch our ownership differs
         ranges = ranges.slice(safeStore.ranges().allUntil(globalSyncId.epoch()), Minimal);
-        RedundantBefore addRedundantBefore = RedundantBefore.create(ranges, Long.MIN_VALUE, Long.MAX_VALUE, globalSyncId, TxnId.NONE);
+        RedundantBefore addRedundantBefore = RedundantBefore.create(ranges, Long.MIN_VALUE, Long.MAX_VALUE, TxnId.NONE, globalSyncId, TxnId.NONE);
         setRedundantBefore(RedundantBefore.merge(redundantBefore, addRedundantBefore));
         DurableBefore addDurableBefore = DurableBefore.create(ranges, globalSyncId, globalSyncId);
         setDurableBefore(DurableBefore.merge(durableBefore, addDurableBefore));
     }
 
+    // TODO (expected): we can immediately truncate dependencies locally once an exclusiveSyncPoint applies, we don't need to wait for the whole shard
+    public void markShardStale(SafeCommandStore safeStore, Timestamp staleSince, Ranges ranges, boolean isSincePrecise)
+    {
+        Timestamp staleUntilAtLeast = staleSince;
+        if (isSincePrecise)
+        {
+            ranges = ranges.slice(safeStore.ranges().allAt(staleSince.epoch()), Minimal);
+        }
+        else
+        {
+            ranges = ranges.slice(safeStore.ranges().allSince(staleSince.epoch()), Minimal);
+            // make sure no in-progress bootstrap attempts will override the stale since for commands whose staleness bounds are unknown
+            staleUntilAtLeast = Timestamp.max(bootstrapBeganAt.lastKey(), staleUntilAtLeast);
+        }
+        agent.onStale(staleSince, ranges);
+
+        RedundantBefore addRedundantBefore = RedundantBefore.create(ranges, TxnId.NONE, TxnId.NONE, TxnId.NONE, staleUntilAtLeast);
+        setRedundantBefore(RedundantBefore.merge(redundantBefore, addRedundantBefore));
+        // find which ranges need to bootstrap, subtracting those already in progress that cover the id
+
+        markUnsafeToRead(ranges);
+    }
+
     // MUST be invoked before CommandStore reference leaks to anyone
     Supplier<EpochReady> initialise(long epoch, Ranges ranges)
     {
@@ -490,7 +531,7 @@ public abstract class CommandStore implements AgentExecutor
         return () -> new EpochReady(epoch, DONE, DONE, DONE, DONE);
     }
 
-    final Ranges safeToReadAt(Timestamp at)
+    public final Ranges safeToReadAt(Timestamp at)
     {
         return safeToRead.floorEntry(at).getValue();
     }
@@ -506,6 +547,9 @@ public abstract class CommandStore implements AgentExecutor
         return redundantBefore;
     }
 
+    protected NavigableMap<TxnId, Ranges> bootstrapBeganAt() { return bootstrapBeganAt; }
+    protected NavigableMap<Timestamp, Ranges> safeToRead() { return safeToRead; }
+
     public final boolean isRejectedIfNotPreAccepted(TxnId txnId, Unseekables<?> participants)
     {
         if (rejectBefore == null)
@@ -521,7 +565,8 @@ public abstract class CommandStore implements AgentExecutor
         // TODO (required): make sure we have no races on HLC around SyncPoint else this resolution may not work (we need to know the micros equivalent timestamp of the snapshot)
         KeyDeps keyDeps = builder.deps.keyDeps;
         redundantBefore().foldl(keyDeps.keys(), (e, b, d, p2, ki, kj) -> {
-            int txnIdx = d.txnIds().find(e.locallyRedundantBefore());
+            boolean isAppliedOrInvalidated = e.locallyAppliedOrInvalidatedBefore.compareTo(e.bootstrappedAt) >= 0;
+            int txnIdx = d.txnIds().find(isAppliedOrInvalidated ? e.locallyAppliedOrInvalidatedBefore : e.bootstrappedAt);
             if (txnIdx < 0) txnIdx = -1 - txnIdx;
             while (ki < kj)
             {
@@ -530,26 +575,86 @@ public abstract class CommandStore implements AgentExecutor
                 if (ti < 0)
                     ti = -1 - ti;
                 for (int i = 0 ; i < ti ; ++i)
-                    b.setAppliedOrInvalidated(txnIdsForKey.getValueIndex(i));
+                    b.removeWaitingOn(txnIdsForKey.getValueIndex(i), isAppliedOrInvalidated);
             }
             return b;
         }, builder, keyDeps, null, ignore -> false);
-        RangeDeps rangeDeps = builder.deps.rangeDeps;
-        // TODO (now): slice to only those ranges we own, maybe don't even construct rangeDeps.covering()
-        redundantBefore().foldl(participants, (e, b, d, rs, pi, pj) -> {
-            // TODO (now): foldlInt so we can track the lower rangeidx bound and not revisit unnecessarily
-            boolean useBootstrap = e.isLocalRedundancyViaBootstrap();
+
+        /**
+         * If we have to handle bootstrapping ranges for range transactions, these may only partially cover the
+         * transaction, in which case we should not remove the transaction as a dependency. But if it is fully
+         * covered by bootstrapping ranges then we *must* remove it as a dependency.
+         */
+        class RangeState
+        {
+            final WaitingOn.Update builder;
             int txnIdx;
+            Range range;
+            Map<Integer, Ranges> partiallyBootstrapping;
+
+            RangeState(WaitingOn.Update builder)
             {
-                int tmp = d.txnIds().find(useBootstrap ? e.bootstrappedAt : e.redundantBefore);
-                txnIdx = tmp < 0 ? -1 - tmp : tmp;
+                this.builder = builder;
             }
-            rangeDeps.forEach(participants, e.range, pi, pj, d, (d0, txnIdx0) -> {
-                if (txnIdx0 < txnIdx)
-                    b.setAppliedOrInvalidatedRangeIdx(txnIdx0);
+
+            /**
+             * Are the participating ranges for the txn fully covered by bootstrapping ranges for this command store
+             */
+            boolean isFullyBootstrapping(int rangeTxnIdx)
+            {
+                if (partiallyBootstrapping == null)
+                    partiallyBootstrapping = new HashMap<>();
+                Ranges prev = partiallyBootstrapping.get(rangeTxnIdx);
+                Ranges remaining = prev;
+                if (remaining == null) remaining = builder.deps.rangeDeps.ranges(rangeTxnIdx);
+                else Invariants.checkState(!remaining.isEmpty());
+                remaining = remaining.subtract(Ranges.of(range));
+                if (prev == null) Invariants.checkState(!remaining.isEmpty());
+                partiallyBootstrapping.put(txnIdx, remaining);
+                return remaining.isEmpty();
+            }
+        }
+        RangeDeps rangeDeps = builder.deps.rangeDeps;
+        // TODO (required, consider): slice to only those ranges we own, maybe don't even construct rangeDeps.covering()
+        redundantBefore().foldl(participants, (e, s, d, ps, pi, pj) -> {
+            // TODO (desired, efficiency): foldlInt so we can track the lower rangeidx bound and not revisit unnecessarily
+            WaitingOn.Update b = s.builder;
+            {
+                // find the txnIdx below which we are known to be fully redundant locally due to having been applied or invalidated
+                int tmp = d.txnIds().find(e.locallyAppliedOrInvalidatedBefore);
+                s.txnIdx = tmp < 0 ? -1 - tmp : tmp;
+            }
+            // remove intersecting transactions with known redundant txnId
+            d.forEach(ps, e.range, pi, pj, b, s, (b0, s0, txnIdx) -> {
+                if (txnIdx < s0.txnIdx)
+                    b0.removeWaitingOnRangeIdx(txnIdx);
             });
-            return b;
-        }, builder, rangeDeps, participants, ignore -> false);
+            if (e.bootstrappedAt.compareTo(e.locallyAppliedOrInvalidatedBefore) > 0)
+            {
+                // if we have any ranges where bootstrap is ahead of the latest known fully redundant txnId,
+                // we have to do a more complicated dance since this may imply only partial redundancy
+                // (we may still depend on the transaction for some other range)
+                {
+                    int tmp = d.txnIds().find(e.bootstrappedAt);
+                    s.txnIdx = tmp < 0 ? -1 - tmp : tmp;
+                }
+                s.range = e.range;
+                d.forEach(ps, e.range, pi, pj, b, s, (b0, s0, txnIdx) -> {
+                    if (txnIdx < s0.txnIdx && b0.isWaitingOnRangeIdx(txnIdx))
+                    {
+                        if (b0.deps.rangeDeps.foldEachRange(txnIdx, s0.range, true, (r1, r2, p) -> p && r1.contains(r2)))
+                        {
+                            b0.removeWaitingOnRangeIdx(txnIdx);
+                        }
+                        else if (s0.isFullyBootstrapping(txnIdx))
+                        {
+                            b0.removeWaitingOnRangeIdx(txnIdx);
+                        }
+                    }
+                });
+            }
+            return s;
+        }, new RangeState(builder), rangeDeps, participants, ignore -> false);
     }
 
     public final boolean hasLocallyRedundantDependencies(TxnId minimumDependencyId, Timestamp executeAt, Participants<?> participantsOfWaitingTxn)
@@ -557,7 +662,7 @@ public abstract class CommandStore implements AgentExecutor
         // TODO (required): consider race conditions when bootstrapping into an active command store, that may have seen a higher txnId than this?
         //   might benefit from maintaining a per-CommandStore largest TxnId register to ensure we allocate a higher TxnId for our ExclSync,
         //   or from using whatever summary records we have for the range, once we maintain them
-        return redundantBefore.status(minimumDependencyId, executeAt, participantsOfWaitingTxn).compareTo(RedundantStatus.PARTIALLY_PRE_BOOTSTRAP) >= 0;
+        return redundantBefore.status(minimumDependencyId, executeAt, participantsOfWaitingTxn).compareTo(RedundantStatus.PARTIALLY_PRE_BOOTSTRAP_OR_STALE) >= 0;
     }
 
     final synchronized void markUnsafeToRead(Ranges ranges)
@@ -566,9 +671,10 @@ public abstract class CommandStore implements AgentExecutor
             setSafeToRead(purgeHistory(safeToRead, ranges));
     }
 
-    final synchronized void markSafeToRead(Timestamp at, Ranges ranges)
+    final synchronized void markSafeToRead(Timestamp forBootstrapAt, Timestamp at, Ranges ranges)
     {
-        setSafeToRead(purgeAndInsert(safeToRead, at, ranges));
+        Ranges validatedSafeToRead = redundantBefore.validateSafeToRead(forBootstrapAt, ranges);
+        setSafeToRead(purgeAndInsert(safeToRead, at, validatedSafeToRead));
     }
 
     private static <T extends Timestamp> ImmutableSortedMap<T, Ranges> bootstrap(T at, Ranges ranges, NavigableMap<T, Ranges> bootstrappedAt)
diff --git a/accord-core/src/main/java/accord/local/CommandStores.java b/accord-core/src/main/java/accord/local/CommandStores.java
index a1308b0e..9c70c8e6 100644
--- a/accord-core/src/main/java/accord/local/CommandStores.java
+++ b/accord-core/src/main/java/accord/local/CommandStores.java
@@ -38,6 +38,7 @@ import accord.api.Key;
 import accord.api.ProgressLog;
 import accord.api.RoutingKey;
 import accord.local.CommandStore.EpochUpdateHolder;
+import accord.primitives.EpochSupplier;
 import accord.primitives.Range;
 import accord.primitives.Ranges;
 import accord.primitives.Routables;
@@ -45,6 +46,7 @@ import accord.primitives.Route;
 import accord.primitives.RoutingKeys;
 import accord.primitives.Timestamp;
 import accord.primitives.TxnId;
+import accord.primitives.Unseekables;
 import accord.topology.Topology;
 import accord.utils.Invariants;
 import accord.utils.MapReduce;
@@ -53,6 +55,8 @@ import accord.utils.RandomSource;
 import accord.utils.async.AsyncChain;
 import accord.utils.async.AsyncChains;
 import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
 import org.agrona.collections.Hashing;
 import org.agrona.collections.Int2ObjectHashMap;
 
@@ -61,6 +65,7 @@ import org.slf4j.LoggerFactory;
 
 import static accord.api.ConfigurationService.EpochReady.done;
 import static accord.local.PreLoadContext.empty;
+import static accord.primitives.EpochSupplier.constant;
 import static accord.primitives.Routables.Slice.Minimal;
 import static accord.utils.Invariants.checkArgument;
 import static java.util.stream.Collectors.toList;
@@ -129,7 +134,9 @@ public abstract class CommandStores
         }
     }
 
-    // TODO (now): split into deterministic and non-deterministic methods; ensure we only use deterministic methods during commands execution
+    // TODO (required): ensure all deterministic results across runs; updating RangesForEpoch should happen once per
+    //  execution batch and be logged
+    // TODO (expected): merge with RedundantBefore, and standardise executeRanges() to treat removing stale ranges the same as adding new epoch ranges
     public static class RangesForEpoch
     {
         final long[] epochs;
@@ -167,11 +174,6 @@ public abstract class CommandStores
             return allAt(txnId);
         }
 
-        public @Nonnull Ranges safeToReadAt(Timestamp at)
-        {
-            return allAt(at).slice(store.safeToReadAt(at), Minimal);
-        }
-
         public @Nonnull Ranges unsafeToReadAt(Timestamp at)
         {
             return allAt(at).subtract(store.safeToReadAt(at));
@@ -184,18 +186,43 @@ public abstract class CommandStores
 
         public @Nonnull Ranges allAt(long epoch)
         {
-            int i = Arrays.binarySearch(epochs, epoch);
-            if (i < 0) i = -2 -i;
+            int i = floorIndex(epoch);
             if (i < 0) return Ranges.EMPTY;
             return ranges[i];
         }
 
-        public @Nonnull Ranges allBetween(Timestamp fromInclusive, Timestamp toInclusive)
+        /**
+         * Extend a previously computed set of Ranges that included {@code fromInclusive}
+         * to include {@code toInclusive}
+         */
+        public @Nonnull Ranges extend(Ranges extend, Timestamp fromInclusive, Timestamp toInclusive)
+        {
+            return extend(extend, fromInclusive.epoch(), toInclusive.epoch());
+        }
+
+        /**
+         * Extend a previously computed set of Ranges that included {@code fromInclusive}
+         * to include ranges up to {@code toInclusive}
+         */
+        public @Nonnull Ranges extend(Ranges extend, long fromInclusive, long toInclusive)
+        {
+            if (fromInclusive == toInclusive)
+                return extend;
+
+            int startIndex = 1 + floorIndex(fromInclusive);
+            int endIndex = 1 + floorIndex(toInclusive);
+            for (int i = startIndex ; i < endIndex; ++i)
+                extend = extend.with(ranges[i]); // want to always return extend for equal ranges, so same ranges are identity-equals
+
+            return extend;
+        }
+
+        public @Nonnull Ranges allBetween(EpochSupplier fromInclusive, EpochSupplier toInclusive)
         {
             return allBetween(fromInclusive.epoch(), toInclusive.epoch());
         }
 
-        public @Nonnull Ranges allBetween(long fromInclusive, Timestamp toInclusive)
+        public @Nonnull Ranges allBetween(long fromInclusive, EpochSupplier toInclusive)
         {
             return allBetween(fromInclusive, toInclusive.epoch());
         }
@@ -231,6 +258,11 @@ public abstract class CommandStores
             return allInternal(0, 1 + floorIndex(toInclusive));
         }
 
+        public @Nonnull Ranges allSince(long fromInclusive)
+        {
+            return allInternal(Math.max(0, floorIndex(fromInclusive)), epochs.length);
+        }
+
         private int floorIndex(long epoch)
         {
             int i = Arrays.binarySearch(epochs, epoch);
@@ -271,6 +303,21 @@ public abstract class CommandStores
             return IntStream.range(0, ranges.length).mapToObj(i -> epochs[i] + ": " + ranges[i])
                             .collect(Collectors.joining(", "));
         }
+
+        public @Nullable EpochSupplier latestEpochWithNewParticipants(long sinceEpoch, Unseekables<?> keysOrRanges)
+        {
+            int i = floorIndex(sinceEpoch);
+            long latest = -1;
+            Ranges existing = i < 0 ? Ranges.EMPTY : ranges[i];
+            while (++i < ranges.length)
+            {
+                if (ranges[i].subtract(existing).intersects(keysOrRanges))
+                    latest = epochs[i];
+                existing = existing.with(ranges[i]);
+            }
+
+            return latest == -1 ? null : constant(latest);
+        }
     }
 
     static class Snapshot
diff --git a/accord-core/src/main/java/accord/local/Commands.java b/accord-core/src/main/java/accord/local/Commands.java
index 3193e70d..88e925b7 100644
--- a/accord-core/src/main/java/accord/local/Commands.java
+++ b/accord-core/src/main/java/accord/local/Commands.java
@@ -67,14 +67,17 @@ import static accord.local.Commands.Cleanup.ERASE;
 import static accord.local.Commands.Cleanup.NO;
 import static accord.local.Commands.Cleanup.TRUNCATE;
 import static accord.local.Commands.EnsureAction.Add;
-import static accord.local.Commands.EnsureAction.Check;
+import static accord.local.Commands.EnsureAction.TryAdd;
 import static accord.local.Commands.EnsureAction.Ignore;
 import static accord.local.Commands.EnsureAction.Set;
 import static accord.local.Commands.EnsureAction.TrySet;
+import static accord.local.RedundantBefore.PreBootstrapOrStale.FULLY;
 import static accord.local.RedundantStatus.NOT_OWNED;
-import static accord.local.RedundantStatus.PRE_BOOTSTRAP;
+import static accord.local.RedundantStatus.PRE_BOOTSTRAP_OR_STALE;
 import static accord.local.SaveStatus.Applying;
 import static accord.local.SaveStatus.Erased;
+import static accord.local.SaveStatus.LocalExecution.ReadyToExclude;
+import static accord.local.SaveStatus.LocalExecution.WaitingToExecute;
 import static accord.local.SaveStatus.TruncatedApply;
 import static accord.local.SaveStatus.TruncatedApplyWithOutcome;
 import static accord.local.SaveStatus.Uninitialised;
@@ -92,7 +95,10 @@ import static accord.local.Status.PreApplied;
 import static accord.local.Status.PreCommitted;
 import static accord.local.Status.ReadyToExecute;
 import static accord.local.Status.Truncated;
+import static accord.primitives.Routables.Slice.Minimal;
 import static accord.primitives.Route.isFullRoute;
+import static accord.primitives.Txn.Kind.ExclusiveSyncPoint;
+import static accord.utils.Invariants.illegalState;
 
 public class Commands
 {
@@ -145,7 +151,7 @@ public class Commands
         if (command.hasBeen(Truncated))
         {
             logger.trace("{}: skipping preaccept - command is truncated", txnId);
-            return command.is(Invalidated) ? AcceptOutcome.Redundant : AcceptOutcome.Truncated;
+            return command.is(Invalidated) ? AcceptOutcome.RejectedBallot : AcceptOutcome.Truncated;
         }
 
         int compareBallots = command.promised().compareTo(ballot);
@@ -165,13 +171,13 @@ public class Commands
             return ballot.equals(Ballot.ZERO) ? AcceptOutcome.Redundant : AcceptOutcome.Success;
         }
 
-        Ranges coordinateRanges = coordinateRanges(safeStore, txnId, acceptEpoch);
-        Invariants.checkState(!coordinateRanges.isEmpty());
-        ProgressShard shard = progressShard(route, progressKey, coordinateRanges);
-        Invariants.checkState(validate(command.status(), command, Ranges.EMPTY, coordinateRanges, shard, route, Set, partialTxn, Set, null, Ignore));
+        Ranges preacceptRanges = preacceptRanges(safeStore, txnId, acceptEpoch);
+        Invariants.checkState(!preacceptRanges.isEmpty());
+        ProgressShard shard = progressShard(route, progressKey, preacceptRanges);
+        Invariants.checkState(validate(command.status(), command, Ranges.EMPTY, preacceptRanges, shard, route, Set, partialTxn, Set, null, Ignore));
 
         // FIXME: this should go into a consumer method
-        CommonAttributes attrs = set(safeStore, command, command, Ranges.EMPTY, coordinateRanges, shard, route, partialTxn, Set, null, Ignore);
+        CommonAttributes attrs = set(safeStore, command, command, Ranges.EMPTY, preacceptRanges, shard, route, partialTxn, Set, null, Ignore);
         if (command.executeAt() == null)
         {
             // unlike in the Accord paper, we partition shards within a node, so that to ensure a total order we must either:
@@ -224,7 +230,7 @@ public class Commands
             if (command.is(Invalidated))
             {
                 logger.trace("{}: skipping accept - command is invalidated", txnId);
-                return AcceptOutcome.Redundant;
+                return AcceptOutcome.RejectedBallot;
             }
 
             if (command.is(Truncated))
@@ -285,7 +291,7 @@ public class Commands
             }
 
             logger.trace("{}: skipping accept invalidated - already committed ({})", command.txnId(), command.status());
-            return AcceptOutcome.Redundant;
+            return AcceptOutcome.RejectedBallot;
         }
 
         if (command.promised().compareTo(ballot) > 0)
@@ -328,16 +334,14 @@ public class Commands
 
         Ranges coordinateRanges = coordinateRanges(safeStore, txnId);
         Ranges acceptRanges = acceptRanges(safeStore, txnId, executeAt, coordinateRanges);
-        // TODO (expected, consider): consider ranges between coordinateRanges and executeRanges? Perhaps don't need them
-        Ranges executeRanges = executeRanges(safeStore, executeAt);
 
         ProgressShard shard = progressShard(route, progressKey, coordinateRanges);
 
-        if (!validate(command.status(), command, acceptRanges, executeRanges, shard, route, Check, partialTxn, Add, partialDeps, Set))
+        if (!validate(command.status(), command, coordinateRanges, acceptRanges, shard, route, Add, partialTxn, Add, partialDeps, Set))
             return CommitOutcome.Insufficient;
 
         // FIXME: split up set
-        CommonAttributes attrs = set(safeStore, command, command, acceptRanges, executeRanges, shard, route, partialTxn, Add, partialDeps, Set);
+        CommonAttributes attrs = set(safeStore, command, command, coordinateRanges, acceptRanges, shard, route, partialTxn, Add, partialDeps, Set);
 
         logger.trace("{}: committed with executeAt: {}, deps: {}", txnId, executeAt, partialDeps);
         WaitingOn waitingOn = initialiseWaitingOn(safeStore, txnId, executeAt, attrs.partialDeps(), attrs.route());
@@ -373,13 +377,10 @@ public class Commands
 
         CommonAttributes attrs = command;
         if (command.route() == null || !command.route().kind().isFullRoute())
-        {
-            if (!route.kind().isFullRoute())
-                return CommitOutcome.Insufficient;
             attrs = updateRoute(command, route);
-        }
 
         safeCommand.precommit(attrs, executeAt);
+        safeStore.progressLog().precommitted(command);
         safeStore.notifyListeners(safeCommand);
         logger.trace("{}: precommitted with executeAt: {}", txnId, executeAt);
         return CommitOutcome.Success;
@@ -452,7 +453,7 @@ public class Commands
         safeStore.notifyListeners(safeCommand, command, command.durableListeners(), safeCommand.transientListeners());
     }
 
-    public enum ApplyOutcome {Success, Redundant, Insufficient}
+    public enum ApplyOutcome { Success, Redundant, Insufficient }
 
     public static ApplyOutcome apply(SafeCommandStore safeStore, SafeCommand safeCommand, TxnId txnId, Route<?> route, @Nullable RoutingKey progressKey, Timestamp executeAt, @Nullable PartialDeps partialDeps, @Nullable PartialTxn partialTxn, Writes writes, Result result)
     {
@@ -471,14 +472,13 @@ public class Commands
 
         Ranges coordinateRanges = coordinateRanges(safeStore, txnId);
         Ranges acceptRanges = acceptRanges(safeStore, txnId, executeAt, coordinateRanges);
-        Ranges executeRanges = executeRanges(safeStore, executeAt);
 
         ProgressShard shard = progressShard(route, progressKey, coordinateRanges);
 
-        if (!validate(command.status(), command, acceptRanges, executeRanges, shard, route, Check, partialTxn, Add, partialDeps, command.hasBeen(Committed) ? Check : TrySet))
+        if (!validate(command.status(), command, coordinateRanges, acceptRanges, shard, route, TryAdd, partialTxn, Add, partialDeps, command.hasBeen(Committed) ? TryAdd : TrySet, safeStore))
             return ApplyOutcome.Insufficient; // TODO (expected, consider): this should probably be an assertion failure if !TrySet
 
-        CommonAttributes attrs = set(safeStore, command, command, acceptRanges, executeRanges, shard, route, partialTxn, Add, partialDeps, command.hasBeen(Committed) ? Check : TrySet);
+        CommonAttributes attrs = set(safeStore, command, command, coordinateRanges, acceptRanges, shard, route, partialTxn, Add, partialDeps, command.hasBeen(Committed) ? TryAdd : TrySet);
 
         WaitingOn waitingOn = !command.hasBeen(Committed) ? initialiseWaitingOn(safeStore, txnId, executeAt, attrs.partialDeps(), attrs.route()) : command.asCommitted().waitingOn();
         safeCommand.preapplied(attrs, executeAt, waitingOn, writes, result);
@@ -534,7 +534,7 @@ public class Commands
     {
         logger.trace("{} applied, setting status to Applied and notifying listeners", txnId);
         SafeCommand safeCommand = safeStore.get(txnId);
-        safeCommand.applied();
+        Command.Executed command = safeCommand.applied();
         safeStore.notifyListeners(safeCommand);
     }
 
@@ -549,14 +549,14 @@ public class Commands
     /**
      * The ranges for which we participate in the consensus decision of when a transaction executes
      */
-    private static Ranges coordinateRanges(SafeCommandStore safeStore, TxnId txnId, long untilEpoch)
+    private static Ranges preacceptRanges(SafeCommandStore safeStore, TxnId txnId, long untilEpoch)
     {
         return safeStore.ranges().allBetween(txnId.epoch(), untilEpoch);
     }
 
     private static Ranges acceptRanges(SafeCommandStore safeStore, TxnId txnId, Timestamp executeAt, Ranges coordinateRanges)
     {
-        return txnId.epoch() == executeAt.epoch() ? coordinateRanges : safeStore.ranges().allBetween(txnId, executeAt);
+        return safeStore.ranges().extend(coordinateRanges, txnId, executeAt);
     }
 
     private static Ranges executeRanges(SafeCommandStore safeStore, Timestamp executeAt)
@@ -669,6 +669,12 @@ public class Commands
                     //      but: if we later support transitive dependency elision this could be dangerous
                     logger.trace("{}: applying no-op", command.txnId());
                     safeCommand.applied();
+                    if (command.txnId().rw() == ExclusiveSyncPoint)
+                    {
+                        Ranges ranges = safeStore.ranges().allAt(command.txnId().epoch());
+                        ranges = command.route().slice(ranges, Minimal).participants().toRanges();
+                        safeStore.commandStore().markExclusiveSyncPointApplied(safeStore, command.txnId(), ranges);
+                    }
                     safeStore.notifyListeners(safeCommand);
                     return true;
                 }
@@ -677,22 +683,30 @@ public class Commands
         }
     }
 
-    protected static WaitingOn initialiseWaitingOn(SafeCommandStore safeStore, TxnId waitingId, Timestamp executeWaitingAt, PartialDeps partialDeps, Route<?> route)
+    protected static WaitingOn initialiseWaitingOn(SafeCommandStore safeStore, TxnId waitingId, Timestamp executeWaitingAt, PartialDeps deps, Route<?> route)
     {
         Ranges ranges = safeStore.ranges().allAt(executeWaitingAt);
-        Unseekables<?> executionParticipants = route.participants().slice(ranges);
-        WaitingOn.Update update = new WaitingOn.Update(ranges, executionParticipants, partialDeps);
+        Unseekables<?> executionParticipants = route.participants().slice(ranges, Minimal);
+        WaitingOn.Update update = new WaitingOn.Update(deps);
+        deps.keyDeps.forEach(ranges, 0, deps.keyDeps.txnIdCount(), update, null, (u, v, i) -> {
+            u.initialiseWaitingOnCommit(i);
+        });
+        // we select range deps on actual participants rather than covered ranges,
+        // since we may otherwise adopt false dependencies for range txns
+        deps.rangeDeps.forEach(executionParticipants, update, (u, i) -> {
+            u.initialiseWaitingOnCommit(u.deps.keyDeps.txnIdCount() + i);
+        });
         return updateWaitingOn(safeStore, waitingId, executeWaitingAt, update, route.participants()).build();
     }
 
-    protected static WaitingOn.Update updateWaitingOn(SafeCommandStore safeStore, TxnId txnId, Timestamp executeAt, WaitingOn.Update update, Participants<?> participants)
+    protected static WaitingOn.Update updateWaitingOn(SafeCommandStore safeStore, TxnId waitingId, Timestamp executeAt, WaitingOn.Update update, Participants<?> participants)
     {
         CommandStore commandStore = safeStore.commandStore();
         TxnId minWaitingOnTxnId = update.minWaitingOnTxnId();
         if (minWaitingOnTxnId != null && commandStore.hasLocallyRedundantDependencies(update.minWaitingOnTxnId(), executeAt, participants))
             safeStore.commandStore().removeRedundantDependencies(participants, update);
 
-        update.forEachWaitingOnCommit(safeStore, update, txnId, executeAt, (safeStore0, upd, id, exec, i) -> {
+        update.forEachWaitingOnCommit(safeStore, update, waitingId, executeAt, (safeStore0, upd, id, exec, i) -> {
             // TODO (expected): load read-only to reduce overhead; upgrade only if we need to remove listener
             SafeCommand dep = safeStore0.ifLoadedAndInitialised(upd.deps.txnId(i));
             if (dep == null || !dep.current().hasBeen(PreCommitted))
@@ -700,7 +714,7 @@ public class Commands
             updateWaitingOn(safeStore0, id, exec, upd, dep);
         });
 
-        update.forEachWaitingOnApply(safeStore, update, txnId, executeAt, (store, upd, id, exec, i) -> {
+        update.forEachWaitingOnApply(safeStore, update, waitingId, executeAt, (store, upd, id, exec, i) -> {
             SafeCommand dep = store.ifLoadedAndInitialised(upd.deps.txnId(i));
             if (dep == null || !dep.current().hasBeen(PreCommitted))
                 return;
@@ -723,13 +737,15 @@ public class Commands
         {
             logger.trace("{}: {} is truncated. Stop listening and removing from waiting on commit set.", waitingId, dependencyId);
             dependencySafeCommand.removeListener(new ProxyListener(waitingId));
-            return waitingOn.removeInvalidatedAppliedOrTruncated(dependencyId);
+            // TODO (required): provide clear mechanism for translating SaveStatus -> isAppliedOrInvalidated
+            //  so can propagate to setAppliedOrInvalidated or removeWaitingOn as appropriate
+            return waitingOn.removeWaitingOn(dependencyId);
         }
         else if (dependency.is(Invalidated))
         {
             logger.trace("{}: {} is invalidated. Stop listening and removing from waiting on commit set.", waitingId, dependencyId);
             dependencySafeCommand.removeListener(new ProxyListener(waitingId));
-            return waitingOn.removeInvalidatedAppliedOrTruncated(dependencyId);
+            return waitingOn.setAppliedOrInvalidated(dependencyId);
         }
         else if (dependency.executeAt().compareTo(executeWaitingAt) > 0 && !waitingId.rw().awaitsFutureDeps())
         {
@@ -742,7 +758,7 @@ public class Commands
         {
             logger.trace("{}: {} has been applied. Stop listening and removing from waiting on apply set.", waitingId, dependencyId);
             dependencySafeCommand.removeListener(new ProxyListener(waitingId));
-            return waitingOn.removeApplied(dependencyId, dependency.asCommitted().waitingOn());
+            return waitingOn.setAppliedAndPropagate(dependencyId, dependency.asCommitted().waitingOn());
         }
         else if (waitingOn.removeWaitingOnCommit(dependencyId))
         {
@@ -755,7 +771,7 @@ public class Commands
         {
             return false;
         }
-        else if (safeStore.isFullyPreBootstrap(dependency, waitingOn.deps.participants(dependency.txnId())))
+        else if (safeStore.isFullyPreBootstrapOrStale(dependency, waitingOn.deps.participants(dependency.txnId())))
         {
             // TODO (expected): erase or otherwise prevent from executing
             return false;
@@ -808,15 +824,22 @@ public class Commands
     // TODO (now): document and justify all calls
     public static void setTruncatedApply(SafeCommandStore safeStore, SafeCommand safeCommand)
     {
-        cleanup(safeStore, safeCommand, TRUNCATE, true);
+        setTruncatedApply(safeStore, safeCommand, null);
+    }
+
+    public static void setTruncatedApply(SafeCommandStore safeStore, SafeCommand safeCommand, Route<?> maybeFullRoute)
+    {
+        cleanup(safeStore, safeCommand, maybeFullRoute, TRUNCATE, true);
     }
 
     public static void setErased(SafeCommandStore safeStore, SafeCommand safeCommand)
     {
-        cleanup(safeStore, safeCommand, ERASE, true);
+        Listeners.Immutable durableListeners = safeCommand.current().durableListeners();
+        Command command = cleanup(safeStore, safeCommand, null, ERASE, true);
+        safeStore.notifyListeners(safeCommand, command, durableListeners, safeCommand.transientListeners());
     }
 
-    public static Command cleanup(SafeCommandStore safeStore, SafeCommand safeCommand, Cleanup cleanup, boolean notifyListeners)
+    public static Command cleanup(SafeCommandStore safeStore, SafeCommand safeCommand, @Nullable Unseekables<?> maybeFullRoute, Cleanup cleanup, boolean notifyListeners)
     {
         Command command = safeCommand.current();
 
@@ -826,7 +849,7 @@ public class Commands
         // TODO (desired): consider if there are better invariants we can impose for undecided transactions, to verify they aren't later committed (should be detected already, but more is better)
         // note that our invariant here is imperfectly applied to keep the code cleaner: we don't verify that the caller was safe to invoke if we don't already have a route in the command and we're only PreCommitted
         Invariants.checkState(command.hasBeen(Applied) || !command.hasBeen(PreCommitted)
-                              || (command.route() == null || Infer.safeToCleanup(safeStore, command, command.route(), command.executeAt()) || safeStore.isFullyPreBootstrap(command, command.route().participants()))
+                              || (command.route() == null || Infer.safeToCleanup(safeStore, command, command.route(), command.executeAt()) || safeStore.isFullyPreBootstrapOrStale(command, command.route().participants()))
         , "Command %s could not be truncated", command);
 
         Command result;
@@ -844,8 +867,8 @@ public class Commands
 
             case TRUNCATE:
                 Invariants.checkState(command.saveStatus().compareTo(TruncatedApply) < 0);
-                if (command.hasBeen(PreCommitted)) result = truncatedApply(command);
-                else result = erased(command); // TODO (expected): check if we are stale; if not, is erase anyway correct outcome?
+                if (command.hasBeen(PreCommitted)) result = truncatedApply(command, Route.tryCastToFullRoute(maybeFullRoute));
+                else result = command; // do nothing; we don't have enough information
                 break;
 
             case ERASE:
@@ -867,7 +890,7 @@ public class Commands
         if (command.saveStatus().compareTo(cleanup.appliesIfNot) >= 0)
             return false;
 
-        cleanup(safeStore, safeCommand, cleanup, true);
+        cleanup(safeStore, safeCommand, maybeFullRoute, cleanup, true);
         return true;
     }
 
@@ -879,7 +902,7 @@ public class Commands
         Route<?> route = command.route();
         if (!Route.isFullRoute(route))
         {
-            if (command.known().hasFullRoute()) throw new IllegalStateException(command.saveStatus() + " should include full route, but only found " + route);
+            if (command.known().hasFullRoute()) illegalState(command.saveStatus() + " should include full route, but only found " + route);
             else if (Route.isFullRoute(maybeFullRoute)) route = Route.castToFullRoute(maybeFullRoute);
             else return Cleanup.NO;
         }
@@ -902,7 +925,7 @@ public class Commands
         if (durableBefore.min(txnId) == Universal)
         {
             if (status.hasBeen(PreCommitted) && !status.hasBeen(Applied)) // TODO (expected): may be stale
-                throw new IllegalStateException("Loading universally-durable command that has been PreCommitted but not Applied");
+                illegalState("Loading universally-durable command that has been PreCommitted but not Applied");
             return Cleanup.ERASE;
         }
 
@@ -919,14 +942,16 @@ public class Commands
                 // TODO (expected): we can impose additional validations here IF we receive an epoch upper bound
                 // TODO (expected): we should be more robust to the presence/absence of executeAt
                 if (route.isParticipatingHomeKey() || redundantBefore.get(txnId, toEpoch, route.homeKey()) == NOT_OWNED)
-                    throw new IllegalStateException("Command " + txnId + " that is being loaded is not owned by this shard on route " + route);
+                    illegalState("Command " + txnId + " that is being loaded is not owned by this shard on route " + route);
             case LIVE:
-            case PARTIALLY_PRE_BOOTSTRAP:
-            case PRE_BOOTSTRAP:
-                return NO;
+            case PARTIALLY_PRE_BOOTSTRAP_OR_STALE:
+            case PRE_BOOTSTRAP_OR_STALE:
+            case REDUNDANT_PRE_BOOTSTRAP_OR_STALE:
             case LOCALLY_REDUNDANT:
-                if (enforceInvariants && status.hasBeen(PreCommitted) && !status.hasBeen(Applied)) // TODO (required): may be stale
-                    throw new IllegalStateException("Loading redundant command that has been PreCommitted but not Applied");
+                return NO;
+            case SHARD_REDUNDANT:
+                if (enforceInvariants && status.hasBeen(PreCommitted) && !status.hasBeen(Applied) && redundantBefore.preBootstrapOrStale(txnId, toEpoch, route.participants()) != FULLY)
+                    illegalState("Loading redundant command that has been PreCommitted but not Applied");
         }
 
         Durability min = durableBefore.min(txnId, route);
@@ -934,17 +959,18 @@ public class Commands
         {
             default:
             case Local:
-            case DurableOrInvalidated:
                 throw new AssertionError("Unexpected durability: " + min);
+
             case NotDurable:
                 if (durability == Majority)
                     return Cleanup.TRUNCATE;
-
                 return Cleanup.TRUNCATE_WITH_OUTCOME;
 
+            case MajorityOrInvalidated:
             case Majority:
                 return Cleanup.TRUNCATE;
 
+            case UniversalOrInvalidated:
             case Universal:
                 return Cleanup.ERASE;
         }
@@ -961,7 +987,7 @@ public class Commands
             return command;
 
         CommonAttributes attrs = route == null ? command : updateRoute(command, route);
-        if (executeAt != null && command.status().hasBeen(Committed) && !command.asCommitted().executeAt().equals(executeAt))
+        if (executeAt != null && command.status().hasBeen(Committed) && !command.executeAt().equals(executeAt))
             safeStore.agent().onInconsistentTimestamp(command, command.asCommitted().executeAt(), executeAt);
         attrs = attrs.mutable().durability(durability);
         command = safeCommand.updateAttributes(attrs);
@@ -1017,13 +1043,15 @@ public class Commands
                         {
                             default: throw new AssertionError("Unexpected redundant status: " + redundantStatus);
                             case NOT_OWNED: throw new AssertionError("Invalid state: waiting for execution of command that is not owned at the execution time");
+                            case SHARD_REDUNDANT:
                             case LOCALLY_REDUNDANT:
-                            case PRE_BOOTSTRAP:
-                                removeRedundantDependencies(safeStore, prevSafe, txnIds[depth], redundantStatus == PRE_BOOTSTRAP);
+                            case REDUNDANT_PRE_BOOTSTRAP_OR_STALE:
+                            case PRE_BOOTSTRAP_OR_STALE:
+                                removeRedundantDependencies(safeStore, prevSafe, txnIds[depth]);
                                 prevSafe = get(safeStore, --depth - 1);
                                 break;
                             case LIVE:
-                            case PARTIALLY_PRE_BOOTSTRAP:
+                            case PARTIALLY_PRE_BOOTSTRAP_OR_STALE:
                                 initialise(safeStore, depth);
                         }
                     }
@@ -1085,18 +1113,23 @@ public class Commands
                 }
                 else if ((directlyBlockedOnCommit = waitingOn.nextWaitingOnCommit()) != null)
                 {
-                    push(directlyBlockedOnCommit, LocalExecution.ReadyToExclude);
+                    push(directlyBlockedOnCommit, WaitingToExecute);
                     prevSafe = curSafe;
                 }
                 else
                 {
-                    if (cur.hasBeen(Committed) && !cur.is(ReadyToExecute) && cur.saveStatus() != Applying && !cur.asCommitted().isWaitingOnDependency())
+                    if (cur.hasBeen(Committed))
                     {
-                        if (!maybeExecute(safeStore, curSafe, false, false))
-                            throw new AssertionError("Is able to Apply, but has not done so");
-                        // loop and re-test the command's status; we may still want to notify blocking, esp. if not homeShard
-                        continue;
+                        if (!cur.is(ReadyToExecute) && cur.saveStatus() != Applying && !cur.asCommitted().isWaitingOnDependency())
+                        {
+                            if (!maybeExecute(safeStore, curSafe, false, false))
+                                throw new AssertionError("Is able to Apply, but has not done so");
+                            // loop and re-test the command's status; we may still want to notify blocking, esp. if not homeShard
+                            continue;
+                        }
                     }
+                    else if (!cur.hasBeen(PreCommitted))
+                        until = ReadyToExclude;
 
                     Timestamp executeAt = prev == null ? cur.executeAt() : prev.executeAt();
                     Participants<?> participants = prev != null // TODO (desired): slightly costly to invert a large partialDeps collection
@@ -1109,22 +1142,23 @@ public class Commands
                         default: throw new AssertionError("Unknown redundant status: " + redundantStatus);
                         case NOT_OWNED: throw new AssertionError("Invalid state: waiting for execution of command that is not owned at the execution time");
                         case LIVE:
-                        case PARTIALLY_PRE_BOOTSTRAP:
+                        case PARTIALLY_PRE_BOOTSTRAP_OR_STALE:
                             logger.trace("{} blocked on {} until {}", txnIds[0], cur.txnId(), until);
                             safeStore.progressLog().waiting(curSafe, until, null, participants);
                             break loop;
 
-                        case PRE_BOOTSTRAP:
+                        case PRE_BOOTSTRAP_OR_STALE:
+                        case REDUNDANT_PRE_BOOTSTRAP_OR_STALE:
                         case LOCALLY_REDUNDANT:
-                            Invariants.checkState(cur.hasBeen(Applied) || !cur.hasBeen(PreCommitted) || redundantStatus == PRE_BOOTSTRAP);
+                        case SHARD_REDUNDANT:
+                            Invariants.checkState(cur.hasBeen(Applied) || !cur.hasBeen(PreCommitted) || redundantStatus == PRE_BOOTSTRAP_OR_STALE);
                             if (prev == null)
                                 return;
 
                             curSafe.removeListener(prev.asListener());
 
                             // we've been applied, invalidated, or are no longer relevant
-                            if (cur.hasBeen(Applied)) removeInvalidatedAppliedOrTruncatedDependency(prevSafe, cur.txnId());
-                            else removeRedundantDependencies(safeStore, prevSafe, cur.txnId(), redundantStatus == PRE_BOOTSTRAP);
+                            removeRedundantDependencies(safeStore, prevSafe, cur.txnId());
 
                             --depth;
                             prevSafe = get(safeStore, depth - 1);
@@ -1184,15 +1218,7 @@ public class Commands
         }
     }
 
-    static Command removeInvalidatedAppliedOrTruncatedDependency(SafeCommand safeCommand, TxnId redundant)
-    {
-        Command.Committed current = safeCommand.current().asCommitted();
-        WaitingOn.Update update = new WaitingOn.Update(current.waitingOn);
-        update.removeInvalidatedAppliedOrTruncated(redundant);
-        return safeCommand.updateWaitingOn(update);
-    }
-
-    static Command removeRedundantDependencies(SafeCommandStore safeStore, SafeCommand safeCommand, TxnId redundant, boolean isPreBootstrap)
+    static Command removeRedundantDependencies(SafeCommandStore safeStore, SafeCommand safeCommand, TxnId redundant)
     {
         CommandStore commandStore = safeStore.commandStore();
         Command.Committed current = safeCommand.current().asCommitted();
@@ -1201,8 +1227,8 @@ public class Commands
         if (minWaitingOnTxnId != null && commandStore.hasLocallyRedundantDependencies(update.minWaitingOnTxnId(), current.executeAt(), current.route().participants()))
             safeStore.commandStore().removeRedundantDependencies(current.route().participants(), update);
 
-        if (isPreBootstrap) update.removeWaitingOn(redundant); // above operation not guaranteed to remove all redundant dependencies; transaction may
-        else update.removeInvalidatedAppliedOrTruncated(redundant);
+        // if we are a range transaction, being redundant for this transaction does not imply we are redundant for all transactions
+        update.removeWaitingOn(redundant);
         return safeCommand.updateWaitingOn(update);
     }
 
@@ -1249,7 +1275,20 @@ public class Commands
         return progressKey.equals(route.homeKey()) ? Home : Local;
     }
 
-    enum EnsureAction { Ignore, Check, Add, TrySet, Set }
+    enum EnsureAction
+    {
+        /** Don't check */
+        Ignore,
+        /** Add, but return false if insufficient for any reason */
+        TryAdd,
+        /** Supplement existing information, asserting that the existing and additional information are independently sufficient,
+         * returning false only if the existing information is absent AND the new information is insufficient. */
+        Add,
+        /** Set, but only return false if insufficient */
+        TrySet,
+        /** Overwrite existing information if sufficient; fail otherwise */
+        Set
+    }
 
     @SuppressWarnings({"unchecked", "rawtypes"})
     private static CommonAttributes set(SafeCommandStore safeStore, Command command, CommonAttributes attrs,
@@ -1320,10 +1359,21 @@ public class Commands
      * Validate we have sufficient information for the route, partialTxn and partialDeps fields, and if so update them;
      * otherwise return false (or throw an exception if an illegal state is encountered)
      */
-    private static boolean validate(Status status, CommonAttributes attrs, Ranges existingRanges, Ranges additionalRanges, ProgressShard shard,
+    private static boolean validate(Status status, CommonAttributes attrs,
+                                    Ranges existingRanges, Ranges additionalRanges, ProgressShard shard,
                                     Route<?> route, EnsureAction ensureRoute,
                                     @Nullable PartialTxn partialTxn, EnsureAction ensurePartialTxn,
                                     @Nullable PartialDeps partialDeps, EnsureAction ensurePartialDeps)
+    {
+        return validate(status, attrs, existingRanges, additionalRanges, shard, route, ensureRoute, partialTxn, ensurePartialTxn, partialDeps, ensurePartialDeps, null);
+    }
+
+    private static boolean validate(Status status, CommonAttributes attrs,
+                                    Ranges existingRanges, Ranges additionalRanges, ProgressShard shard,
+                                    Route<?> route, EnsureAction ensureRoute,
+                                    @Nullable PartialTxn partialTxn, EnsureAction ensurePartialTxn,
+                                    @Nullable PartialDeps partialDeps, EnsureAction ensurePartialDeps,
+                                    @Nullable SafeCommandStore permitStaleMissing)
     {
         if (shard == Unsure)
             return false;
@@ -1332,12 +1382,12 @@ public class Commands
         switch (ensureRoute)
         {
             default: throw new AssertionError("Unexpected action: " + ensureRoute);
-            case Check:
+            case TryAdd:
+            case Add:
                 if (!isFullRoute(attrs.route()) && !isFullRoute(route))
                     return false;
             case Ignore:
                 break;
-            case Add:
             case Set:
                 if (!isFullRoute(route))
                     throw new IllegalArgumentException("Incomplete route (" + route + ") sent to home shard");
@@ -1353,90 +1403,80 @@ public class Commands
             Invariants.checkState(status != Accepted && status != AcceptedInvalidate);
 
         // validate new partial txn
-        if (!validate(ensurePartialTxn, existingRanges, additionalRanges, covers(attrs.partialTxn()), covers(partialTxn), "txn", partialTxn))
+        if (!validate(ensurePartialTxn, existingRanges, additionalRanges, covers(attrs.partialTxn()), covers(partialTxn), permitStaleMissing, "txn", partialTxn))
             return false;
 
-        Invariants.checkState(partialTxn == null || attrs.txnId().rw() == null || attrs.txnId().rw().equals(partialTxn.kind()), "Transaction has different kind to its TxnId");
-        Invariants.checkState(!shard.isHome() || ensurePartialTxn == Ignore || hasQuery(attrs.partialTxn()) || hasQuery(partialTxn), "Home transaction should include query");
+        Invariants.checkState(partialTxn == null || attrs.txnId().rw().equals(partialTxn.kind()), "Transaction has different kind to its TxnId");
+        Invariants.checkState(partialTxn == null || !shard.isHome() || ensurePartialTxn == Ignore || hasQuery(attrs.partialTxn()) || hasQuery(partialTxn), "Home transaction should include query");
 
-        return validate(ensurePartialDeps, existingRanges, additionalRanges, covers(attrs.partialDeps()), covers(partialDeps), "deps", partialDeps);
+        return validate(ensurePartialDeps, existingRanges, additionalRanges, covers(attrs.partialDeps()), covers(partialDeps), permitStaleMissing, "deps", partialDeps);
     }
 
     // FIXME (immutable-state): has this been removed?
-    private static boolean validate(EnsureAction action, Ranges existingRanges, Ranges additionalRanges,
-                                    Ranges existing, Ranges adding, String kind, Object obj)
+    private static boolean validate(EnsureAction action, Ranges existingRanges, Ranges requiredRanges,
+                                    Ranges existing, Ranges adding, @Nullable SafeCommandStore permitStaleMissing,
+                                    String kind, Object obj)
     {
         switch (action)
         {
             default: throw new IllegalStateException("Unexpected action: " + action);
             case Ignore:
-                break;
+                return true;
 
             case TrySet:
-                if (adding != null)
+            case Set:
+                if (containsAll(adding, requiredRanges, permitStaleMissing))
+                    return true;
+
+                if (action == Set)
+                    illegalState("Incomplete " + kind + " (" + obj + ") provided; does not cover " + requiredRanges.subtract(adding));
+
+                return false;
+
+            case TryAdd:
+            case Add:
+                if (existing == null)
                 {
-                    if (!adding.containsAll(existingRanges))
-                        return false;
+                    if (adding == null)
+                        return false; // we don't want to permit a null value for txn/deps, even if we are stale for all participating ranges, as it breaks assumptions elsewhere
 
-                    if (additionalRanges != existingRanges && !adding.containsAll(additionalRanges))
+                    if (!adding.containsAll(existingRanges))
                         return false;
 
-                    break;
+                    return validate(action == TryAdd ? TrySet : Set, existingRanges, requiredRanges, existing, adding, permitStaleMissing, kind, obj);
                 }
-            case Set:
-                // failing any of these tests is always an illegal state
-                Invariants.checkState(adding != null);
-                if (!adding.containsAll(existingRanges))
-                    throw new IllegalArgumentException("Incomplete " + kind + " (" + obj + ") provided; does not cover " + existingRanges);
 
-                if (additionalRanges != existingRanges && !adding.containsAll(additionalRanges))
-                    throw new IllegalArgumentException("Incomplete " + kind + " (" + obj + ") provided; does not cover " + additionalRanges);
-                break;
+                Invariants.checkState(existing.containsAll(existingRanges), "Existing ranges insufficient");
+                if (requiredRanges == existingRanges)
+                    return true;
 
-            case Check:
-            case Add:
                 if (adding == null)
-                {
-                    if (existing == null)
-                        return false;
+                    return permitStaleMissing != null && containsAll(Ranges.EMPTY, requiredRanges.subtract(existing), permitStaleMissing);
 
-                    Invariants.checkState(existing.containsAll(existingRanges));
-                    if (existingRanges != additionalRanges && !existing.containsAll(additionalRanges))
-                    {
-                        if (action == Check)
-                            return false;
+                requiredRanges = requiredRanges.subtract(existing);
+                if (containsAll(adding, requiredRanges, permitStaleMissing))
+                    return true;
 
-                        throw new IllegalArgumentException("Missing additional " + kind + "; existing does not cover " + additionalRanges.subtract(existingRanges));
-                    }
-                }
-                else if (existing != null)
-                {
-                    Ranges covering = adding.with(existing);
-                    Invariants.checkState(covering.containsAll(existingRanges));
-                    if (existingRanges != additionalRanges && !covering.containsAll(additionalRanges))
-                    {
-                        if (action == Check)
-                            return false;
+                if (action == Add)
+                    illegalState("Incomplete " + kind + " (" + obj + ") provided; does not cover " + requiredRanges.subtract(adding));
 
-                        throw new IllegalArgumentException("Incomplete additional " + kind + " (" + obj + ") provided; does not cover " + additionalRanges.subtract(existingRanges));
-                    }
-                }
-                else
-                {
-                    if (!adding.containsAll(existingRanges))
-                        return false;
+                return false;
+        }
+    }
 
-                    if (existingRanges != additionalRanges && !adding.containsAll(additionalRanges))
-                    {
-                        if (action == Check)
-                            return false;
+    private static boolean containsAll(Ranges adding, Ranges requiredRanges, @Nullable SafeCommandStore permitStaleMissing)
+    {
+        if (adding.containsAll(requiredRanges))
+            return true;
 
-                        throw new IllegalArgumentException("Incomplete additional " + kind + " (" + obj + ") provided; does not cover " + additionalRanges.subtract(existingRanges));
-                    }
-                }
-                break;
+        if (permitStaleMissing != null)
+        {
+            Ranges staleRanges = permitStaleMissing.commandStore().redundantBefore().staleRanges();
+            requiredRanges = requiredRanges.subtract(staleRanges);
+            if (adding.containsAll(requiredRanges))
+                return true;
         }
 
-        return true;
+        return false;
     }
 }
diff --git a/accord-core/src/main/java/accord/local/DurableBefore.java b/accord-core/src/main/java/accord/local/DurableBefore.java
index abe8580e..b8c13d98 100644
--- a/accord-core/src/main/java/accord/local/DurableBefore.java
+++ b/accord-core/src/main/java/accord/local/DurableBefore.java
@@ -27,15 +27,15 @@ import accord.local.Status.Durability;
 import accord.primitives.Participants;
 import accord.primitives.Range;
 import accord.primitives.Ranges;
-import accord.primitives.Routables;
 import accord.primitives.TxnId;
+import accord.primitives.Unseekables;
 import accord.utils.Invariants;
 import accord.utils.ReducingIntervalMap;
 import accord.utils.ReducingRangeMap;
 
-import static accord.local.Status.Durability.Majority;
+import static accord.local.Status.Durability.MajorityOrInvalidated;
 import static accord.local.Status.Durability.NotDurable;
-import static accord.local.Status.Durability.Universal;
+import static accord.local.Status.Durability.UniversalOrInvalidated;
 
 public class DurableBefore extends ReducingRangeMap<DurableBefore.Entry>
 {
@@ -84,7 +84,7 @@ public class DurableBefore extends ReducingRangeMap<DurableBefore.Entry>
         public Durability get(TxnId txnId)
         {
             if (txnId.compareTo(majorityBefore) < 0)
-                return txnId.compareTo(universalBefore) < 0 ? Universal : Majority;
+                return txnId.compareTo(universalBefore) < 0 ? UniversalOrInvalidated : MajorityOrInvalidated;
             return NotDurable;
         }
 
@@ -175,14 +175,14 @@ public class DurableBefore extends ReducingRangeMap<DurableBefore.Entry>
         return ReducingIntervalMap.merge(a, b, DurableBefore.Entry::max, Builder::new);
     }
 
-    public Durability min(TxnId txnId, Routables<?> routables)
+    public Durability min(TxnId txnId, Unseekables<?> unseekables)
     {
-        return notDurableIfNull(foldl(routables, Entry::mergeMin, null, txnId, test -> test == NotDurable));
+        return notDurableIfNull(foldl(unseekables, Entry::mergeMin, null, txnId, test -> test == NotDurable));
     }
 
-    public Durability max(TxnId txnId, Routables<?> routables)
+    public Durability max(TxnId txnId, Unseekables<?> unseekables)
     {
-        return notDurableIfNull(foldl(routables, Entry::mergeMax, null, txnId, test -> test == Universal));
+        return notDurableIfNull(foldl(unseekables, Entry::mergeMax, null, txnId, test -> test == UniversalOrInvalidated));
     }
 
     public Durability get(TxnId txnId, RoutingKey participant)
@@ -191,14 +191,14 @@ public class DurableBefore extends ReducingRangeMap<DurableBefore.Entry>
         return entry == null ? NotDurable : entry.get(txnId);
     }
 
-    public boolean isUniversal(TxnId txnId, Routables<?> participants)
+    public boolean isUniversal(TxnId txnId, Unseekables<?> participants)
     {
-        return min(txnId, participants) == Universal;
+        return min(txnId, participants) == UniversalOrInvalidated;
     }
 
     public boolean isUniversal(TxnId txnId, RoutingKey participant)
     {
-        return get(txnId, participant) == Universal;
+        return get(txnId, participant) == UniversalOrInvalidated;
     }
 
     public boolean isSomeShardDurable(TxnId txnId, Participants<?> participants, Durability durability)
@@ -209,9 +209,9 @@ public class DurableBefore extends ReducingRangeMap<DurableBefore.Entry>
     public Durability min(TxnId txnId)
     {
         if (min.universalBefore.compareTo(txnId) > 0)
-            return Universal;
+            return UniversalOrInvalidated;
         if (min.majorityBefore.compareTo(txnId) > 0)
-            return Majority;
+            return MajorityOrInvalidated;
         return NotDurable;
     }
 
diff --git a/accord-core/src/main/java/accord/local/Node.java b/accord-core/src/main/java/accord/local/Node.java
index b7115b1b..2c4b8e68 100644
--- a/accord-core/src/main/java/accord/local/Node.java
+++ b/accord-core/src/main/java/accord/local/Node.java
@@ -85,6 +85,7 @@ import accord.topology.TopologyManager;
 import accord.utils.MapReduceConsume;
 import accord.utils.RandomSource;
 import accord.utils.async.AsyncChain;
+import accord.utils.async.AsyncExecutor;
 import accord.utils.async.AsyncResult;
 import accord.utils.async.AsyncResults;
 import net.nicoulaj.compilecommand.annotations.Inline;
@@ -494,7 +495,7 @@ public class Node implements ConfigurationService.Listener, NodeTimeService
         messageSink.send(to, send, executor, callback);
     }
 
-    private void checkStore(AgentExecutor executor)
+    private void checkStore(AsyncExecutor executor)
     {
         CommandStore current = CommandStore.maybeCurrent();
         if (current != null && current != executor)
diff --git a/accord-core/src/main/java/accord/local/RedundantBefore.java b/accord-core/src/main/java/accord/local/RedundantBefore.java
index 4359f1c2..b8410f18 100644
--- a/accord-core/src/main/java/accord/local/RedundantBefore.java
+++ b/accord-core/src/main/java/accord/local/RedundantBefore.java
@@ -18,24 +18,32 @@
 
 package accord.local;
 
+import java.util.Objects;
+import java.util.function.BiFunction;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
 import accord.api.RoutingKey;
+import accord.primitives.Deps;
 import accord.primitives.EpochSupplier;
 import accord.primitives.Participants;
 import accord.primitives.Range;
 import accord.primitives.Ranges;
+import accord.primitives.Seekables;
 import accord.primitives.Timestamp;
 import accord.primitives.TxnId;
 import accord.utils.Invariants;
 import accord.utils.ReducingIntervalMap;
 import accord.utils.ReducingRangeMap;
 
+import static accord.local.RedundantBefore.PreBootstrapOrStale.FULLY;
+import static accord.local.RedundantBefore.PreBootstrapOrStale.POST_BOOTSTRAP;
+import static accord.local.RedundantBefore.PreBootstrapOrStale.PARTIALLY;
 import static accord.local.RedundantStatus.LIVE;
-import static accord.local.RedundantStatus.NOT_OWNED;
-import static accord.local.RedundantStatus.PRE_BOOTSTRAP;
 import static accord.local.RedundantStatus.LOCALLY_REDUNDANT;
+import static accord.local.RedundantStatus.NOT_OWNED;
+import static accord.local.RedundantStatus.PRE_BOOTSTRAP_OR_STALE;
+import static accord.local.RedundantStatus.SHARD_REDUNDANT;
 
 public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
 {
@@ -47,19 +55,62 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
         }
     }
 
+    public enum PreBootstrapOrStale { NOT_OWNED, FULLY, PARTIALLY, POST_BOOTSTRAP }
+
     public static class Entry
     {
+        // TODO (desired): we don't need to maintain this now, and can simplify our builder, by migrating to ReducingRangeMap.foldWithBounds
         public final Range range;
         public final long startEpoch, endEpoch;
-        public final @Nonnull TxnId redundantBefore, bootstrappedAt;
 
-        public Entry(Range range, long startEpoch, long endEpoch, @Nonnull TxnId redundantBefore, @Nonnull TxnId bootstrappedAt)
+        /**
+         * Represents the maximum TxnId we know to have fully executed until locally for the range in question.
+         * Unless we are stale or pre-bootstrap, in which case no such guarantees can be made.
+         */
+        public final @Nonnull TxnId locallyAppliedOrInvalidatedBefore;
+
+        /**
+         * Represents the maximum TxnId we know to have fully executed until across all healthy replicas for the range in question.
+         * Unless we are stale or pre-bootstrap, in which case no such guarantees can be made.
+         */
+        public final @Nonnull TxnId shardAppliedOrInvalidatedBefore;
+
+        /**
+         * bootstrappedAt defines the txnId bounds we expect to maintain data for locally.
+         *
+         * We can bootstrap ranges at different times, and have a transaction that participates in both ranges -
+         * in this case one of the portions of the transaction may be totally unordered with respect to other transactions
+         * in that range because both occur prior to the bootstrappedAt point, so their dependencies are entirely erased.
+         * We can also re-bootstrap the same range because bootstrap failed, and leave dangling transactions to execute
+         * which then execute in an unordered fashion.
+         *
+         * See also {@link CommandStore#safeToRead}.
+         */
+        public final @Nonnull TxnId bootstrappedAt;
+
+        /**
+         * The maximum of each of locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore and bootstrappedAt.
+         */
+        public final @Nonnull TxnId minConflict;
+
+        /**
+         * staleUntilAtLeast provides a minimum TxnId until which we know we will be unable to completely execute
+         * transactions locally for the impacted range.
+         *
+         * See also {@link CommandStore#safeToRead}.
+         */
+        public final @Nullable Timestamp staleUntilAtLeast;
+
+        public Entry(Range range, long startEpoch, long endEpoch, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId bootstrappedAt, @Nullable Timestamp staleUntilAtLeast)
         {
             this.range = range;
             this.startEpoch = startEpoch;
             this.endEpoch = endEpoch;
-            this.redundantBefore = redundantBefore;
+            this.locallyAppliedOrInvalidatedBefore = locallyAppliedOrInvalidatedBefore;
+            this.shardAppliedOrInvalidatedBefore = shardAppliedOrInvalidatedBefore;
             this.bootstrappedAt = bootstrappedAt;
+            this.minConflict = Timestamp.max(Timestamp.max(locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore), bootstrappedAt);
+            this.staleUntilAtLeast = staleUntilAtLeast;
         }
 
         public static Entry reduce(Entry a, Entry b)
@@ -69,28 +120,42 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
 
         private static Entry merge(Range range, Entry a, Entry b)
         {
+            if (a.startEpoch > b.endEpoch)
+                return a;
+
+            if (b.startEpoch > a.endEpoch)
+                return b;
+
             long startEpoch = Long.max(a.startEpoch, b.startEpoch);
             long endEpoch = Long.min(a.endEpoch, b.endEpoch);
-            int cr = a.redundantBefore.compareTo(b.redundantBefore);
+            int cl = a.locallyAppliedOrInvalidatedBefore.compareTo(b.locallyAppliedOrInvalidatedBefore);
+            int cs = a.shardAppliedOrInvalidatedBefore.compareTo(b.shardAppliedOrInvalidatedBefore);
             int cb = a.bootstrappedAt.compareTo(b.bootstrappedAt);
+            int csu = compareStaleUntilAtLeast(a.staleUntilAtLeast, b.staleUntilAtLeast);
 
-            if (range.equals(a.range) && startEpoch == a.startEpoch && endEpoch == a.endEpoch && cr >= 0 && cb >= 0)
+            if (range.equals(a.range) && startEpoch == a.startEpoch && endEpoch == a.endEpoch && cl >= 0 && cb >= 0 && cs >= 0 && csu >= 0)
                 return a;
-            if (range.equals(b.range) && startEpoch == b.startEpoch && endEpoch == b.endEpoch && cr <= 0 && cb <= 0)
+            if (range.equals(b.range) && startEpoch == b.startEpoch && endEpoch == b.endEpoch && cl <= 0 && cb <= 0 && cs <= 0 && csu <= 0)
                 return b;
 
-            TxnId redundantBefore = cr >= 0 ? a.redundantBefore : b.redundantBefore;
+            TxnId locallyAppliedOrInvalidatedBefore = cl >= 0 ? a.locallyAppliedOrInvalidatedBefore : b.locallyAppliedOrInvalidatedBefore;
+            TxnId shardAppliedOrInvalidatedBefore = cs >= 0 ? a.shardAppliedOrInvalidatedBefore : b.shardAppliedOrInvalidatedBefore;
             TxnId bootstrappedAt = cb >= 0 ? a.bootstrappedAt : b.bootstrappedAt;
+            Timestamp staleUntilAtLeast = csu >= 0 ? a.staleUntilAtLeast : b.staleUntilAtLeast;
 
             // if our redundantBefore predates bootstrappedAt, we should clear it to avoid erroneously treating
             // transactions prior as locally redundant when they may simply have not applied yet, since we may
             // permit the sync point that defines redundancy to apply locally without waiting for these earlier
             // transactions, since we now consider them to be bootstrapping
             // TODO (desired): revisit later as semantics here evolve
-            if (bootstrappedAt.compareTo(redundantBefore) >= 0)
-                redundantBefore = TxnId.NONE;
-
-            return new Entry(range, startEpoch, endEpoch, redundantBefore, bootstrappedAt);
+            if (bootstrappedAt.compareTo(locallyAppliedOrInvalidatedBefore) >= 0)
+                locallyAppliedOrInvalidatedBefore = TxnId.NONE;
+            if (bootstrappedAt.compareTo(shardAppliedOrInvalidatedBefore) >= 0)
+                shardAppliedOrInvalidatedBefore = TxnId.NONE;
+            if (staleUntilAtLeast != null && bootstrappedAt.compareTo(staleUntilAtLeast) >= 0)
+                staleUntilAtLeast = null;
+
+            return new Entry(range, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, bootstrappedAt, staleUntilAtLeast);
         }
 
         static @Nonnull RedundantStatus getAndMerge(Entry entry, @Nonnull RedundantStatus prev, TxnId txnId, EpochSupplier executeAt)
@@ -107,36 +172,99 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
             return entry.get(txnId);
         }
 
+        static PreBootstrapOrStale getAndMerge(Entry entry, @Nonnull PreBootstrapOrStale prev, TxnId txnId, EpochSupplier executeAt)
+        {
+            if (prev == PARTIALLY || entry == null || entry.outOfBounds(txnId, executeAt))
+                return prev;
+
+            boolean isPreBootstrapOrStale = entry.staleUntilAtLeast != null || entry.bootstrappedAt.compareTo(txnId) > 0;
+            return isPreBootstrapOrStale ? prev == POST_BOOTSTRAP ? PARTIALLY : FULLY
+                                         : prev == FULLY          ? PARTIALLY : POST_BOOTSTRAP;
+        }
+
+        static Timestamp getAndMergeMinConflict(Entry entry, @Nonnull Timestamp prev)
+        {
+            if (entry == null)
+                return prev;
+            return Timestamp.max(prev, entry.minConflict);
+        }
+
+        static <T extends Deps> Deps.AbstractBuilder<T> collectDep(Entry entry, @Nonnull Deps.AbstractBuilder<T> prev, @Nonnull EpochSupplier minEpoch, @Nonnull EpochSupplier executeAt)
+        {
+            if (entry == null || entry.outOfBounds(minEpoch, executeAt))
+                return prev;
+
+            if (entry.shardAppliedOrInvalidatedBefore.compareTo(Timestamp.NONE) > 0)
+                prev.add(entry.range, entry.shardAppliedOrInvalidatedBefore);
+
+            return prev;
+        }
+
+        static Ranges validateSafeToRead(Entry entry, @Nonnull Ranges safeToRead, Timestamp bootstrapAt, Object ignore)
+        {
+            if (entry == null)
+                return safeToRead;
+
+            if (bootstrapAt.compareTo(entry.bootstrappedAt) < 0 || (entry.staleUntilAtLeast != null && bootstrapAt.compareTo(entry.staleUntilAtLeast) < 0))
+                return safeToRead.subtract(Ranges.of(entry.range));
+
+            return safeToRead;
+        }
+
+        static Ranges expectToExecute(Entry entry, @Nonnull Ranges executeRanges, TxnId txnId, @Nullable Timestamp executeAt)
+        {
+            if (entry == null || (executeAt == null ? entry.outOfBounds(txnId) : entry.outOfBounds(txnId, executeAt)))
+                return executeRanges;
+
+            if (txnId.compareTo(entry.bootstrappedAt) < 0 || entry.staleUntilAtLeast != null)
+                return executeRanges.subtract(Ranges.of(entry.range));
+
+            return executeRanges;
+        }
+
         RedundantStatus get(TxnId txnId)
         {
-            if (redundantBefore.compareTo(txnId) > 0)
+            // we have to first check bootstrappedAt, since we are not locally redundant for the covered range
+            // if the txnId is partially pre-bootstrap (since we may not have applied it for this range)
+            if (staleUntilAtLeast != null || bootstrappedAt.compareTo(txnId) > 0)
+                return PRE_BOOTSTRAP_OR_STALE;
+            if (locallyAppliedOrInvalidatedBefore.compareTo(txnId) > 0)
+            {
+                if (shardAppliedOrInvalidatedBefore.compareTo(txnId) > 0)
+                    return SHARD_REDUNDANT;
                 return LOCALLY_REDUNDANT;
-            if (bootstrappedAt.compareTo(txnId) > 0)
-                return PRE_BOOTSTRAP;
+            }
             return LIVE;
         }
 
-        public final boolean isLocalRedundancyViaBootstrap()
+        private static int compareStaleUntilAtLeast(@Nullable Timestamp a, @Nullable Timestamp b)
         {
-            return redundantBefore.compareTo(bootstrappedAt) < 0;
+            boolean aIsNull = a == null, bIsNull = b == null;
+            if (aIsNull != bIsNull) return aIsNull ? -1 : 1;
+            return aIsNull ? 0 : a.compareTo(b);
         }
 
-        public final TxnId locallyRedundantBefore()
+        public final TxnId shardRedundantBefore()
         {
-            return TxnId.min(bootstrappedAt, redundantBefore);
+            return shardAppliedOrInvalidatedBefore;
         }
 
         // TODO (required, consider): this admits the range of epochs that cross the two timestamps, which matches our
         //   behaviour elsewhere but we probably want to only interact with the two point epochs in which we participate,
         //   but note hasRedundantDependencies which really does want to scan a range
-        private boolean outOfBounds(Timestamp lb, EpochSupplier ub)
+        private boolean outOfBounds(EpochSupplier lb, EpochSupplier ub)
         {
             return ub.epoch() < startEpoch || lb.epoch() >= endEpoch;
         }
 
+        private boolean outOfBounds(Timestamp lb)
+        {
+            return lb.epoch() >= endEpoch;
+        }
+
         Entry withRange(Range range)
         {
-            return new Entry(range, startEpoch, endEpoch, redundantBefore, bootstrappedAt);
+            return new Entry(range, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, bootstrappedAt, staleUntilAtLeast);
         }
 
         public boolean equals(Object that)
@@ -153,7 +281,10 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
         {
             return this.startEpoch == that.startEpoch
                    && this.endEpoch == that.endEpoch
-                   && this.redundantBefore.equals(that.redundantBefore);
+                   && this.locallyAppliedOrInvalidatedBefore.equals(that.locallyAppliedOrInvalidatedBefore)
+                   && this.shardAppliedOrInvalidatedBefore.equals(that.shardAppliedOrInvalidatedBefore)
+                   && this.bootstrappedAt.equals(that.bootstrappedAt)
+                   && Objects.equals(this.staleUntilAtLeast, that.staleUntilAtLeast);
         }
 
         @Override
@@ -162,27 +293,68 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
             return "("
                    + (startEpoch == Long.MIN_VALUE ? "-\u221E" : Long.toString(startEpoch)) + ","
                    + (endEpoch == Long.MAX_VALUE ? "\u221E" : Long.toString(endEpoch)) + ","
-                   + (redundantBefore.compareTo(bootstrappedAt) >= 0 ? redundantBefore + ")" : bootstrappedAt + "*)");
+                   + (locallyAppliedOrInvalidatedBefore.compareTo(bootstrappedAt) >= 0 ? locallyAppliedOrInvalidatedBefore + ")" : bootstrappedAt + "*)");
         }
     }
 
     public static RedundantBefore EMPTY = new RedundantBefore();
 
+    private final Ranges staleRanges;
+
     private RedundantBefore()
     {
+        staleRanges = Ranges.EMPTY;
     }
 
     RedundantBefore(boolean inclusiveEnds, RoutingKey[] starts, Entry[] values)
     {
         super(inclusiveEnds, starts, values);
+        staleRanges = extractStaleRanges(values);
     }
 
-    public static RedundantBefore create(Ranges ranges, long startEpoch, long endEpoch, @Nonnull TxnId redundantBefore, @Nonnull TxnId bootstrappedAt)
+    private static Ranges extractStaleRanges(Entry[] values)
+    {
+        int countStaleRanges = 0;
+        for (Entry entry : values)
+        {
+            if (entry != null && entry.staleUntilAtLeast != null)
+                ++countStaleRanges;
+        }
+
+        if (countStaleRanges == 0)
+            return Ranges.EMPTY;
+
+        Range[] staleRanges = new Range[countStaleRanges];
+        countStaleRanges = 0;
+        for (Entry entry : values)
+        {
+            if (entry != null && entry.staleUntilAtLeast != null)
+                staleRanges[countStaleRanges++] = entry.range;
+        }
+        return Ranges.ofSortedAndDeoverlapped(staleRanges).mergeTouching();
+    }
+
+    public static RedundantBefore create(Ranges ranges, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId bootstrappedAt)
+    {
+        return create(ranges, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, bootstrappedAt, null);
+    }
+
+    public static RedundantBefore create(Ranges ranges, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId bootstrappedAt, @Nullable Timestamp staleUntilAtLeast)
+    {
+        return create(ranges, Long.MIN_VALUE, Long.MAX_VALUE, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, bootstrappedAt, staleUntilAtLeast);
+    }
+
+    public static RedundantBefore create(Ranges ranges, long startEpoch, long endEpoch, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId bootstrappedAt)
+    {
+        return create(ranges, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, bootstrappedAt, null);
+    }
+
+    public static RedundantBefore create(Ranges ranges, long startEpoch, long endEpoch, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId bootstrappedAt, @Nullable Timestamp staleUntilAtLeast)
     {
         if (ranges.isEmpty())
             return new RedundantBefore();
 
-        Entry entry = new Entry(null, startEpoch, endEpoch, redundantBefore, bootstrappedAt);
+        Entry entry = new Entry(null, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, bootstrappedAt, staleUntilAtLeast);
         Builder builder = new Builder(ranges.get(0).endInclusive(), ranges.size() * 2);
         for (int i = 0 ; i < ranges.size() ; ++i)
         {
@@ -211,6 +383,57 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
         return foldl(participants, Entry::getAndMerge, NOT_OWNED, txnId, executeAt, ignore -> false);
     }
 
+    /**
+     * RedundantStatus.REDUNDANT overrides PRE_BOOTSTRAP; to avoid complicating that state machine,
+     * for cases where we care independently about the overall pre-bootstrap state we have a separate mechanism
+     */
+    public PreBootstrapOrStale preBootstrapOrStale(TxnId txnId, @Nullable EpochSupplier executeAt, Participants<?> participants)
+    {
+        if (executeAt == null) executeAt = txnId;
+        return foldl(participants, Entry::getAndMerge, PreBootstrapOrStale.NOT_OWNED, txnId, executeAt, r -> r == PARTIALLY);
+    }
+
+    public Timestamp minConflict(Seekables<?, ?> participants)
+    {
+        return foldl(participants, Entry::getAndMergeMinConflict, Timestamp.NONE, ignore -> false);
+    }
+
+    public <T extends Deps> Deps.AbstractBuilder<T> collectDeps(Seekables<?, ?> participants, Deps.AbstractBuilder<T> builder, EpochSupplier minEpoch, EpochSupplier executeAt)
+    {
+        return foldl(participants, Entry::collectDep, builder, minEpoch, executeAt, ignore -> false);
+    }
+
+    public Ranges validateSafeToRead(Timestamp forBootstrapAt, Ranges ranges)
+    {
+        return foldl(ranges, Entry::validateSafeToRead, ranges, forBootstrapAt, null, r -> false);
+    }
+
+    /**
+     * Subtract any ranges we consider stale or pre-bootstrap
+     */
+    public Ranges expectToExecute(TxnId txnId, @Nonnull Timestamp executeAt, Ranges ranges)
+    {
+        Invariants.checkArgument(executeAt != null, "executeAt must not be null");
+        return foldl(ranges, Entry::expectToExecute, ranges, txnId, executeAt, r -> false);
+    }
+
+    /**
+     * Subtract any ranges we consider stale or pre-bootstrap at any point
+     */
+    public Ranges everExpectToExecute(TxnId txnId, Ranges ranges)
+    {
+        return foldl(ranges, Entry::expectToExecute, ranges, txnId, null, r -> false);
+    }
+
+    /**
+     * RedundantStatus.REDUNDANT overrides PRE_BOOTSTRAP; to avoid complicating that state machine,
+     * for cases where we care independently about the overall pre-bootstrap state we have a separate mechanism
+     */
+    public Ranges staleRanges()
+    {
+        return staleRanges;
+    }
+
     static class Builder extends ReducingIntervalMap.Builder<RoutingKey, Entry, RedundantBefore>
     {
         protected Builder(boolean inclusiveEnds, int capacity)
@@ -224,6 +447,23 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
             return a.equalsIgnoreRange(b);
         }
 
+        // we maintain the range that an Entry applies to for ease of some other functionality - this logic is applied here in append and above by overriding equals.
+        // entries may be null for ranges we do not have any knowledge for; the final "entry" we append is expected to be null by the ReducingIntervalMap.Builder
+        // TODO (desired): consider migrating to foldlWithBounds
+        @Override
+        public void append(RoutingKey start, @Nullable Entry value, BiFunction<Entry, Entry, Entry> reduce)
+        {
+            if (value != null && value.range.start().compareTo(start) < 0)
+                value = value.withRange(value.range.newRange(start, value.range.end()));
+
+            int tailIdx = values.size() - 1;
+            super.append(start, value, reduce);
+
+            Entry tailValue; // TODO (desired): clean up maintenance of accurate range bounds
+            if (values.size() - 2 == tailIdx && tailIdx >= 0 && (tailValue = values.get(tailIdx)) != null && tailValue.range.end().compareTo(start) > 0)
+                values.set(tailIdx, tailValue.withRange(tailValue.range.newRange(tailValue.range.start(), start)));
+        }
+
         @Override
         protected Entry mergeEqual(Entry a, Entry b)
         {
@@ -231,7 +471,7 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
             return new Entry(a.range.newRange(
                 a.range.start().compareTo(b.range.start()) <= 0 ? a.range.start() : b.range.start(),
                 a.range.end().compareTo(b.range.end()) >= 0 ? a.range.end() : b.range.end()
-            ), a.startEpoch, a.endEpoch, a.redundantBefore, a.bootstrappedAt);
+            ), a.startEpoch, a.endEpoch, a.locallyAppliedOrInvalidatedBefore, a.shardAppliedOrInvalidatedBefore, a.bootstrappedAt, a.staleUntilAtLeast);
         }
 
         @Override
diff --git a/accord-core/src/main/java/accord/local/RedundantStatus.java b/accord-core/src/main/java/accord/local/RedundantStatus.java
index 15a5ccc8..63eecd34 100644
--- a/accord-core/src/main/java/accord/local/RedundantStatus.java
+++ b/accord-core/src/main/java/accord/local/RedundantStatus.java
@@ -24,22 +24,52 @@ import accord.utils.Invariants;
 
 public enum RedundantStatus
 {
-    /** None of the relevant ranges are owned by the command store */
+    /**
+     * None of the relevant ranges are owned by the command store
+     */
     NOT_OWNED,
 
-    /** Some of the relevant ranges are owned by the command store and valid for execution */
+    /**
+     * Some of the relevant ranges are owned by the command store and valid for execution
+     */
     LIVE,
 
-    /** The relevant owned ranges are part live and part pre-bootstrap */
-    PARTIALLY_PRE_BOOTSTRAP,
+    /**
+     * The relevant owned ranges are part live and part pre-bootstrap or stale
+     */
+    PARTIALLY_PRE_BOOTSTRAP_OR_STALE,
 
-    /** The relevant owned ranges are ALL pre-bootstrap, meaning we are fetching the transaction's entire result
-     * from another node's snapshot already */
-    PRE_BOOTSTRAP,
+    /**
+     * The relevant owned ranges are ALL pre-bootstrap or stale, meaning we are either
+     *  1) fetching the transaction's entire result from another node's snapshot already; or
+     *  2) the range is stale and _must be bootstrapped_
+     */
+    PRE_BOOTSTRAP_OR_STALE,
 
-    /** The relevant owned ranges are fully redundant, meaning they are known applied or invalidated via a sync point
-     * that has applied locally */
+    /**
+     * Some of the relevant owned ranges are redundant, meaning any intersecting transaction is known to be either applied
+     * or invalidated via a sync point that has applied locally, but other ranges are known to be pre-bootstrap or stale
+     * so that we cannot treat the transaction as having been fully applied locally.
+     *
+     * Note that this status overrides PRE_BOOTSTRAP_OR_STALE, since it implies the transaction has applied.
+     */
+    REDUNDANT_PRE_BOOTSTRAP_OR_STALE,
+
+    /**
+     * The relevant owned ranges are redundant, meaning any intersecting transaction is known to be either applied or invalidated
+     * via a sync point that has applied locally.
+     *
+     * Note that this status overrides PRE_BOOTSTRAP_OR_STALE, since it implies the transaction has applied.
+     */
     LOCALLY_REDUNDANT,
+
+    /**
+     * The relevant owned ranges are redundant, meaning any intersecting transaction is known to be either applied or invalidated
+     * via a sync point that has applied locally and on all healthy shards.
+     *
+     * Note that this status overrides PRE_BOOTSTRAP_OR_STALE, since it implies the transaction has applied.
+     */
+    SHARD_REDUNDANT,
     ;
 
     private EnumMap<RedundantStatus, RedundantStatus> merge;
@@ -49,33 +79,59 @@ public enum RedundantStatus
         NOT_OWNED.merge = new EnumMap<>(RedundantStatus.class);
         NOT_OWNED.merge.put(NOT_OWNED, NOT_OWNED);
         NOT_OWNED.merge.put(LIVE, LIVE);
-        NOT_OWNED.merge.put(PARTIALLY_PRE_BOOTSTRAP, PARTIALLY_PRE_BOOTSTRAP);
-        NOT_OWNED.merge.put(PRE_BOOTSTRAP, PRE_BOOTSTRAP);
+        NOT_OWNED.merge.put(PARTIALLY_PRE_BOOTSTRAP_OR_STALE, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
+        NOT_OWNED.merge.put(PRE_BOOTSTRAP_OR_STALE, PRE_BOOTSTRAP_OR_STALE);
+        NOT_OWNED.merge.put(REDUNDANT_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
         NOT_OWNED.merge.put(LOCALLY_REDUNDANT, LOCALLY_REDUNDANT);
+        NOT_OWNED.merge.put(SHARD_REDUNDANT, SHARD_REDUNDANT);
         LIVE.merge = new EnumMap<>(RedundantStatus.class);
         LIVE.merge.put(NOT_OWNED, LIVE);
         LIVE.merge.put(LIVE, LIVE);
-        LIVE.merge.put(PARTIALLY_PRE_BOOTSTRAP, PARTIALLY_PRE_BOOTSTRAP);
-        LIVE.merge.put(PRE_BOOTSTRAP, PARTIALLY_PRE_BOOTSTRAP);
-        LIVE.merge.put(LOCALLY_REDUNDANT, LOCALLY_REDUNDANT);
-        PARTIALLY_PRE_BOOTSTRAP.merge = new EnumMap<>(RedundantStatus.class);
-        PARTIALLY_PRE_BOOTSTRAP.merge.put(NOT_OWNED, PARTIALLY_PRE_BOOTSTRAP);
-        PARTIALLY_PRE_BOOTSTRAP.merge.put(LIVE, PARTIALLY_PRE_BOOTSTRAP);
-        PARTIALLY_PRE_BOOTSTRAP.merge.put(PARTIALLY_PRE_BOOTSTRAP, PARTIALLY_PRE_BOOTSTRAP);
-        PARTIALLY_PRE_BOOTSTRAP.merge.put(PRE_BOOTSTRAP, PARTIALLY_PRE_BOOTSTRAP);
-        PARTIALLY_PRE_BOOTSTRAP.merge.put(LOCALLY_REDUNDANT, LOCALLY_REDUNDANT);
-        PRE_BOOTSTRAP.merge = new EnumMap<>(RedundantStatus.class);
-        PRE_BOOTSTRAP.merge.put(NOT_OWNED, PRE_BOOTSTRAP);
-        PRE_BOOTSTRAP.merge.put(LIVE, PARTIALLY_PRE_BOOTSTRAP);
-        PRE_BOOTSTRAP.merge.put(PARTIALLY_PRE_BOOTSTRAP, PARTIALLY_PRE_BOOTSTRAP);
-        PRE_BOOTSTRAP.merge.put(PRE_BOOTSTRAP, PRE_BOOTSTRAP);
-        PRE_BOOTSTRAP.merge.put(LOCALLY_REDUNDANT, LOCALLY_REDUNDANT);
+        LIVE.merge.put(PARTIALLY_PRE_BOOTSTRAP_OR_STALE, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
+        LIVE.merge.put(PRE_BOOTSTRAP_OR_STALE, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
+        LIVE.merge.put(REDUNDANT_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+        LIVE.merge.put(LOCALLY_REDUNDANT, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+        LIVE.merge.put(SHARD_REDUNDANT, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+        PARTIALLY_PRE_BOOTSTRAP_OR_STALE.merge = new EnumMap<>(RedundantStatus.class);
+        PARTIALLY_PRE_BOOTSTRAP_OR_STALE.merge.put(NOT_OWNED, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
+        PARTIALLY_PRE_BOOTSTRAP_OR_STALE.merge.put(LIVE, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
+        PARTIALLY_PRE_BOOTSTRAP_OR_STALE.merge.put(PARTIALLY_PRE_BOOTSTRAP_OR_STALE, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
+        PARTIALLY_PRE_BOOTSTRAP_OR_STALE.merge.put(PRE_BOOTSTRAP_OR_STALE, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
+        PARTIALLY_PRE_BOOTSTRAP_OR_STALE.merge.put(REDUNDANT_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+        PARTIALLY_PRE_BOOTSTRAP_OR_STALE.merge.put(LOCALLY_REDUNDANT, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+        PARTIALLY_PRE_BOOTSTRAP_OR_STALE.merge.put(SHARD_REDUNDANT, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+        PRE_BOOTSTRAP_OR_STALE.merge = new EnumMap<>(RedundantStatus.class);
+        PRE_BOOTSTRAP_OR_STALE.merge.put(NOT_OWNED, PRE_BOOTSTRAP_OR_STALE);
+        PRE_BOOTSTRAP_OR_STALE.merge.put(LIVE, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
+        PRE_BOOTSTRAP_OR_STALE.merge.put(PARTIALLY_PRE_BOOTSTRAP_OR_STALE, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
+        PRE_BOOTSTRAP_OR_STALE.merge.put(PRE_BOOTSTRAP_OR_STALE, PRE_BOOTSTRAP_OR_STALE);
+        PRE_BOOTSTRAP_OR_STALE.merge.put(REDUNDANT_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+        PRE_BOOTSTRAP_OR_STALE.merge.put(LOCALLY_REDUNDANT, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+        PRE_BOOTSTRAP_OR_STALE.merge.put(SHARD_REDUNDANT, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+        REDUNDANT_PRE_BOOTSTRAP_OR_STALE.merge = new EnumMap<>(RedundantStatus.class);
+        REDUNDANT_PRE_BOOTSTRAP_OR_STALE.merge.put(NOT_OWNED, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+        REDUNDANT_PRE_BOOTSTRAP_OR_STALE.merge.put(LIVE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE); // TODO (expected): should this be an invalid combination?
+        REDUNDANT_PRE_BOOTSTRAP_OR_STALE.merge.put(PARTIALLY_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+        REDUNDANT_PRE_BOOTSTRAP_OR_STALE.merge.put(PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+        REDUNDANT_PRE_BOOTSTRAP_OR_STALE.merge.put(REDUNDANT_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+        REDUNDANT_PRE_BOOTSTRAP_OR_STALE.merge.put(LOCALLY_REDUNDANT, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+        REDUNDANT_PRE_BOOTSTRAP_OR_STALE.merge.put(SHARD_REDUNDANT, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
         LOCALLY_REDUNDANT.merge = new EnumMap<>(RedundantStatus.class);
         LOCALLY_REDUNDANT.merge.put(NOT_OWNED, LOCALLY_REDUNDANT);
-        LOCALLY_REDUNDANT.merge.put(LIVE, LOCALLY_REDUNDANT);
-        LOCALLY_REDUNDANT.merge.put(PARTIALLY_PRE_BOOTSTRAP, LOCALLY_REDUNDANT);
-        LOCALLY_REDUNDANT.merge.put(PRE_BOOTSTRAP, LOCALLY_REDUNDANT);
+        LOCALLY_REDUNDANT.merge.put(LIVE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE); // TODO (expected): should this be an invalid combination?
+        LOCALLY_REDUNDANT.merge.put(PARTIALLY_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+        LOCALLY_REDUNDANT.merge.put(PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+        LOCALLY_REDUNDANT.merge.put(REDUNDANT_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
         LOCALLY_REDUNDANT.merge.put(LOCALLY_REDUNDANT, LOCALLY_REDUNDANT);
+        LOCALLY_REDUNDANT.merge.put(SHARD_REDUNDANT, LOCALLY_REDUNDANT);
+        SHARD_REDUNDANT.merge = new EnumMap<>(RedundantStatus.class);
+        SHARD_REDUNDANT.merge.put(NOT_OWNED, SHARD_REDUNDANT);
+        SHARD_REDUNDANT.merge.put(LIVE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE); // TODO (expected): should this be an invalid combination?
+        SHARD_REDUNDANT.merge.put(PARTIALLY_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+        SHARD_REDUNDANT.merge.put(PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+        SHARD_REDUNDANT.merge.put(REDUNDANT_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+        SHARD_REDUNDANT.merge.put(LOCALLY_REDUNDANT, LOCALLY_REDUNDANT);
+        SHARD_REDUNDANT.merge.put(SHARD_REDUNDANT, SHARD_REDUNDANT);
     }
 
     public RedundantStatus merge(RedundantStatus that)
diff --git a/accord-core/src/main/java/accord/local/SafeCommand.java b/accord-core/src/main/java/accord/local/SafeCommand.java
index a02e294a..69413246 100644
--- a/accord-core/src/main/java/accord/local/SafeCommand.java
+++ b/accord-core/src/main/java/accord/local/SafeCommand.java
@@ -57,8 +57,6 @@ public abstract class SafeCommand
 
     private <C extends Command> C update(C update)
     {
-        // TODO (now): reenable this check
-//        Invariants.checkState(current() == null || !CommandStore.current().isTruncated(current()));
         set(update);
         return update;
     }
diff --git a/accord-core/src/main/java/accord/local/SafeCommandStore.java b/accord-core/src/main/java/accord/local/SafeCommandStore.java
index eb9574e1..c1eb61d7 100644
--- a/accord-core/src/main/java/accord/local/SafeCommandStore.java
+++ b/accord-core/src/main/java/accord/local/SafeCommandStore.java
@@ -33,16 +33,15 @@ import accord.primitives.Ranges;
 import accord.primitives.Seekable;
 import accord.primitives.Seekables;
 import accord.primitives.Timestamp;
-import accord.primitives.Txn.Kind;
+import accord.primitives.Txn.Kind.Kinds;
 import accord.primitives.TxnId;
 import accord.primitives.Unseekables;
 import javax.annotation.Nullable;
 
+import com.google.common.base.Predicates;
+
 import static accord.local.Commands.Cleanup.NO;
-import static accord.local.RedundantStatus.PRE_BOOTSTRAP;
-import static accord.primitives.Txn.Kind.ExclusiveSyncPoint;
-import static accord.primitives.Txn.Kind.Read;
-import static accord.primitives.Txn.Kind.Write;
+import static accord.local.RedundantBefore.PreBootstrapOrStale.FULLY;
 
 /**
  * A CommandStore with exclusive access; a reference to this should not be retained outside of the scope of the method
@@ -52,9 +51,9 @@ import static accord.primitives.Txn.Kind.Write;
  */
 public abstract class SafeCommandStore
 {
-    public interface CommandFunction<I, O>
+    public interface CommandFunction<P1, I, O>
     {
-        O apply(Seekable keyOrRange, TxnId txnId, Timestamp executeAt, Status status, I in);
+        O apply(P1 p1, Seekable keyOrRange, TxnId txnId, Timestamp executeAt, Status status, I in);
     }
 
     public enum TestTimestamp
@@ -65,57 +64,6 @@ public abstract class SafeCommandStore
         EXECUTES_AFTER
     }
     public enum TestDep { WITH, WITHOUT, ANY_DEPS }
-    public enum TestKind implements Predicate<Kind>
-    {
-        Ws, RorWs, WsOrSyncPoint, SyncPoints, Any;
-
-        @Override
-        public boolean test(Kind kind)
-        {
-            switch (this)
-            {
-                default: throw new AssertionError();
-                case Any: return true;
-                case WsOrSyncPoint: return kind == Write || kind == Kind.SyncPoint || kind == ExclusiveSyncPoint;
-                case SyncPoints: return kind == Kind.SyncPoint || kind == ExclusiveSyncPoint;
-                case Ws: return kind == Write;
-                case RorWs: return kind == Read || kind == Write;
-            }
-        }
-
-        public static TestKind conflicts(Kind kind)
-        {
-            switch (kind)
-            {
-                default: throw new AssertionError();
-                case Read:
-                case NoOp:
-                    return Ws;
-                case Write:
-                    return RorWs;
-                case SyncPoint:
-                case ExclusiveSyncPoint:
-                    return Any;
-            }
-        }
-
-        public static TestKind shouldHaveWitnessed(Kind kind)
-        {
-            switch (kind)
-            {
-                default: throw new AssertionError();
-                case Read:
-                    return WsOrSyncPoint;
-                case Write:
-                    return Any;
-                case SyncPoint:
-                case ExclusiveSyncPoint:
-                case NoOp:
-                    return SyncPoints;
-            }
-        }
-
-    }
 
     /**
      * If the transaction exists (with some associated data) in the CommandStore, return it. Otherwise return null.
@@ -212,16 +160,18 @@ public abstract class SafeCommandStore
      * Visits keys first and then ranges, both in ascending order.
      * Within each key or range visits TxnId in ascending order of queried timestamp.
      */
-    public abstract <T> T mapReduceWithTerminate(Seekables<?, ?> keysOrRanges, Ranges slice,
-                       TestKind testKind, TestTimestamp testTimestamp, Timestamp timestamp,
-                       TestDep testDep, @Nullable TxnId depId,
-                       @Nullable Status minStatus, @Nullable Status maxStatus,
-                       CommandFunction<T, T> map, T accumulate, Predicate<T> terminate);
-    public abstract <T> T mapReduce(Seekables<?, ?> keysOrRanges, Ranges slice,
-                    TestKind testKind, TestTimestamp testTimestamp, Timestamp timestamp,
-                    TestDep testDep, @Nullable TxnId depId, @Nullable Status minStatus, @Nullable Status maxStatus,
-                    CommandFunction<T, T> map, T accumulate, T terminalValue);
+    public <P1, T> T mapReduce(Seekables<?, ?> keys, Ranges slice,
+                                        Kinds testKind, TestTimestamp testTimestamp, Timestamp timestamp,
+                                        TestDep testDep, @Nullable TxnId depId, @Nullable Status minStatus, @Nullable Status maxStatus,
+                                        CommandFunction<P1, T, T> map, P1 p1, T initialValue)
+    {
+        return mapReduce(keys, slice, testKind, testTimestamp, timestamp, testDep, depId, minStatus, maxStatus, map, p1, initialValue, Predicates.alwaysFalse());
+    }
 
+    public abstract <P1, T> T mapReduce(Seekables<?, ?> keys, Ranges slice,
+                                        Kinds testKind, TestTimestamp testTimestamp, Timestamp timestamp,
+                                        TestDep testDep, @Nullable TxnId depId, @Nullable Status minStatus, @Nullable Status maxStatus,
+                                        CommandFunction<P1, T, T> map, P1 p1, T initialValue, Predicate<? super T> terminate);
     protected abstract void register(Seekables<?, ?> keysOrRanges, Ranges slice, Command command);
     protected abstract void register(Seekable keyOrRange, Ranges slice, Command command);
 
@@ -247,9 +197,9 @@ public abstract class SafeCommandStore
 
     // if we have to re-bootstrap (due to failed bootstrap or catching up on a range) then we may
     // have dangling redundant commands; these can safely be executed locally because we are a timestamp store
-    final boolean isFullyPreBootstrap(Command command, Participants<?> forKeys)
+    final boolean isFullyPreBootstrapOrStale(Command command, Participants<?> forKeys)
     {
-        return commandStore().redundantBefore().status(command.txnId(), command.executeAtOrTxnId(), forKeys) == PRE_BOOTSTRAP;
+        return commandStore().redundantBefore().preBootstrapOrStale(command.txnId(), command.executeAtOrTxnId(), forKeys) == FULLY;
     }
 
     public void notifyListeners(SafeCommand safeCommand)
diff --git a/accord-core/src/main/java/accord/local/SaveStatus.java b/accord-core/src/main/java/accord/local/SaveStatus.java
index b547fbb0..1043c321 100644
--- a/accord-core/src/main/java/accord/local/SaveStatus.java
+++ b/accord-core/src/main/java/accord/local/SaveStatus.java
@@ -23,7 +23,9 @@ import accord.primitives.Ballot;
 import accord.primitives.Timestamp;
 import accord.primitives.TxnId;
 
+import static accord.local.SaveStatus.LocalExecution.CleaningUp;
 import static accord.local.SaveStatus.LocalExecution.NotReady;
+import static accord.local.Status.Definition.DefinitionErased;
 import static accord.local.Status.Definition.DefinitionKnown;
 import static accord.local.Status.Definition.DefinitionUnknown;
 import static accord.local.Status.Known.Apply;
@@ -32,7 +34,9 @@ import static accord.local.Status.Known.ExecuteAtOnly;
 import static accord.local.Status.Known.Nothing;
 import static accord.local.Status.KnownDeps.*;
 import static accord.local.Status.KnownExecuteAt.*;
-import static accord.local.Status.NotDefined;
+import static accord.local.Status.KnownRoute.Covering;
+import static accord.local.Status.KnownRoute.Full;
+import static accord.local.Status.KnownRoute.Maybe;
 import static accord.local.Status.Outcome.Unknown;
 import static accord.local.Status.Truncated;
 
@@ -49,29 +53,39 @@ public enum SaveStatus
     // TODO (expected): erase Uninitialised in Context once command finishes
     // TODO (expected): we can use Uninitialised in several places to simplify/better guarantee correct behaviour with truncation
     Uninitialised                   (Status.NotDefined),
+    // TODO (expected): reify PreAcceptedNotDefined and NotDefinedWithSomeRoute (latter to semantically represent outcome of InformHome)
     NotDefined                      (Status.NotDefined),
     PreAccepted                     (Status.PreAccepted),
+    // note: AcceptedInvalidate and AcceptedInvalidateWithDefinition clear any proposed Deps.
+    // This means voters recovering an earlier transaction will not consider the record when excluding the possibility of a fast-path commit.
+    // This is safe, because any Accept that may override the AcceptedInvalidate will construct new Deps that must now witness the recovering transaction.
     AcceptedInvalidate              (Status.AcceptedInvalidate),
-    AcceptedInvalidateWithDefinition(Status.AcceptedInvalidate,    DefinitionKnown,   ExecuteAtUnknown,  DepsUnknown,  Unknown),
+    AcceptedInvalidateWithDefinition(Status.AcceptedInvalidate,    Full,     DefinitionKnown,   ExecuteAtUnknown,  DepsUnknown,  Unknown),
     Accepted                        (Status.Accepted),
-    AcceptedWithDefinition          (Status.Accepted,              DefinitionKnown,   ExecuteAtProposed, DepsProposed, Unknown),
-    PreCommitted                    (Status.PreCommitted),
-    PreCommittedWithAcceptedDeps    (Status.PreCommitted,          DefinitionUnknown, ExecuteAtKnown,    DepsProposed, Unknown),
-    PreCommittedWithDefinition      (Status.PreCommitted,          DefinitionKnown,   ExecuteAtKnown,    DepsUnknown,  Unknown),
-    PreCommittedWithDefinitionAndAcceptedDeps(Status.PreCommitted, DefinitionKnown,   ExecuteAtKnown,    DepsProposed, Unknown),
-    Committed                       (Status.Committed),
-    ReadyToExecute                  (Status.ReadyToExecute,                                                                                    LocalExecution.ReadyToExecute),
-    PreApplied                      (Status.PreApplied),
-    Applying                        (Status.PreApplied),
-    Applied                         (Status.Applied,                                                                                           LocalExecution.Applied),
+    AcceptedWithDefinition          (Status.Accepted,              Full,     DefinitionKnown,   ExecuteAtProposed, DepsProposed, Unknown),
+    PreCommitted                    (Status.PreCommitted,                                                                                          LocalExecution.ReadyToExclude),
+    PreCommittedWithAcceptedDeps    (Status.PreCommitted,          Covering, DefinitionUnknown, ExecuteAtKnown,    DepsProposed, Unknown,          LocalExecution.ReadyToExclude),
+    PreCommittedWithDefinition      (Status.PreCommitted,          Full,     DefinitionKnown,   ExecuteAtKnown,    DepsUnknown,  Unknown,          LocalExecution.ReadyToExclude),
+    PreCommittedWithDefinitionAndAcceptedDeps(Status.PreCommitted, Full,     DefinitionKnown,   ExecuteAtKnown,    DepsProposed, Unknown,          LocalExecution.ReadyToExclude),
+    Committed                       (Status.Committed,                                                                                             LocalExecution.WaitingToExecute),
+    ReadyToExecute                  (Status.ReadyToExecute,                                                                                        LocalExecution.ReadyToExecute),
+    PreApplied                      (Status.PreApplied,                                                                                            LocalExecution.WaitingToApply),
+    Applying                        (Status.PreApplied,                                                                                            LocalExecution.Applying),
+    // similar to Truncated, but doesn't imply we have any global knowledge about application
+    Applied                         (Status.Applied,                                                                                               LocalExecution.Applied),
     // TruncatedApplyWithDeps is a state never adopted within a single replica; it is however a useful state we may enter by combining state from multiple replicas
-    TruncatedApplyWithDeps          (Status.Truncated,             DefinitionUnknown, ExecuteAtKnown,    DepsKnown,    Outcome.Apply,          LocalExecution.CleaningUp),
-    TruncatedApplyWithOutcome       (Status.Truncated,             DefinitionUnknown, ExecuteAtKnown,    DepsUnknown,  Outcome.Apply,          LocalExecution.CleaningUp),
-    TruncatedApply                  (Status.Truncated,             DefinitionUnknown, ExecuteAtKnown,    DepsUnknown,  Outcome.WasApply,       LocalExecution.CleaningUp),
-    Erased                          (Status.Truncated,             DefinitionUnknown, ExecuteAtUnknown,  DepsUnknown,  Outcome.Erased,         LocalExecution.CleaningUp),
-    Invalidated                     (Status.Invalidated,                                                                                       LocalExecution.CleaningUp),
+    // TODO (desired): if we migrate away from SaveStatus in CheckStatusOk towards Known we can remove this TruncatedApplyWithDeps
+    TruncatedApplyWithDeps          (Status.Truncated,             Full,    DefinitionErased, ExecuteAtKnown,    DepsKnown,    Outcome.Apply,    CleaningUp),
+    TruncatedApplyWithOutcome       (Status.Truncated,             Full,    DefinitionErased, ExecuteAtKnown,    DepsErased,   Outcome.Apply,    CleaningUp),
+    TruncatedApply                  (Status.Truncated,             Full,    DefinitionErased, ExecuteAtKnown,    DepsErased,   Outcome.WasApply, CleaningUp),
+    // Expunged means the command is either entirely pre-bootstrap or stale and we don't have enough information to move it through the normal redundant->truncation path (i.e. it might be truncated, it might be applied)
+    Erased                          (Status.Truncated,             Maybe,   DefinitionErased, ExecuteAtErased,   DepsErased,   Outcome.Erased,   CleaningUp),
+    Invalidated                     (Status.Invalidated,                                                                                         CleaningUp),
     ;
 
+    /**
+     * Note that this is a LOCAL concept ONLY, and should not be used to infer anything remotely.
+     */
     public enum LocalExecution
     {
         NotReady(Nothing),
@@ -143,14 +157,14 @@ public enum SaveStatus
         this(status, phase, status.minKnown, execution);
     }
 
-    SaveStatus(Status status, Definition definition, KnownExecuteAt executeAt, KnownDeps deps, Outcome outcome)
+    SaveStatus(Status status, KnownRoute route, Definition definition, KnownExecuteAt executeAt, KnownDeps deps, Outcome outcome)
     {
-        this(status, definition, executeAt, deps, outcome, NotReady);
+        this(status, route, definition, executeAt, deps, outcome, NotReady);
     }
 
-    SaveStatus(Status status, Definition definition, KnownExecuteAt executeAt, KnownDeps deps, Outcome outcome, LocalExecution execution)
+    SaveStatus(Status status, KnownRoute route, Definition definition, KnownExecuteAt executeAt, KnownDeps deps, Outcome outcome, LocalExecution execution)
     {
-        this(status, status.phase, new Known(definition, executeAt, deps, outcome), execution);
+        this(status, status.phase, new Known(route, definition, executeAt, deps, outcome), execution);
     }
 
     SaveStatus(Status status, Phase phase, Known known, LocalExecution execution)
@@ -188,30 +202,34 @@ public enum SaveStatus
         }
     }
 
+    // TODO (expected): merge Known only, and ensure 1:1 mapping so can reconstruct composite
     // TODO (expected, testing): exhaustive testing, particularly around PreCommitted
     public static SaveStatus get(Status status, Known known)
     {
+        if (known.isInvalidated())
+            return Invalidated;
+
         switch (status)
         {
             default: throw new AssertionError("Unexpected status: " + status);
-            case NotDefined: return NotDefined;
-            case PreAccepted: return PreAccepted;
+            case NotDefined: return known.executeAt.isDecidedAndKnownToExecute() ? PreCommitted : NotDefined;
+            case PreAccepted: return known.executeAt.isDecidedAndKnownToExecute() ? PreCommittedWithDefinition : PreAccepted;
             case AcceptedInvalidate:
                 // AcceptedInvalidate logically clears any proposed deps and executeAt
-                if (!known.executeAt.hasDecidedExecuteAt())
+                if (!known.executeAt.isDecidedAndKnownToExecute())
                     return known.isDefinitionKnown() ? AcceptedInvalidateWithDefinition : AcceptedInvalidate;
                 // If we know the executeAt decision then we do not clear it, and fall-through to PreCommitted
                 // however, we still clear the deps, as any deps we might have previously seen proposed are now expired
                 // TODO (expected, consider): consider clearing Command.partialDeps in this case also
                 known = known.with(DepsUnknown);
             case Accepted:
-                if (!known.executeAt.hasDecidedExecuteAt())
+                if (!known.executeAt.isDecidedAndKnownToExecute())
                     return known.isDefinitionKnown() ? AcceptedWithDefinition : Accepted;
                 // if the decision is known, we're really PreCommitted
             case PreCommitted:
                 if (known.isDefinitionKnown())
-                    return known.deps.hasProposedOrDecidedDeps() ? PreCommittedWithDefinitionAndAcceptedDeps : PreCommittedWithDefinition;
-                return known.deps.hasProposedOrDecidedDeps() ? PreCommittedWithAcceptedDeps : PreCommitted;
+                    return known.deps == DepsProposed ? PreCommittedWithDefinitionAndAcceptedDeps : PreCommittedWithDefinition;
+                return known.deps == DepsProposed ? PreCommittedWithAcceptedDeps : PreCommitted;
             case Committed: return Committed;
             case ReadyToExecute: return ReadyToExecute;
             case PreApplied: return PreApplied;
@@ -225,12 +243,14 @@ public enum SaveStatus
         switch (status.status)
         {
             // most statuses already know everything they can
+            case NotDefined:
+            case PreAccepted:
             case Accepted:
             case AcceptedInvalidate:
             case PreCommitted:
                 if (known.isSatisfiedBy(status.known))
                     return status;
-                return get(status.status, status.known.merge(known));
+                return get(status.status, status.known.atLeast(known));
 
             case Truncated:
                 switch (status)
@@ -259,6 +279,7 @@ public enum SaveStatus
         return status;
     }
 
+    // TODO (expected): tighten up distinction of "preferKnowledge" and its interaction with CheckStatus
     public static SaveStatus merge(SaveStatus a, Ballot acceptedA, SaveStatus b, Ballot acceptedB, boolean preferKnowledge)
     {
         // we first enrich cleanups with the knowledge of the other, to avoid counter-intuitive situations where
diff --git a/accord-core/src/main/java/accord/local/SerializerSupport.java b/accord-core/src/main/java/accord/local/SerializerSupport.java
index abdd7f19..6816d4e2 100644
--- a/accord-core/src/main/java/accord/local/SerializerSupport.java
+++ b/accord-core/src/main/java/accord/local/SerializerSupport.java
@@ -133,7 +133,7 @@ public class SerializerSupport
         {
             Propagate propagate = messageProvider.propagateCommit();
             txn = propagate.partialTxn;
-            deps = propagate.partialDeps;
+            deps = propagate.committedDeps;
         }
         else
         {
@@ -175,7 +175,7 @@ public class SerializerSupport
         {
             Propagate propagate = messageProvider.propagateApply();
             txn = propagate.partialTxn;
-            deps = propagate.partialDeps;
+            deps = propagate.committedDeps;
             writes = propagate.writes;
             result = propagate.result;
         }
@@ -221,7 +221,7 @@ public class SerializerSupport
             {
                 Propagate propagateCommit = messageProvider.propagateCommit();
                 txn = propagateCommit.partialTxn;
-                deps = propagateCommit.partialDeps;
+                deps = propagateCommit.committedDeps;
             }
             else if (witnessed.contains(COMMIT_MINIMAL_REQ))
             {
@@ -340,7 +340,7 @@ public class SerializerSupport
                 else if (witnessed.contains(PROPAGATE_COMMIT_MSG))
                 {
                     Propagate propagate = messageProvider.propagateCommit();
-                    return new TxnAndDeps(propagate.partialTxn, propagate.partialDeps);
+                    return new TxnAndDeps(propagate.partialTxn, propagate.committedDeps);
                 }
                 else
                 {
@@ -359,7 +359,7 @@ public class SerializerSupport
                 else if (witnessed.contains(PROPAGATE_APPLY_MSG))
                 {
                     Propagate propagate = messageProvider.propagateApply();
-                    return new TxnAndDeps(propagate.partialTxn, propagate.partialDeps);
+                    return new TxnAndDeps(propagate.partialTxn, propagate.committedDeps);
                 }
                 else if (witnessed.contains(COMMIT_MAXIMAL_REQ))
                 {
@@ -369,7 +369,7 @@ public class SerializerSupport
                 else if (witnessed.contains(PROPAGATE_COMMIT_MSG))
                 {
                     Propagate propagate = messageProvider.propagateCommit();
-                    return new TxnAndDeps(propagate.partialTxn, propagate.partialDeps);
+                    return new TxnAndDeps(propagate.partialTxn, propagate.committedDeps);
                 }
                 else if (witnessed.contains(COMMIT_MINIMAL_REQ))
                 {
diff --git a/accord-core/src/main/java/accord/local/Status.java b/accord-core/src/main/java/accord/local/Status.java
index bf0f3bef..1f67b4b4 100644
--- a/accord-core/src/main/java/accord/local/Status.java
+++ b/accord-core/src/main/java/accord/local/Status.java
@@ -30,21 +30,25 @@ import java.util.function.Predicate;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
 import static accord.local.Status.Definition.*;
 import static accord.local.Status.Known.*;
 import static accord.local.Status.KnownDeps.*;
 import static accord.local.Status.KnownExecuteAt.*;
+import static accord.local.Status.KnownRoute.Covering;
+import static accord.local.Status.KnownRoute.Full;
+import static accord.local.Status.KnownRoute.Maybe;
 import static accord.local.Status.Outcome.*;
 import static accord.local.Status.Phase.*;
 
 public enum Status
 {
     NotDefined        (None,      Nothing),
-    PreAccepted       (PreAccept, DefinitionOnly),
-    AcceptedInvalidate(Accept,    DefinitionUnknown, ExecuteAtUnknown, DepsUnknown, Unknown), // may or may not have witnessed
-    Accepted          (Accept,    DefinitionUnknown, ExecuteAtProposed, DepsProposed, Unknown), // may or may not have witnessed
+    PreAccepted       (PreAccept, DefinitionAndRoute),
+    AcceptedInvalidate(Accept,    Maybe,          DefinitionUnknown, ExecuteAtUnknown,  DepsUnknown,  Unknown), // may or may not have witnessed
+    Accepted          (Accept,    Covering,       DefinitionUnknown, ExecuteAtProposed, DepsProposed, Unknown), // may or may not have witnessed
 
     /**
      * PreCommitted is a peculiar state, half-way between Accepted and Committed.
@@ -71,16 +75,16 @@ public enum Status
      * To solve this problem we simply permit the executeAt we discover for B to be propagated to A* without
      * its dependencies. Though this does complicate the state machine a little.
      */
-    PreCommitted      (Accept,  DefinitionUnknown, ExecuteAtKnown,   DepsUnknown, Unknown),
+    PreCommitted      (Accept,  Maybe, DefinitionUnknown, ExecuteAtKnown,   DepsUnknown, Unknown),
 
-    Committed         (Commit,  DefinitionKnown,   ExecuteAtKnown,   DepsKnown,   Unknown),
+    Committed         (Commit,  Full,  DefinitionKnown,   ExecuteAtKnown,   DepsKnown,   Unknown),
     // TODO (expected): do we need ReadyToExecute here, or can we keep it to SaveStatus only?
-    ReadyToExecute    (Commit,  DefinitionKnown,   ExecuteAtKnown,   DepsKnown,   Unknown),
+    ReadyToExecute    (Commit,  Full,  DefinitionKnown,   ExecuteAtKnown,   DepsKnown,   Unknown),
     // TODO (expected): do we need both PreApplied and Applied here, or can we keep them to SaveStatus only?
-    PreApplied        (Persist, DefinitionKnown,   ExecuteAtKnown,   DepsKnown,   Outcome.Apply),
-    Applied           (Persist, DefinitionKnown,   ExecuteAtKnown,   DepsKnown,   Outcome.Apply),
-    Truncated         (Cleanup, DefinitionUnknown, ExecuteAtUnknown, DepsUnknown, Unknown),
-    Invalidated       (Persist, NoOp,              NoExecuteAt,      NoDeps,      Outcome.Invalidated),
+    PreApplied        (Persist, Full,  DefinitionKnown,   ExecuteAtKnown,   DepsKnown,   Outcome.Apply),
+    Applied           (Persist, Full,  DefinitionKnown,   ExecuteAtKnown,   DepsKnown,   Outcome.Apply),
+    Truncated         (Cleanup, Maybe, DefinitionErased,  ExecuteAtErased,  DepsErased,  Outcome.Erased),
+    Invalidated       (Persist, Maybe, NoOp,              NoExecuteAt,      NoDeps,      Outcome.Invalidated),
     ;
 
     /**
@@ -101,40 +105,89 @@ public enum Status
      * A vector of various facets of knowledge about, or required for, processing a transaction.
      * Each property is treated independently, there is no precedence relationship between them.
      * Each property's values are however ordered with respect to each other.
+     *
+     * This information does not need to be consistent with
      */
     public static class Known
     {
-        public static final Known Nothing           = new Known(DefinitionUnknown, ExecuteAtUnknown, DepsUnknown, Unknown);
-        public static final Known DefinitionOnly    = new Known(DefinitionKnown,   ExecuteAtUnknown, DepsUnknown, Unknown);
-        public static final Known ExecuteAtOnly     = new Known(DefinitionUnknown, ExecuteAtKnown,   DepsUnknown, Unknown);
-        public static final Known Decision          = new Known(DefinitionKnown,   ExecuteAtKnown,   DepsKnown,   Unknown);
-        public static final Known Apply             = new Known(DefinitionUnknown, ExecuteAtKnown,   DepsKnown,   Outcome.Apply);
-        public static final Known Invalidated       = new Known(DefinitionUnknown, ExecuteAtUnknown, DepsUnknown, Outcome.Invalidated);
-
+        public static final Known Nothing            = new Known(Maybe, DefinitionUnknown, ExecuteAtUnknown, DepsUnknown, Unknown);
+        // TODO (expected): deprecate DefinitionOnly
+        public static final Known DefinitionOnly     = new Known(Maybe, DefinitionKnown,   ExecuteAtUnknown, DepsUnknown, Unknown);
+        public static final Known DefinitionAndRoute = new Known(Full,  DefinitionKnown,   ExecuteAtUnknown, DepsUnknown, Unknown);
+        public static final Known ExecuteAtOnly      = new Known(Maybe, DefinitionUnknown, ExecuteAtKnown,   DepsUnknown, Unknown);
+        public static final Known Decision           = new Known(Full,  DefinitionKnown,   ExecuteAtKnown,   DepsKnown,   Unknown);
+        public static final Known Apply              = new Known(Full,  DefinitionUnknown, ExecuteAtKnown,   DepsKnown,   Outcome.Apply);
+        public static final Known Invalidated        = new Known(Maybe, DefinitionUnknown, ExecuteAtUnknown, DepsUnknown, Outcome.Invalidated);
+
+        public final KnownRoute route;
         public final Definition definition;
         public final KnownExecuteAt executeAt;
         public final KnownDeps deps;
         public final Outcome outcome;
 
-        public Known(Definition definition, KnownExecuteAt executeAt, KnownDeps deps, Outcome outcome)
+        public Known(Known copy)
         {
+            this.route = copy.route;
+            this.definition = copy.definition;
+            this.executeAt = copy.executeAt;
+            this.deps = copy.deps;
+            this.outcome = copy.outcome;
+        }
+
+        public Known(KnownRoute route, Definition definition, KnownExecuteAt executeAt, KnownDeps deps, Outcome outcome)
+        {
+            this.route = route;
             this.definition = definition;
             this.executeAt = executeAt;
             this.deps = deps;
             this.outcome = outcome;
         }
 
-        public Known merge(Known with)
+        public Known atLeast(Known that)
         {
-            Definition maxDefinition = definition.compareTo(with.definition) >= 0 ? definition : with.definition;
-            KnownExecuteAt maxExecuteAt = executeAt.compareTo(with.executeAt) >= 0 ? executeAt : with.executeAt;
-            KnownDeps maxDeps = deps.compareTo(with.deps) >= 0 ? deps : with.deps;
-            Outcome maxOutcome = outcome.compareTo(with.outcome) >= 0 ? outcome : with.outcome;
-            if (maxDefinition == definition && maxExecuteAt == executeAt && maxDeps == deps &&  maxOutcome == outcome)
-                return this;
-            if (maxDefinition == with.definition && maxExecuteAt == with.executeAt && maxDeps == with.deps && maxOutcome == with.outcome)
+            // TODO (expected): validate no inconsistent combos
+            KnownRoute maxRoute = route.atLeast(that.route);
+            Definition maxDefinition = definition.atLeast(that.definition);
+            KnownExecuteAt maxExecuteAt = executeAt.atLeast(that.executeAt);
+            KnownDeps maxDeps = deps.atLeast(that.deps);
+            Outcome maxOutcome = outcome.atLeast(that.outcome);
+            return selectOrCreate(that, maxRoute, maxDefinition, maxExecuteAt, maxDeps, maxOutcome);
+        }
+
+        public Known reduce(Known that)
+        {
+            KnownRoute maxRoute = route.reduce(that.route);
+            Definition minDefinition = definition.reduce(that.definition);
+            KnownExecuteAt maxExecuteAt = executeAt.reduce(that.executeAt);
+            KnownDeps minDeps = deps.reduce(that.deps);
+            Outcome maxOutcome = outcome.reduce(that.outcome);
+            return selectOrCreate(that, maxRoute, minDefinition, maxExecuteAt, minDeps, maxOutcome);
+        }
+
+        public Known validForAll()
+        {
+            KnownRoute maxRoute = route.validForAll();
+            Definition minDefinition = definition.validForAll();
+            KnownExecuteAt maxExecuteAt = executeAt.validForAll();
+            KnownDeps minDeps = deps.validForAll();
+            Outcome maxOutcome = outcome.validForAll();
+            return selectOrCreate(maxRoute, minDefinition, maxExecuteAt, minDeps, maxOutcome);
+        }
+
+        @Nonnull
+        private Known selectOrCreate(Known with, KnownRoute maxRoute, Definition maxDefinition, KnownExecuteAt maxExecuteAt, KnownDeps maxDeps, Outcome maxOutcome)
+        {
+            if (maxRoute == with.route && maxDefinition == with.definition && maxExecuteAt == with.executeAt && maxDeps == with.deps && maxOutcome == with.outcome)
                 return with;
-            return new Known(maxDefinition, maxExecuteAt, maxDeps, maxOutcome);
+            return selectOrCreate(maxRoute, maxDefinition, maxExecuteAt, maxDeps, maxOutcome);
+        }
+
+        @Nonnull
+        private Known selectOrCreate(KnownRoute maxRoute, Definition maxDefinition, KnownExecuteAt maxExecuteAt, KnownDeps maxDeps, Outcome maxOutcome)
+        {
+            if (maxRoute == route && maxDefinition == definition && maxExecuteAt == executeAt && maxDeps == deps &&  maxOutcome == outcome)
+                return this;
+            return new Known(maxRoute, maxDefinition, maxExecuteAt, maxDeps, maxOutcome);
         }
 
         public boolean isSatisfiedBy(Known that)
@@ -172,17 +225,52 @@ public enum Status
         {
             if (outcome == newOutcome)
                 return this;
-            return new Known(definition, executeAt, deps, newOutcome);
+            return new Known(route, definition, executeAt, deps, newOutcome);
         }
 
         public Known with(KnownDeps newDeps)
         {
             if (deps == newDeps)
                 return this;
-            return new Known(definition, executeAt, newDeps, outcome);
+            return new Known(route, definition, executeAt, newDeps, outcome);
+        }
+
+        /**
+         * Convert this Known to one that represents the knowledge that can be propagated to another replica without
+         * further information available to that replica, e.g. we cannot apply without also knowing the definition.
+         */
+        // TODO (expected): merge propagates and propagatesStatus
+        public Known propagates()
+        {
+            if (isInvalidated())
+                return Invalidated;
+
+            if (definition == DefinitionUnknown)
+                return executeAt.isDecidedAndKnownToExecute() ? ExecuteAtOnly : Nothing;
+
+            KnownExecuteAt executeAt = this.executeAt;
+            if (!executeAt.isDecidedAndKnownToExecute())
+                return DefinitionOnly;
+
+            // cannot propagate proposed deps; and cannot propagate known deps without executeAt
+            KnownDeps deps = this.deps;
+            if (!deps.hasDecidedDeps())
+                return SaveStatus.PreCommittedWithDefinition.known;
+
+            switch (outcome)
+            {
+                default: throw new AssertionError("Unhandled outcome: " + outcome);
+                case Unknown:
+                case WasApply:
+                case Erased:
+                    return Committed.minKnown;
+
+                case Apply:
+                    return PreApplied.minKnown;
+            }
         }
 
-        public Status propagate()
+        public Status propagatesStatus()
         {
             switch (outcome)
             {
@@ -195,14 +283,14 @@ public enum Status
 
                 case Apply:
                 case WasApply:
-                    if (executeAt.hasDecidedExecuteAt() && definition.isKnown() && deps.hasDecidedDeps())
+                    if (executeAt.isDecidedAndKnownToExecute() && definition.isKnown() && deps.hasDecidedDeps())
                         return PreApplied;
 
                 case Unknown:
-                    if (executeAt.hasDecidedExecuteAt() && definition.isKnown() && deps.hasDecidedDeps())
+                    if (executeAt.isDecidedAndKnownToExecute() && definition.isKnown() && deps.hasDecidedDeps())
                         return Committed;
 
-                    if (executeAt.hasDecidedExecuteAt())
+                    if (executeAt.isDecidedAndKnownToExecute())
                         return PreCommitted;
 
                     if (definition.isKnown())
@@ -212,7 +300,7 @@ public enum Status
                         throw new IllegalStateException();
             }
 
-            return NotDefined;
+            return outcome == WasApply ? Truncated : NotDefined;
         }
 
         public boolean isDefinitionKnown()
@@ -220,6 +308,11 @@ public enum Status
             return definition.isKnown();
         }
 
+        public boolean hasDefinitionBeenKnown()
+        {
+            return definition.isKnown() || outcome.isDecided();
+        }
+
         /**
          * The command may have an incomplete route when this is false
          */
@@ -228,25 +321,141 @@ public enum Status
             return definition.isKnown() || outcome.isOrWasApply();
         }
 
+        public boolean isTruncated()
+        {
+            switch (outcome)
+            {
+                default: throw new AssertionError("Unhandled outcome: " + outcome);
+                case Invalidated:
+                case Unknown:
+                    return false;
+                case Apply:
+                    // since Apply is universal, we can
+                    return deps == DepsErased;
+                case Erased:
+                case WasApply:
+                    return true;
+            }
+        }
+
         public boolean canProposeInvalidation()
         {
             return deps.canProposeInvalidation() && executeAt.canProposeInvalidation() && outcome.canProposeInvalidation();
         }
 
+        public Known subtract(Known subtract)
+        {
+            if (!subtract.isSatisfiedBy(this))
+                return Known.Nothing;
+
+            Definition newDefinition = subtract.definition.compareTo(definition) >= 0 ? DefinitionUnknown : definition;
+            KnownExecuteAt newExecuteAt = subtract.executeAt.compareTo(executeAt) >= 0 ? ExecuteAtUnknown : executeAt;
+            KnownDeps newDeps = subtract.deps.compareTo(deps) >= 0 ? DepsUnknown : deps;
+            Outcome newOutcome = outcome.subtract(subtract.outcome);
+            return new Known(route, newDefinition, newExecuteAt, newDeps, newOutcome);
+        }
+
+        public boolean isDecided()
+        {
+            return executeAt.isDecided() || outcome.isDecided();
+        }
+
+        public boolean isDecidedToExecute()
+        {
+            return executeAt.isDecidedAndKnownToExecute() || outcome.isOrWasApply();
+        }
+
         public String toString()
         {
             return Stream.of(definition.isKnown() ? "Definition" : null,
-                             executeAt.hasDecidedExecuteAt() ? "ExecuteAt" : null,
+                             executeAt.isDecidedAndKnownToExecute() ? "ExecuteAt" : null,
                              deps.hasDecidedDeps() ? "Deps" : null,
-                             outcome.isOrWasApply() ? "Outcome" : null
+                             outcome.isDecided() ? outcome.toString() : null
             ).filter(Objects::nonNull).collect(Collectors.joining(",", "[", "]"));
         }
+
+        @Override
+        public boolean equals(Object o)
+        {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Known that = (Known) o;
+            return route == that.route && definition == that.definition && executeAt == that.executeAt && deps == that.deps && outcome == that.outcome;
+        }
+
+        @Override
+        public int hashCode()
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        public boolean hasDefinition()
+        {
+            return definition.isKnown();
+        }
+
+        public boolean hasDecidedDeps()
+        {
+            return deps.hasDecidedDeps();
+        }
+
+        public boolean isInvalidated()
+        {
+            return outcome.isInvalidated();
+        }
+    }
+
+    public enum KnownRoute
+    {
+        /**
+         * A route may or may not be known, but it may not cover (or even intersect) this shard.
+         * The route should be relied upon only if it is a FullRoute.
+         */
+        Maybe,
+
+        /**
+         * A route is known that covers the ranges this shard participates in.
+         * Note that if the status is less than Committed, this may not be the final set of owned ranges,
+         * and the route may not cover whatever this is decided as.
+         *
+         * This status primarily exists to communicate semantically to the reader.
+         */
+        Covering,
+
+        /**
+         * The full route is known. <i>Generally</i> this coincides with knowing the Definition.
+         */
+        Full
+        ;
+
+        public boolean hasFull()
+        {
+            return this == Full;
+        }
+
+        public KnownRoute reduce(KnownRoute that)
+        {
+            if (this == that) return this;
+            if (this == Full || that == Full) return Full;
+            return Maybe;
+        }
+
+        public KnownRoute validForAll()
+        {
+            return this == Covering ? Maybe : this;
+        }
+
+        public KnownRoute atLeast(KnownRoute that)
+        {
+            return this.compareTo(that) >= 0 ? this : that;
+        }
     }
 
     public enum KnownExecuteAt
     {
         /**
-         * No decision is known to have been reached
+         * No decision is known to have been reached. If executeAt is not null, it represents either when
+         * the transaction was witnessed, or some earlier ExecuteAtProposed that was invalidated by AcceptedInvalidate
          */
         ExecuteAtUnknown,
 
@@ -255,6 +464,11 @@ public enum Status
          */
         ExecuteAtProposed,
 
+        /**
+         * A decision to execute or invalidate the transaction is known to have been reached and since been cleaned up
+         */
+        ExecuteAtErased,
+
         /**
          * A decision to execute the transaction is known to have been reached, and the associated executeAt timestamp
          */
@@ -266,19 +480,35 @@ public enum Status
         NoExecuteAt
         ;
 
-        public boolean hasDecidedExecuteAt()
+        /**
+         * Is known to have agreed to execute or not; but the decision is not known (maybe erased)
+         */
+        public boolean isDecided()
         {
-            return compareTo(ExecuteAtKnown) >= 0;
+            return compareTo(ExecuteAtErased) >= 0;
         }
 
-        public boolean hasDecidedNonZeroExecuteAt()
+        /**
+         * Is known to execute, and when.
+         */
+        public boolean isDecidedAndKnownToExecute()
         {
             return this == ExecuteAtKnown;
         }
 
-        public final Timestamp executeAtIfKnownElseTxnId(TxnId txnId, Timestamp executeAt)
+        public KnownExecuteAt atLeast(KnownExecuteAt that)
+        {
+            return compareTo(that) >= 0 ? this : that;
+        }
+
+        public KnownExecuteAt reduce(KnownExecuteAt that)
+        {
+            return atLeast(that);
+        }
+
+        public KnownExecuteAt validForAll()
         {
-            return this == ExecuteAtKnown ? executeAt : txnId;
+            return compareTo(ExecuteAtErased) <= 0 ? ExecuteAtUnknown : this;
         }
 
         public boolean canProposeInvalidation()
@@ -300,6 +530,12 @@ public enum Status
          */
         DepsProposed,
 
+        /**
+         * A decision to execute or invalidate the transaction is known to have been reached, and any associated
+         * dependencies for the shard(s) in question have been cleaned up.
+         */
+        DepsErased,
+
         /**
          * A decision to execute the transaction is known to have been reached, and the associated dependencies
          * for the shard(s) in question are known for the coordination and execution epochs.
@@ -322,11 +558,6 @@ public enum Status
             return this == DepsKnown;
         }
 
-        public boolean isDecided()
-        {
-            return compareTo(DepsKnown) >= 0;
-        }
-
         public boolean canProposeInvalidation()
         {
             return this == DepsUnknown;
@@ -336,6 +567,21 @@ public enum Status
         {
             return this == DepsProposed || this == DepsKnown;
         }
+
+        public KnownDeps atLeast(KnownDeps that)
+        {
+            return compareTo(that) >= 0 ? this : that;
+        }
+
+        public KnownDeps reduce(KnownDeps that)
+        {
+            return compareTo(that) <= 0 ? this : that;
+        }
+
+        public KnownDeps validForAll()
+        {
+            return this == NoDeps ? NoDeps : DepsUnknown;
+        }
     }
 
     /**
@@ -349,26 +595,47 @@ public enum Status
         DefinitionUnknown,
 
         /**
-         * The definition is known
-         *
-         * TODO (expected, clarity): distinguish between known for coordination epoch and known for commit/execute
+         * The definition was known, but has been erased
          */
-        DefinitionKnown,
+        DefinitionErased,
 
         /**
          * The definition is irrelevant, as the transaction has been invalidated and may be treated as a no-op
          */
-        NoOp;
+        NoOp,
 
-        public boolean canProposeInvalidation()
+        /**
+         * The definition is known
+         *
+         * TODO (expected, clarity): distinguish between known for coordination epoch and known for commit/execute
+         */
+        DefinitionKnown;
+
+        public boolean isKnown()
         {
-            return this == DefinitionUnknown;
+            return this == DefinitionKnown;
         }
 
-        public boolean isKnown()
+        public boolean isOrWasKnown()
         {
             return this == DefinitionKnown;
         }
+
+        public Definition atLeast(Definition that)
+        {
+            return compareTo(that) >= 0 ? this : that;
+        }
+
+        // combine info about two shards into a composite representation
+        public Definition reduce(Definition that)
+        {
+            return compareTo(that) <= 0 ? this : that;
+        }
+
+        public Definition validForAll()
+        {
+            return this == NoOp ? NoOp : DefinitionUnknown;
+        }
     }
 
     /**
@@ -382,26 +649,25 @@ public enum Status
         Unknown,
 
         /**
-         * The outcome is known
+         * The transaction has been *completely cleaned up* - this means it has been made
+         * durable at every live replica of every shard we contacted
          */
-        Apply,
+        Erased,
 
         /**
          * The transaction has been cleaned-up, but was applied and the relevant portion of its outcome has been cleaned up
-         * TODO (expected): is this state helpful? Feels like we can do without it
          */
         WasApply,
 
         /**
-         * The transaction is known to have been invalidated
+         * The outcome is known
          */
-        Invalidated,
+        Apply,
 
         /**
-         * The transaction has been *completely cleaned up* - this means it has been made
-         * durable at every live replica of every shard we contacted
+         * The transaction is known to have been invalidated
          */
-        Erased
+        Invalidated
         ;
 
         public boolean isOrWasApply()
@@ -441,9 +707,33 @@ public enum Status
             return this == Invalidated;
         }
 
-        public boolean isTruncated()
+        public Outcome atLeast(Outcome that)
+        {
+            return this.compareTo(that) >= 0 ? this : that;
+        }
+
+        // outcomes are universal - any replica of any shard may propagate its outcome to any other replica of any other shard
+        public Outcome reduce(Outcome that)
         {
-            return this == Erased || this == WasApply;
+            return atLeast(that);
+        }
+
+        /**
+         * Do not imply truncation where none has happened
+         */
+        public Outcome validForAll()
+        {
+            return compareTo(WasApply) <= 0 ? Unknown : this;
+        }
+
+        public Outcome subtract(Outcome that)
+        {
+            return this.compareTo(that) <= 0 ? Unknown : this;
+        }
+
+        public boolean isDecided()
+        {
+            return this != Unknown;
         }
     }
 
@@ -456,22 +746,51 @@ public enum Status
      * Represents the durability of a transaction's Persist phase.
      * NotDurable: the outcome has not been durably recorded
      * Local:      the outcome has been durably recorded at least locally
+     * ShardUniversalOrInvalidated: the outcome has been durably recorded at all healthy replicas of the shard, or is invalidated
+     * ShardUniversal:      the outcome has been durably recorded at all healthy replicas of the shard
+     * MajorityOrInvalidated:   the outcome has been durably recorded to a majority of each participating shard
      * Majority:   the outcome has been durably recorded to a majority of each participating shard
-     * Universal:  the outcome has been durably recorded to every participating replica
-     * DurableOrInvalidated:  the outcome was either invalidated, or has been durably recorded to every replica
+     * Universal:  the outcome has been durably recorded to every healthy replica
+     * DurableOrInvalidated:  the outcome was either invalidated, or has been durably recorded to every healthy replica
      */
     public enum Durability
     {
-        NotDurable, Local, Majority, Universal, DurableOrInvalidated;
+        NotDurable, Local, ShardUniversal,
+        MajorityOrInvalidated, Majority,
+        UniversalOrInvalidated, Universal;
 
         public boolean isDurable()
         {
-            return compareTo(Majority) >= 0 && compareTo(DurableOrInvalidated) < 0;
+            return this == Majority || this == Universal;
         }
 
         public boolean isDurableOrInvalidated()
         {
-            return compareTo(Majority) >= 0;
+            return compareTo(MajorityOrInvalidated) >= 0;
+        }
+
+        public boolean isMaybeInvalidated()
+        {
+            return this == NotDurable || this == MajorityOrInvalidated || this == UniversalOrInvalidated;
+        }
+
+        public static Durability merge(Durability a, Durability b)
+        {
+            int c = a.compareTo(b);
+            if (c < 0) { Durability tmp = a; a = b; b = tmp; }
+            // if we know we are applied, we can remove the OrInvalidated qualifier
+            if (a == UniversalOrInvalidated && (b == Majority || b == ShardUniversal || b == Local)) a = Universal;
+            if ((a == ShardUniversal) && (b == Local || b == NotDurable)) a = Local;
+            if (b == NotDurable && a.compareTo(MajorityOrInvalidated) < 0) a = NotDurable;
+            return a;
+        }
+
+        public static Durability mergeAtLeast(Durability a, Durability b)
+        {
+            int c = a.compareTo(b);
+            if (c < 0) { Durability tmp = a; a = b; b = tmp; }
+            if (a == UniversalOrInvalidated && (b == Majority || b == ShardUniversal || b == Local)) a = Universal;
+            return a;
         }
     }
 
@@ -484,10 +803,10 @@ public enum Status
         this.minKnown = minKnown;
     }
 
-    Status(Phase phase, Definition definition, KnownExecuteAt executeAt, KnownDeps deps, Status.Outcome outcome)
+    Status(Phase phase, KnownRoute route, Definition definition, KnownExecuteAt executeAt, KnownDeps deps, Status.Outcome outcome)
     {
         this.phase = phase;
-        this.minKnown = new Known(definition, executeAt, deps, outcome);
+        this.minKnown = new Known(route, definition, executeAt, deps, outcome);
     }
 
     // TODO (desired, clarity): investigate all uses of hasBeen, and migrate as many as possible to testing
@@ -512,7 +831,8 @@ public enum Status
             Ballot accepted = getAccepted.apply(item);
             boolean update = max == null
                           || maxStatus.phase.compareTo(status.phase) < 0
-                          || (status.phase.equals(Phase.Accept) && maxAccepted.compareTo(accepted) < 0);
+                          || (status.phase == Accept && maxAccepted.compareTo(accepted) < 0)
+                          || status.phase != Accept && maxStatus.phase == status.phase && maxStatus.compareTo(status) > 0;
 
             if (!update)
                 continue;
@@ -534,9 +854,4 @@ public enum Status
             return a;
         return b;
     }
-
-    public static Status simpleMax(Status a, Status b)
-    {
-        return a.compareTo(b) >= 0 ? a : b;
-    }
 }
diff --git a/accord-core/src/main/java/accord/messages/AbstractExecute.java b/accord-core/src/main/java/accord/messages/AbstractExecute.java
index 804c101a..6784f88a 100644
--- a/accord-core/src/main/java/accord/messages/AbstractExecute.java
+++ b/accord-core/src/main/java/accord/messages/AbstractExecute.java
@@ -38,6 +38,7 @@ import accord.primitives.Timestamp;
 import accord.primitives.TxnId;
 import accord.topology.Topologies;
 
+import static accord.local.SaveStatus.LocalExecution.ReadyToExecute;
 import static accord.local.SaveStatus.LocalExecution.WaitingToExecute;
 import static accord.local.Status.Committed;
 import static accord.messages.ReadData.ReadNack.NotCommitted;
@@ -210,7 +211,11 @@ public abstract class AbstractExecute extends ReadData implements Command.Transi
 
                 safeStore.progressLog().waiting(safeCommand, WaitingToExecute, null, readScope);
                 if (status == Committed) return null;
-                else return NotCommitted;
+                else
+                {
+                    safeStore.progressLog().waiting(safeCommand, ReadyToExecute, null, readScope);
+                    return NotCommitted;
+                }
             case OBSOLETE:
                 state = State.OBSOLETE;
                 return Redundant;
@@ -268,6 +273,8 @@ public abstract class AbstractExecute extends ReadData implements Command.Transi
                 throw new IllegalStateException("ReadOk was sent, yet ack called again");
             case OBSOLETE:
                 logger.debug("After the read completed for txn {}, the result was marked obsolete", txnId);
+                if (fail != null)
+                    node.agent().onUncaughtException(fail);
                 break;
             case PENDING:
                 state = State.RETURNED;
@@ -283,18 +290,18 @@ public abstract class AbstractExecute extends ReadData implements Command.Transi
         return new ReadOk(unavailable, data);
     }
 
-    private void removeListener(SafeCommandStore safeStore, TxnId txnId)
-    {
-        SafeCommand safeCommand = safeStore.ifInitialised(txnId);
-        safeCommand.removeListener(this);
-    }
-
     @Override
     protected void cancel()
     {
         node.commandStores().mapReduceConsume(this, waitingOn.stream(), forEach(in -> removeListener(in, txnId), node.agent()));
     }
 
+    private void removeListener(SafeCommandStore safeStore, TxnId txnId)
+    {
+        SafeCommand safeCommand = safeStore.ifInitialised(txnId);
+        safeCommand.removeListener(this);
+    }
+
     @Override
     public String toString()
     {
diff --git a/accord-core/src/main/java/accord/messages/Accept.java b/accord-core/src/main/java/accord/messages/Accept.java
index 44abe4fd..5e069ab5 100644
--- a/accord-core/src/main/java/accord/messages/Accept.java
+++ b/accord-core/src/main/java/accord/messages/Accept.java
@@ -29,6 +29,7 @@ import accord.local.SafeCommand;
 import accord.local.SafeCommandStore;
 import accord.primitives.Ballot;
 import accord.primitives.Deps;
+import accord.primitives.EpochSupplier;
 import accord.primitives.FullRoute;
 import accord.primitives.PartialDeps;
 import accord.primitives.PartialRoute;
@@ -109,7 +110,8 @@ public class Accept extends TxnRequest.WithUnsynced<Accept.AcceptReply>
 
     private PartialDeps calculatePartialDeps(SafeCommandStore safeStore)
     {
-        return PreAccept.calculatePartialDeps(safeStore, txnId, keys, executeAt, safeStore.ranges().allBetween(minUnsyncedEpoch, executeAt));
+        Ranges ranges = safeStore.ranges().allBetween(minUnsyncedEpoch, txnId);
+        return PreAccept.calculatePartialDeps(safeStore, txnId, keys, EpochSupplier.constant(minUnsyncedEpoch), executeAt, ranges);
     }
 
     @Override
diff --git a/accord-core/src/main/java/accord/messages/Apply.java b/accord-core/src/main/java/accord/messages/Apply.java
index a2e874be..02983598 100644
--- a/accord-core/src/main/java/accord/messages/Apply.java
+++ b/accord-core/src/main/java/accord/messages/Apply.java
@@ -42,6 +42,7 @@ import accord.primitives.TxnId;
 import accord.primitives.Unseekables;
 import accord.primitives.Writes;
 import accord.topology.Topologies;
+import accord.utils.Invariants;
 
 public class Apply extends TxnRequest<ApplyReply>
 {
@@ -72,6 +73,7 @@ public class Apply extends TxnRequest<ApplyReply>
     protected Apply(Kind kind, Id to, Topologies participates, Topologies executes, TxnId txnId, Route<?> route, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result)
     {
         super(to, participates, route, txnId);
+        Invariants.checkState(txnId.rw() != Txn.Kind.Write || writes != null);
         Ranges slice = kind == Kind.Maximal || executes == participates ? scope.covering() : executes.computeRangesForNode(to);
         // TODO (desired): it's wasteful to encode the full set of ranges owned by the recipient node;
         //     often it will be cheaper to include the FullRoute for Deps scope (or come up with some other safety-preserving encoding scheme)
diff --git a/accord-core/src/main/java/accord/messages/ApplyThenWaitUntilApplied.java b/accord-core/src/main/java/accord/messages/ApplyThenWaitUntilApplied.java
index 02b4fc19..e590f7e5 100644
--- a/accord-core/src/main/java/accord/messages/ApplyThenWaitUntilApplied.java
+++ b/accord-core/src/main/java/accord/messages/ApplyThenWaitUntilApplied.java
@@ -114,6 +114,12 @@ public class ApplyThenWaitUntilApplied extends WaitUntilApplied
             node.agent().onLocalBarrier(partialTxnKeys, txnId);
     }
 
+    @Override
+    public MessageType type()
+    {
+        return MessageType.APPLY_THEN_WAIT_UNTIL_APPLIED_REQ;
+    }
+
     @Override
     public String toString()
     {
diff --git a/accord-core/src/main/java/accord/messages/BeginInvalidation.java b/accord-core/src/main/java/accord/messages/BeginInvalidation.java
index 5a3b0dbe..790a4115 100644
--- a/accord-core/src/main/java/accord/messages/BeginInvalidation.java
+++ b/accord-core/src/main/java/accord/messages/BeginInvalidation.java
@@ -187,11 +187,6 @@ public class BeginInvalidation extends AbstractEpochRequest<BeginInvalidation.In
             return Status.max(invalidateReplies, r -> r.status, r -> r.accepted, invalidateReply -> true);
         }
 
-        public static InvalidateReply maxNotTruncated(List<InvalidateReply> invalidateReplies)
-        {
-            return Status.max(invalidateReplies, r -> r.status, r -> r.accepted, invalidateReply -> true);
-        }
-
         public static RoutingKey findHomeKey(List<InvalidateReply> invalidateOks)
         {
             for (InvalidateReply ok : invalidateOks)
diff --git a/accord-core/src/main/java/accord/messages/BeginRecovery.java b/accord-core/src/main/java/accord/messages/BeginRecovery.java
index e23a764c..56adf49a 100644
--- a/accord-core/src/main/java/accord/messages/BeginRecovery.java
+++ b/accord-core/src/main/java/accord/messages/BeginRecovery.java
@@ -46,7 +46,6 @@ import accord.utils.Invariants;
 
 import static accord.local.SafeCommandStore.TestDep.WITH;
 import static accord.local.SafeCommandStore.TestDep.WITHOUT;
-import static accord.local.SafeCommandStore.TestKind.shouldHaveWitnessed;
 import static accord.local.SafeCommandStore.TestTimestamp.EXECUTES_AFTER;
 import static accord.local.SafeCommandStore.TestTimestamp.STARTED_AFTER;
 import static accord.local.SafeCommandStore.TestTimestamp.STARTED_BEFORE;
@@ -118,7 +117,8 @@ public class BeginRecovery extends TxnRequest<BeginRecovery.RecoverReply>
         PartialDeps deps = command.partialDeps();
         if (!command.known().deps.hasProposedOrDecidedDeps())
         {
-            deps = calculatePartialDeps(safeStore, txnId, partialTxn.keys(), txnId, safeStore.ranges().coordinates(txnId));
+            // TODO (required): consider whether we are safe ignoring the concept of minUnsyncedEpoch here
+            deps = calculatePartialDeps(safeStore, txnId, partialTxn.keys(), txnId, txnId, safeStore.ranges().coordinates(txnId));
         }
 
         boolean rejectsFastPath;
@@ -335,32 +335,32 @@ public class BeginRecovery extends TxnRequest<BeginRecovery.RecoverReply>
         }
     }
 
-    private static Deps acceptedStartedBeforeWithoutWitnessing(SafeCommandStore commandStore, TxnId startedBefore, Ranges ranges, Seekables<?, ?> keys)
+    private static Deps acceptedStartedBeforeWithoutWitnessing(SafeCommandStore safeStore, TxnId startedBefore, Ranges ranges, Seekables<?, ?> keys)
     {
         try (Deps.Builder builder = Deps.builder())
         {
             // any transaction that started
-            commandStore.mapReduce(keys, ranges, shouldHaveWitnessed(startedBefore.rw()), STARTED_BEFORE, startedBefore, WITHOUT, startedBefore, Accepted, PreCommitted,
-                    (keyOrRange, txnId, executeAt, saveStatus, prev) -> {
-                        if (executeAt.compareTo(startedBefore) > 0)
-                            builder.add(keyOrRange, txnId);
-                        return builder;
-                    }, builder, null);
+            safeStore.mapReduce(keys, ranges, startedBefore.rw().witnesses(), STARTED_BEFORE, startedBefore, WITHOUT, startedBefore, Accepted, PreCommitted,
+                    (startedBefore0, keyOrRange, txnId, executeAt, status, prev) -> {
+                        if (executeAt.compareTo(startedBefore0) > 0)
+                            prev.add(keyOrRange, txnId);
+                        return prev;
+                    }, startedBefore, builder);
             return builder.build();
         }
     }
 
-    private static Deps committedStartedBeforeAndWitnessed(SafeCommandStore commandStore, TxnId startedBefore, Ranges ranges, Seekables<?, ?> keys)
+    private static Deps committedStartedBeforeAndWitnessed(SafeCommandStore safeStore, TxnId startedBefore, Ranges ranges, Seekables<?, ?> keys)
     {
         try (Deps.Builder builder = Deps.builder())
         {
-            commandStore.mapReduce(keys, ranges, shouldHaveWitnessed(startedBefore.rw()), STARTED_BEFORE, startedBefore, WITH, startedBefore, Committed, null,
-                    (keyOrRange, txnId, executeAt, saveStatus, prev) -> builder.add(keyOrRange, txnId), builder, (Deps.AbstractBuilder<Deps>)null);
+            safeStore.mapReduce(keys, ranges, startedBefore.rw().witnesses(), STARTED_BEFORE, startedBefore, WITH, startedBefore, Committed, null,
+                    (p1, keyOrRange, txnId, executeAt, status, prev) -> prev.add(keyOrRange, txnId), null, (Deps.AbstractBuilder<Deps>)builder);
             return builder.build();
         }
     }
 
-    private static boolean hasAcceptedStartedAfterWithoutWitnessing(SafeCommandStore commandStore, TxnId startedAfter, Ranges ranges, Seekables<?, ?> keys)
+    private static boolean hasAcceptedStartedAfterWithoutWitnessing(SafeCommandStore safeStore, TxnId startedAfter, Ranges ranges, Seekables<?, ?> keys)
     {
         /*
          * The idea here is to discover those transactions that were started after us and have been Accepted
@@ -368,12 +368,14 @@ public class BeginRecovery extends TxnRequest<BeginRecovery.RecoverReply>
          * the fast path. This is central to safe recovery, as if every transaction that executes later has
          * witnessed us we are safe to propose the pre-accept timestamp regardless, whereas if any transaction
          * has not witnessed us we can safely invalidate (us).
+         *
+         * TODO (required): consider carefully how _adding_ ranges to a CommandStore affects this
          */
-        return commandStore.mapReduce(keys, ranges, shouldHaveWitnessed(startedAfter.rw()), STARTED_AFTER, startedAfter, WITHOUT, startedAfter, Accepted, PreCommitted,
-                (keyOrRange, txnId, executeAt, saveStatus, prev) -> true, false, true);
+        return safeStore.mapReduce(keys, ranges, startedAfter.rw().witnesses(), STARTED_AFTER, startedAfter, WITHOUT, startedAfter, Accepted, PreCommitted,
+                (p1, keyOrRange, txnId, executeAt, status, prev) -> true, null, false, i -> i);
     }
 
-    private static boolean hasCommittedExecutesAfterWithoutWitnessing(SafeCommandStore commandStore, TxnId startedAfter, Ranges ranges, Seekables<?, ?> keys)
+    private static boolean hasCommittedExecutesAfterWithoutWitnessing(SafeCommandStore safeStore, TxnId startedAfter, Ranges ranges, Seekables<?, ?> keys)
     {
         /*
          * The idea here is to discover those transactions that have been decided to execute after us
@@ -382,7 +384,7 @@ public class BeginRecovery extends TxnRequest<BeginRecovery.RecoverReply>
          * witnessed us we are safe to propose the pre-accept timestamp regardless, whereas if any transaction
          * has not witnessed us we can safely invalidate it.
          */
-        return commandStore.mapReduce(keys, ranges, shouldHaveWitnessed(startedAfter.rw()), EXECUTES_AFTER, startedAfter, WITHOUT, startedAfter, Committed, null,
-                (keyOrRange, txnId, executeAt, saveStatus, prev) -> true,false, true);
+        return safeStore.mapReduce(keys, ranges, startedAfter.rw().witnesses(), EXECUTES_AFTER, startedAfter, WITHOUT, startedAfter, Committed, null,
+                (p1, keyOrRange, txnId, executeAt, status, prev) -> true, null, false, i -> i);
     }
 }
diff --git a/accord-core/src/main/java/accord/messages/CheckStatus.java b/accord-core/src/main/java/accord/messages/CheckStatus.java
index 4c8ae207..5feec410 100644
--- a/accord-core/src/main/java/accord/messages/CheckStatus.java
+++ b/accord-core/src/main/java/accord/messages/CheckStatus.java
@@ -18,11 +18,16 @@
 
 package accord.messages;
 
+import java.util.Arrays;
+import java.util.Objects;
+import java.util.function.Predicate;
 import javax.annotation.Nullable;
 
 import accord.api.Result;
 import accord.api.RoutingKey;
 import accord.coordinate.Infer;
+import accord.coordinate.Infer.InvalidIfNot;
+import accord.coordinate.Infer.IsPreempted;
 import accord.local.Command;
 import accord.local.Commands;
 import accord.local.Node;
@@ -32,14 +37,16 @@ import accord.local.SafeCommand;
 import accord.local.SafeCommandStore;
 import accord.local.SaveStatus;
 import accord.local.Status;
+import accord.primitives.AbstractKeys;
 import accord.primitives.Ballot;
 import accord.primitives.EpochSupplier;
 import accord.primitives.PartialDeps;
 import accord.primitives.PartialRoute;
 import accord.primitives.PartialTxn;
-import accord.primitives.Participants;
 import accord.primitives.ProgressToken;
+import accord.primitives.Range;
 import accord.primitives.Ranges;
+import accord.primitives.Routables;
 import accord.primitives.Route;
 import accord.primitives.Timestamp;
 import accord.primitives.TxnId;
@@ -48,17 +55,25 @@ import accord.primitives.Writes;
 import accord.topology.Topologies;
 import accord.utils.Invariants;
 import accord.utils.MapReduceConsume;
+import accord.utils.ReducingIntervalMap;
+import accord.utils.ReducingRangeMap;
 
+import javax.annotation.Nonnull;
+
+import static accord.coordinate.Infer.InvalidIfNot.NotKnownToBeInvalid;
+import static accord.coordinate.Infer.IsPreempted.NotPreempted;
+import static accord.coordinate.Infer.IsPreempted.Preempted;
 import static accord.local.Status.Committed;
-import static accord.local.Status.Definition;
 import static accord.local.Status.Durability;
+import static accord.local.Status.Durability.Local;
+import static accord.local.Status.Durability.Majority;
+import static accord.local.Status.Durability.ShardUniversal;
+import static accord.local.Status.Durability.Universal;
 import static accord.local.Status.Known;
-import static accord.local.Status.KnownDeps;
-import static accord.local.Status.KnownExecuteAt;
+import static accord.local.Status.KnownDeps.DepsErased;
 import static accord.local.Status.NotDefined;
-import static accord.local.Status.Outcome;
-import static accord.local.Status.Phase;
 import static accord.local.Status.Truncated;
+import static accord.messages.CheckStatus.WithQuorum.HasQuorum;
 import static accord.messages.TxnRequest.computeScope;
 import static accord.primitives.Route.castToRoute;
 import static accord.primitives.Route.isRoute;
@@ -70,18 +85,19 @@ public class CheckStatus extends AbstractEpochRequest<CheckStatus.CheckStatusRep
 
     public static class SerializationSupport
     {
-        public static CheckStatusOk createOk(Ranges truncated, Status invalidIfNotAtLeast, SaveStatus status, SaveStatus maxStatus, Ballot promised, Ballot accepted, @Nullable Timestamp executeAt,
+        public static CheckStatusOk createOk(FoundKnownMap map, SaveStatus maxKnowledgeStatus, SaveStatus maxStatus, Ballot promised, Ballot accepted, @Nullable Timestamp executeAt,
                                              boolean isCoordinating, Durability durability,
                                              @Nullable Route<?> route, @Nullable RoutingKey homeKey)
         {
-            return new CheckStatusOk(truncated, invalidIfNotAtLeast, status, maxStatus, promised, accepted, executeAt, isCoordinating, durability, route, homeKey);
+            return new CheckStatusOk(map, maxKnowledgeStatus, maxStatus, promised, accepted, executeAt, isCoordinating, durability, route, homeKey);
         }
-        public static CheckStatusOk createOk(Ranges truncated, Status invalidIfNotAtLeast, SaveStatus status, SaveStatus maxStatus, Ballot promised, Ballot accepted, @Nullable Timestamp executeAt,
+        public static CheckStatusOk createOk(FoundKnownMap map, SaveStatus maxKnowledgeStatus, SaveStatus maxStatus,
+                                             Ballot promised, Ballot accepted, @Nullable Timestamp executeAt,
                                              boolean isCoordinating, Durability durability,
                                              @Nullable Route<?> route, @Nullable RoutingKey homeKey,
                                              PartialTxn partialTxn, PartialDeps committedDeps, Writes writes, Result result)
         {
-            return new CheckStatusOkFull(truncated, invalidIfNotAtLeast, status, maxStatus, promised, accepted, executeAt, isCoordinating, durability, route, homeKey,
+            return new CheckStatusOkFull(map, maxKnowledgeStatus, maxStatus, promised, accepted, executeAt, isCoordinating, durability, route, homeKey,
                                          partialTxn, committedDeps, writes, result);
         }
     }
@@ -142,17 +158,23 @@ public class CheckStatus extends AbstractEpochRequest<CheckStatus.CheckStatusRep
         if (!command.has(Known.DefinitionOnly) && Route.isRoute(query) && safeStore.ranges().allAt(txnId.epoch()).contains(Route.castToRoute(query).homeKey()))
             Commands.informHome(safeStore, safeCommand, Route.castToRoute(query));
 
-        Ranges truncated = command.hasBeen(Truncated) ? safeStore.ranges().allBetween(command.txnId().epoch(), command.executeAtOrTxnId().epoch()) : Ranges.EMPTY;
-        Status invalidIfNotAtLeast = invalidIfNotAtLeast(safeStore);
+        Ranges ranges = safeStore.ranges().allBetween(command.txnId().epoch(), command.executeAtIfKnownOrTxnId().epoch());
+        InvalidIfNot invalidIfNotAtLeast = invalidIfNotAtLeast(safeStore);
         boolean isCoordinating = isCoordinating(node, command);
+        Durability durability = command.durability();
+        Route<?> route = command.route();
+        if (Route.isFullRoute(route))
+            durability = Durability.mergeAtLeast(durability, safeStore.commandStore().durableBefore().min(txnId, route));
+
         switch (includeInfo)
         {
             default: throw new IllegalStateException();
             case No:
             case Route:
-                return new CheckStatusOk(truncated, isCoordinating, invalidIfNotAtLeast, includeInfo == IncludeInfo.No ? null : command.route(), command);
+                Route<?> respondWithRoute = includeInfo == IncludeInfo.No ? null : route;
+                return new CheckStatusOk(ranges, isCoordinating, invalidIfNotAtLeast, respondWithRoute, durability, command);
             case All:
-                return new CheckStatusOkFull(truncated, isCoordinating, invalidIfNotAtLeast, command);
+                return new CheckStatusOkFull(ranges, isCoordinating, invalidIfNotAtLeast, durability, command);
         }
     }
 
@@ -181,7 +203,7 @@ public class CheckStatus extends AbstractEpochRequest<CheckStatus.CheckStatusRep
         else node.reply(replyTo, replyContext, ok, null);
     }
 
-    private Status invalidIfNotAtLeast(SafeCommandStore safeStore)
+    private InvalidIfNot invalidIfNotAtLeast(SafeCommandStore safeStore)
     {
         return Infer.invalidIfNotAtLeast(safeStore, txnId, query);
     }
@@ -191,40 +213,326 @@ public class CheckStatus extends AbstractEpochRequest<CheckStatus.CheckStatusRep
         boolean isOk();
     }
 
+    /**
+     * This is slightly different to Known, in that it represents the knowledge we have obtained from any shard that
+     * can be applied to any other shard, along with any information we might use to infer invalidation.
+     */
+    public static class FoundKnown extends Known
+    {
+        public static final FoundKnown Nothing = new FoundKnown(Known.Nothing, NotKnownToBeInvalid, NotPreempted);
+        public static final FoundKnown Invalidated = new FoundKnown(Known.Invalidated, NotKnownToBeInvalid, NotPreempted);
+
+        public final InvalidIfNot invalidIfNot;
+        public final IsPreempted isPreempted;
+
+        public FoundKnown(Known known, InvalidIfNot invalidIfNot, IsPreempted isPreempted)
+        {
+            super(known);
+            this.invalidIfNot = invalidIfNot;
+            this.isPreempted = isPreempted;
+        }
+
+        public FoundKnown atLeast(FoundKnown with)
+        {
+            Known known = super.atLeast(with);
+            if (known == this)
+                return this;
+            return new FoundKnown(known, invalidIfNot.atLeast(with.invalidIfNot), isPreempted.merge(with.isPreempted));
+        }
+
+        public FoundKnown reduce(FoundKnown with)
+        {
+            Known known = super.reduce(with);
+            if (known == this)
+                return this;
+            return new FoundKnown(known, invalidIfNot.reduce(with.invalidIfNot), isPreempted.validForBoth(with.isPreempted));
+        }
+
+        public FoundKnown validForAll()
+        {
+            Known known = super.validForAll();
+            if (known == this)
+                return this;
+            return new FoundKnown(known, NotKnownToBeInvalid, NotPreempted);
+        }
+
+        public boolean isTruncated()
+        {
+            switch (outcome)
+            {
+                default: throw new AssertionError("Unhandled outcome: " + outcome);
+                case Invalidated:
+                case Unknown:
+                    return false;
+                case Apply:
+                    // since Apply is universal, we can
+                    return deps == DepsErased;
+                case Erased:
+                case WasApply:
+                    return true;
+            }
+        }
+
+        public boolean canProposeInvalidation()
+        {
+            return deps.canProposeInvalidation() && executeAt.canProposeInvalidation() && outcome.canProposeInvalidation();
+        }
+
+        public boolean inferInvalidWithQuorum()
+        {
+            return invalidIfNot.inferInvalidWithQuorum(isPreempted, this);
+        }
+
+        @Override
+        public boolean equals(Object o)
+        {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            FoundKnown that = (FoundKnown) o;
+            return route == that.route && definition == that.definition && executeAt == that.executeAt && deps == that.deps && outcome == that.outcome && invalidIfNot == that.invalidIfNot && isPreempted == that.isPreempted;
+        }
+    }
+
+    // TODO (expected, consider): build only with keys/ranges found in command stores, not the covering ranges of command stores?
+    //   by including the whole covering ranges we can infer
+    public static class FoundKnownMap extends ReducingRangeMap<FoundKnown>
+    {
+        public static class SerializerSupport
+        {
+            public static FoundKnownMap create(boolean inclusiveEnds, RoutingKey[] ends, FoundKnown[] values)
+            {
+                return new FoundKnownMap(inclusiveEnds, ends, values);
+            }
+        }
+
+        private transient final FoundKnown validForAll;
+
+        private FoundKnownMap()
+        {
+            this.validForAll = FoundKnown.Nothing;
+        }
+
+        private FoundKnownMap(boolean inclusiveEnds, RoutingKey[] starts, FoundKnown[] values)
+        {
+            this(inclusiveEnds, starts, values, FoundKnown.Nothing);
+        }
+
+        private FoundKnownMap(boolean inclusiveEnds, RoutingKey[] starts, FoundKnown[] values, FoundKnown validForAll)
+        {
+            super(inclusiveEnds, starts, values);
+            this.validForAll = validForAll;
+        }
+
+        public static FoundKnownMap create(Unseekables<?> keysOrRanges, SaveStatus saveStatus, InvalidIfNot invalidIfNot, Ballot promised)
+        {
+            FoundKnown known = new FoundKnown(saveStatus.known, invalidIfNot, promised.equals(Ballot.ZERO) ? NotPreempted : Preempted);
+            if (keysOrRanges.isEmpty())
+                return new FoundKnownMap();
+
+            switch (keysOrRanges.domain())
+            {
+                default: throw new AssertionError("Unhandled domain type: " + keysOrRanges.domain());
+                case Range:
+                {
+                    Ranges ranges = (Ranges)keysOrRanges;
+                    Builder builder = new Builder(ranges.get(0).endInclusive(), 2 * ranges.size());
+                    for (Range range : ranges)
+                    {
+                        builder.append(range.start(), known, FoundKnown::atLeast);
+                        builder.append(range.end(), null, FoundKnown::atLeast);
+                    }
+                    return builder.build();
+                }
+                case Key:
+                {
+                    AbstractKeys<RoutingKey> keys = (AbstractKeys<RoutingKey>) keysOrRanges;
+                    Builder builder = new Builder(keys.get(0).asRange().endInclusive(), 2 * keys.size());
+                    for (RoutingKey key : keys)
+                    {
+                        Range range = key.asRange();
+                        builder.append(range.start(), known, FoundKnown::atLeast);
+                        builder.append(range.end(), null, FoundKnown::atLeast);
+                    }
+                    return builder.build();
+                }
+            }
+        }
+
+        public static FoundKnownMap merge(FoundKnownMap a, FoundKnownMap b)
+        {
+            return ReducingRangeMap.merge(a, b, FoundKnown::atLeast, Builder::new);
+        }
+
+        private FoundKnownMap finish(Unseekables<?> routeOrParticipants, WithQuorum withQuorum)
+        {
+            FoundKnown validForAll;
+            switch (withQuorum)
+            {
+                default: throw new AssertionError("Unhandled WithQuorum: " + withQuorum);
+                case HasQuorum: validForAll = foldlWithDefault(routeOrParticipants, FoundKnownMap::reduceInferredOrKnownForWithQuorum, FoundKnown.Nothing, null, i -> false); break;
+                case NoQuorum: validForAll = foldlWithDefault(routeOrParticipants, FoundKnownMap::reduceKnownFor, FoundKnown.Nothing, null, i -> false); break;
+            }
+            validForAll = this.validForAll.atLeast(validForAll).validForAll();
+            return with(validForAll);
+        }
+
+        private FoundKnownMap with(FoundKnown validForAll)
+        {
+            if (validForAll.equals(this.validForAll))
+                return this;
+
+            int i = 0;
+            for (; i < size(); ++i)
+            {
+                FoundKnown pre = values[i];
+                if (pre == null)
+                    continue;
+
+                FoundKnown post = pre.atLeast(validForAll);
+                if (!pre.equals(post))
+                    break;
+            }
+
+            if (i == size())
+                return new FoundKnownMap(inclusiveEnds(), starts, values, validForAll);
+
+            RoutingKey[] newStarts = new RoutingKey[size() + 1];
+            FoundKnown[] newValues = new FoundKnown[size()];
+            System.arraycopy(starts, 0, newStarts, 0, i);
+            System.arraycopy(values, 0, newValues, 0, i);
+            int count = i;
+            while (i < size())
+            {
+                FoundKnown pre = values[i++];
+                FoundKnown post = pre == null ? null : pre.atLeast(validForAll);
+                if (count == 0 || !Objects.equals(post, newValues[count - 1]))
+                {
+                    newStarts[count] = starts[i-1];
+                    newValues[count++] = post;
+                }
+            }
+            newStarts[count] = starts[size()];
+            if (count != newValues.length)
+            {
+                newValues = Arrays.copyOf(newValues, count);
+                newStarts = Arrays.copyOf(newStarts, count + 1);
+            }
+            return new FoundKnownMap(inclusiveEnds(), newStarts, newValues, validForAll);
+        }
+
+        public boolean hasTruncated(Routables<?> routables)
+        {
+            return foldlWithDefault(routables, (known, prev) -> known.isTruncated(), FoundKnown.Nothing, false, i -> i);
+        }
+
+        public boolean hasTruncated()
+        {
+            return foldl((known, prev) -> known.isTruncated(), false, i -> i);
+        }
+
+        public Known knownFor(Routables<?> routables)
+        {
+            return validForAll.atLeast(foldlWithDefault(routables, FoundKnownMap::reduceKnownFor, FoundKnown.Nothing, null, i -> false));
+        }
+
+        public Ranges matchingRanges(Predicate<FoundKnown> match)
+        {
+            return foldlWithBounds((known, ranges, start, end) -> match.test(known) ? ranges.with(Ranges.of(start.asRange().newRange(start, end))) : ranges, Ranges.EMPTY, i -> false);
+        }
+
+        private static FoundKnown reduceInferredOrKnownForWithQuorum(FoundKnown foundKnown, @Nullable FoundKnown prev)
+        {
+            if (foundKnown.inferInvalidWithQuorum())
+                foundKnown = foundKnown.atLeast(FoundKnown.Invalidated);
+
+            if (prev == null)
+                return foundKnown;
+
+            return prev.reduce(foundKnown);
+        }
+
+        private static Known reduceKnownFor(FoundKnown foundKnown, @Nullable Known prev)
+        {
+            if (prev == null)
+                return foundKnown;
+
+            return prev.reduce(foundKnown);
+        }
+
+        private static FoundKnown reduceKnownFor(FoundKnown foundKnown, @Nullable FoundKnown prev)
+        {
+            if (prev == null)
+                return foundKnown;
+
+            return prev.reduce(foundKnown);
+        }
+
+        public Ranges knownFor(Known required, Ranges expect)
+        {
+            // TODO (desired): implement and use foldlWithDefaultAndBounds so can subtract rather than add
+            return foldlWithBounds(expect, (known, prev, start, end) -> {
+                if (!required.isSatisfiedBy(known))
+                    return prev;
+
+                return prev.with(Ranges.of(start.asRange().newRange(start, end)));
+            }, Ranges.EMPTY, i -> false);
+        }
+
+        static class Builder extends ReducingIntervalMap.Builder<RoutingKey, FoundKnown, FoundKnownMap>
+        {
+            protected Builder(boolean inclusiveEnds, int capacity)
+            {
+                super(inclusiveEnds, capacity);
+            }
+
+            @Override
+            protected FoundKnownMap buildInternal()
+            {
+                return new FoundKnownMap(inclusiveEnds, starts.toArray(new RoutingKey[0]), values.toArray(new FoundKnown[0]));
+            }
+        }
+    }
+
     public static class CheckStatusOk implements CheckStatusReply
     {
-        public final Ranges truncated;
-        public final Status invalidIfNotAtLeast;
-        public final SaveStatus saveStatus; // the maximum non-truncated status; or truncated if all responses are truncated
-        public final SaveStatus maxSaveStatus;
-        public final Ballot promised;
-        public final Ballot accepted;
+        public final FoundKnownMap map;
+        // TODO (required): tighten up constraints here to ensure we only report truncated when the range is Durable
+        // TODO (expected): stop using saveStatus and maxSaveStatus - move to only Known
+        //   care needed when merging Accepted and AcceptedInvalidate; might be easier to retain saveStatus only for merging these cases
+        public final SaveStatus maxKnowledgeSaveStatus, maxSaveStatus;
+        public final Ballot promised, accepted;
         public final @Nullable Timestamp executeAt; // not set if invalidating or invalidated
         public final boolean isCoordinating;
-        public final Durability durability; // i.e. on all shards
+        public final Durability durability;
         public final @Nullable Route<?> route;
         public final @Nullable RoutingKey homeKey;
 
-        public CheckStatusOk(Ranges truncated, boolean isCoordinating, Status invalidIfNotAtLeast, Command command)
+        public CheckStatusOk(Ranges ranges, boolean isCoordinating, InvalidIfNot invalidIfNot, Durability durability, Command command)
         {
-            this(truncated, isCoordinating, invalidIfNotAtLeast, command.route(), command);
+            this(ranges, isCoordinating, invalidIfNot, command.route(), durability, command);
         }
 
-        public CheckStatusOk(Ranges truncated, boolean isCoordinating, Status invalidIfNotAtLeast, Route<?> route, Command command)
+        public CheckStatusOk(Ranges ranges, boolean isCoordinating, InvalidIfNot invalidIfNot, Route<?> route, Durability durability, Command command)
         {
-            this(truncated, invalidIfNotAtLeast, command.saveStatus(), command.saveStatus(), command.promised(), command.accepted(), command.executeAt(),
-                 isCoordinating, command.durability(), route, command.homeKey());
+            this(ranges, invalidIfNot, command.saveStatus(), command.promised(), command.accepted(), command.executeAt(),
+                 isCoordinating, durability, route, command.homeKey());
         }
 
-        private CheckStatusOk(Ranges truncated, Status invalidIfNotAtLeast, SaveStatus saveStatus, SaveStatus maxSaveStatus,
-                              Ballot promised, Ballot accepted, @Nullable Timestamp executeAt,
+        private CheckStatusOk(Ranges ranges, InvalidIfNot invalidIfNot, SaveStatus saveStatus, Ballot promised,
+                              Ballot accepted, @Nullable Timestamp executeAt,
                               boolean isCoordinating, Durability durability,
                               @Nullable Route<?> route, @Nullable RoutingKey homeKey)
         {
-            this.truncated = truncated;
-            this.invalidIfNotAtLeast = invalidIfNotAtLeast;
-            this.saveStatus = saveStatus;
+            this(FoundKnownMap.create(ranges, saveStatus, invalidIfNot, promised), saveStatus, saveStatus, promised, accepted, executeAt, isCoordinating, durability, route, homeKey);
+        }
+
+        private CheckStatusOk(FoundKnownMap map, SaveStatus maxKnowledgeSaveStatus, SaveStatus maxSaveStatus, Ballot promised, Ballot accepted,
+                              @Nullable Timestamp executeAt, boolean isCoordinating, Durability durability,
+                              @Nullable Route<?> route, @Nullable RoutingKey homeKey)
+        {
+            this.map = map;
             this.maxSaveStatus = maxSaveStatus;
+            this.maxKnowledgeSaveStatus = maxKnowledgeSaveStatus;
             this.promised = promised;
             this.accepted = accepted;
             this.executeAt = executeAt;
@@ -240,22 +548,96 @@ public class CheckStatus extends AbstractEpochRequest<CheckStatus.CheckStatusRep
             return new ProgressToken(durability, status, promised, accepted);
         }
 
-        public boolean isTruncated()
+        public Timestamp executeAtIfKnown()
+        {
+            if (maxKnown().executeAt.isDecidedAndKnownToExecute())
+                return executeAt;
+            return null;
+        }
+
+        public CheckStatusOk finish(Unseekables<?> routeOrParticipants, WithQuorum withQuorum)
+        {
+            CheckStatusOk finished = this;
+            if (withQuorum == HasQuorum)
+            {
+                Durability durability = this.durability;
+                if (durability == Local) durability = Majority;
+                else if (durability == ShardUniversal) durability = Universal;
+                finished = merge(durability);
+            }
+            if (Route.isRoute(routeOrParticipants))
+            {
+                finished = finished.merge(Route.castToRoute(routeOrParticipants));
+                return finished.with(finished.map.finish(finished.route, withQuorum));
+            }
+            else
+            {
+                return finished.with(finished.map.finish(Unseekables.merge(routeOrParticipants, (Unseekables) finished.route), withQuorum));
+            }
+        }
+
+        /**
+         * NOTE: if the response is *incomplete* this does not detect possible truncation, it only indicates if the
+         * combination of the responses we received represents truncation
+         */
+        public boolean isTruncatedResponse()
+        {
+            return map.hasTruncated();
+        }
+
+        public boolean isTruncatedResponse(Routables<?> routables)
+        {
+            return map.hasTruncated(routables);
+        }
+
+        public Ranges truncatedResponse()
+        {
+            return map.matchingRanges(FoundKnown::isTruncated);
+        }
+
+        public CheckStatusOk merge(@Nonnull Route<?> route)
         {
-            return maxSaveStatus.phase == Phase.Cleanup;
+            Route<?> mergedRoute = Route.merge((Route)this.route, route);
+            if (mergedRoute == this.route)
+                return this;
+            return new CheckStatusOk(map, maxKnowledgeSaveStatus, maxSaveStatus, promised, accepted, executeAt,
+                                         isCoordinating, durability, mergedRoute, homeKey);
         }
 
-        public boolean inferInvalidated(WithQuorum withQuorum)
+        public CheckStatusOk merge(@Nonnull Durability durability)
         {
-            return Infer.inferInvalidated(withQuorum, invalidIfNotAtLeast, saveStatus, maxSaveStatus);
+            durability = Durability.merge(durability, this.durability);
+            if (durability == this.durability)
+                return this;
+            return new CheckStatusOk(map, maxKnowledgeSaveStatus, maxSaveStatus, promised, accepted, executeAt,
+                                         isCoordinating, durability, route, homeKey);
         }
 
-        public Known inferredOrKnown(WithQuorum withQuorum)
+        CheckStatusOk with(@Nonnull FoundKnownMap newMap)
         {
-            if (inferInvalidated(withQuorum))
-                return Known.Invalidated;
+            if (newMap == this.map)
+                return this;
+            return new CheckStatusOk(newMap, maxKnowledgeSaveStatus, maxSaveStatus, promised, accepted, executeAt,
+                                         isCoordinating, durability, route, homeKey);
+        }
 
-            return saveStatus.known;
+        // TODO (required): harden markShardStale against unnecessary actions by utilising inferInvalidated==MAYBE and performing a global query
+        public Known knownFor(Routables<?> participants)
+        {
+            Known known = map.knownFor(participants);
+            Invariants.checkState(!known.hasFullRoute() || Route.isFullRoute(route));
+            Invariants.checkState(!known.outcome.isInvalidated() || (!maxKnowledgeSaveStatus.known.isDecidedToExecute() && !maxSaveStatus.known.isDecidedToExecute()));
+            Invariants.checkState(!(maxSaveStatus.known.outcome.isInvalidated() || maxKnowledgeSaveStatus.known.outcome.isInvalidated()) || !known.isDecidedToExecute());
+            // TODO (desired): make sure these match identically, rather than only ensuring Route.isFullRoute (either by coercing it here or by ensuring it at callers)
+            return known;
+        }
+
+        // it is assumed that we are invoking this for a transaction that will execute;
+        // the result may be erroneous if the transaction is invalidated, as logically this can apply to all ranges
+        public Ranges knownFor(Known required, Ranges expect)
+        {
+            Invariants.checkState(maxSaveStatus != SaveStatus.Invalidated);
+            return map.knownFor(required, expect);
         }
 
         @Override
@@ -268,7 +650,9 @@ public class CheckStatus extends AbstractEpochRequest<CheckStatus.CheckStatusRep
         public String toString()
         {
             return "CheckStatusOk{" +
-                   "status:" + saveStatus +
+                   "map:" + map +
+                   "maxNotTruncatedSaveStatus:" + maxKnowledgeSaveStatus +
+                   "maxSaveStatus:" + maxSaveStatus +
                    ", promised:" + promised +
                    ", accepted:" + accepted +
                    ", executeAt:" + executeAt +
@@ -281,10 +665,10 @@ public class CheckStatus extends AbstractEpochRequest<CheckStatus.CheckStatusRep
 
         boolean preferSelf(CheckStatusOk that)
         {
-            if ((this.saveStatus.is(Truncated) && !this.saveStatus.is(NotDefined)) || (that.saveStatus.is(Truncated) && !that.saveStatus.is(NotDefined)))
-                return this.saveStatus.compareTo(that.saveStatus) <= 0;
+            if ((this.maxKnowledgeSaveStatus.is(Truncated) && !this.maxKnowledgeSaveStatus.is(NotDefined)) || (that.maxKnowledgeSaveStatus.is(Truncated) && !that.maxKnowledgeSaveStatus.is(NotDefined)))
+                return this.maxKnowledgeSaveStatus.compareTo(that.maxKnowledgeSaveStatus) <= 0;
 
-            return this.saveStatus.compareTo(that.saveStatus) >= 0;
+            return this.maxKnowledgeSaveStatus.compareTo(that.maxKnowledgeSaveStatus) >= 0;
         }
 
         public CheckStatusOk merge(CheckStatusOk that)
@@ -300,31 +684,43 @@ public class CheckStatus extends AbstractEpochRequest<CheckStatus.CheckStatusRep
             CheckStatusOk defer = prefer == this ? that : this;
 
             // then select the max along each criteria, preferring the coordinator
-            CheckStatusOk maxStatus = SaveStatus.max(prefer, prefer.saveStatus, prefer.accepted, defer, defer.saveStatus, defer.accepted, true);
-            SaveStatus mergeStatus = SaveStatus.merge(prefer.saveStatus, prefer.accepted, defer.saveStatus, defer.accepted, true);
-            SaveStatus mergeMaxStatus = SaveStatus.merge(prefer.saveStatus, prefer.accepted, defer.saveStatus, defer.accepted, false);
+            FoundKnownMap mergeMap = FoundKnownMap.merge(prefer.map, defer.map);
+            CheckStatusOk maxStatus = SaveStatus.max(prefer, prefer.maxKnowledgeSaveStatus, prefer.accepted, defer, defer.maxKnowledgeSaveStatus, defer.accepted, true);
+            SaveStatus mergeMaxKnowledgeStatus = SaveStatus.merge(prefer.maxKnowledgeSaveStatus, prefer.accepted, defer.maxKnowledgeSaveStatus, defer.accepted, true);
+            SaveStatus mergeMaxStatus = SaveStatus.merge(prefer.maxSaveStatus, prefer.accepted, defer.maxSaveStatus, defer.accepted, false);
             CheckStatusOk maxPromised = prefer.promised.compareTo(defer.promised) >= 0 ? prefer : defer;
             CheckStatusOk maxAccepted = prefer.accepted.compareTo(defer.accepted) >= 0 ? prefer : defer;
-            CheckStatusOk maxDurability = prefer.durability.compareTo(defer.durability) >= 0 ? prefer : defer;
             CheckStatusOk maxHomeKey = prefer.homeKey != null || defer.homeKey == null ? prefer : defer;
-            CheckStatusOk maxExecuteAt = prefer.saveStatus.known.executeAt.compareTo(defer.saveStatus.known.executeAt) >= 0 ? prefer : defer;
+            CheckStatusOk maxExecuteAt = prefer.maxKnown().executeAt.compareTo(defer.maxKnown().executeAt) >= 0 ? prefer : defer;
             Route<?> mergedRoute = Route.merge(prefer.route, (Route)defer.route);
-            Ranges mergedTruncated = prefer.truncated.with(defer.truncated);
-            Status mergedInvalidIfNotAtLeast = Status.simpleMax(prefer.invalidIfNotAtLeast, defer.invalidIfNotAtLeast);
+            Durability mergedDurability = Durability.merge(prefer.durability, defer.durability);
 
             // if the maximum (or preferred equal) is the same on all dimensions, return it
-            if (mergeStatus == maxStatus.saveStatus && maxStatus == maxPromised && maxStatus == maxAccepted && maxStatus == maxDurability
-                && maxStatus == maxHomeKey && maxStatus == maxExecuteAt && maxStatus.route == mergedRoute
-                && maxStatus.truncated.equals(mergedTruncated) && maxStatus.invalidIfNotAtLeast == mergedInvalidIfNotAtLeast)
+            if (mergeMaxKnowledgeStatus == maxStatus.maxKnowledgeSaveStatus
+                && mergeMaxStatus == maxStatus.maxSaveStatus
+                && maxStatus == maxPromised && maxStatus == maxAccepted
+                && maxStatus == maxHomeKey && maxStatus == maxExecuteAt
+                && maxStatus.route == mergedRoute
+                && maxStatus.map.equals(mergeMap)
+                && maxStatus.durability == mergedDurability)
             {
                 return maxStatus;
             }
 
             // otherwise assemble the maximum of each, and propagate isCoordinating from the origin we selected the promise from
             boolean isCoordinating = maxPromised == prefer ? prefer.isCoordinating : defer.isCoordinating;
-            return new CheckStatusOk(mergedTruncated, mergedInvalidIfNotAtLeast, mergeStatus, mergeMaxStatus,
-                                     maxPromised.promised, maxAccepted.accepted, maxExecuteAt.executeAt,
-                                     isCoordinating, maxDurability.durability, mergedRoute, maxHomeKey.homeKey);
+            return new CheckStatusOk(mergeMap, mergeMaxKnowledgeStatus, mergeMaxStatus, maxPromised.promised, maxAccepted.accepted, maxExecuteAt.executeAt,
+                                     isCoordinating, mergedDurability, mergedRoute, maxHomeKey.homeKey);
+        }
+
+        public Known maxKnown()
+        {
+            return map.foldl(Known::atLeast, Known.Nothing, i -> false);
+        }
+
+        public InvalidIfNot maxInvalidIfNot()
+        {
+            return map.foldl((known, prev) -> known.invalidIfNot.atLeast(prev), NotKnownToBeInvalid, InvalidIfNot::isMax);
         }
 
         @Override
@@ -341,26 +737,57 @@ public class CheckStatus extends AbstractEpochRequest<CheckStatus.CheckStatusRep
         public final Writes writes;
         public final Result result;
 
-        public CheckStatusOkFull(Ranges truncated, boolean isCoordinating, Status invalidIfNotAtLeast, Command command)
+        public CheckStatusOkFull(Ranges ranges, boolean isCoordinating, InvalidIfNot invalidIfNot, Durability durability, Command command)
         {
-            super(truncated, isCoordinating, invalidIfNotAtLeast, command);
+            super(ranges, isCoordinating, invalidIfNot, durability, command);
             this.partialTxn = command.partialTxn();
             this.committedDeps = command.status().compareTo(Committed) >= 0 ? command.partialDeps() : null;
             this.writes = command.writes();
             this.result = command.result();
         }
 
-        protected CheckStatusOkFull(Ranges truncated, Status invalidIfNotCommitted, SaveStatus status, SaveStatus maxStatus, Ballot promised, Ballot accepted, Timestamp executeAt,
-                                  boolean isCoordinating, Durability durability, Route<?> route,
-                                  RoutingKey homeKey, PartialTxn partialTxn, PartialDeps committedDeps, Writes writes, Result result)
+        protected CheckStatusOkFull(FoundKnownMap map, SaveStatus maxNotTruncatedSaveStatus, SaveStatus maxSaveStatus, Ballot promised, Ballot accepted, Timestamp executeAt,
+                                    boolean isCoordinating, Durability durability, Route<?> route,
+                                    RoutingKey homeKey, PartialTxn partialTxn, PartialDeps committedDeps, Writes writes, Result result)
         {
-            super(truncated, invalidIfNotCommitted, status, maxStatus, promised, accepted, executeAt, isCoordinating, durability, route, homeKey);
+            super(map, maxNotTruncatedSaveStatus, maxSaveStatus, promised, accepted, executeAt, isCoordinating, durability, route, homeKey);
             this.partialTxn = partialTxn;
             this.committedDeps = committedDeps;
             this.writes = writes;
             this.result = result;
         }
 
+        public CheckStatusOkFull finish(Route<?> route, WithQuorum withQuorum)
+        {
+            return (CheckStatusOkFull) super.finish(route, withQuorum);
+        }
+
+        public CheckStatusOkFull merge(@Nonnull Route<?> route)
+        {
+            Route<?> mergedRoute = Route.merge((Route)this.route, route);
+            if (mergedRoute == this.route)
+                return this;
+            return new CheckStatusOkFull(map, maxKnowledgeSaveStatus, maxSaveStatus, promised, accepted, executeAt,
+                                     isCoordinating, durability, mergedRoute, homeKey, partialTxn, committedDeps, writes, result);
+        }
+
+        public CheckStatusOkFull merge(@Nonnull Durability durability)
+        {
+            durability = Durability.merge(durability, this.durability);
+            if (durability == this.durability)
+                return this;
+            return new CheckStatusOkFull(map, maxKnowledgeSaveStatus, maxSaveStatus, promised, accepted, executeAt,
+                                     isCoordinating, durability, route, homeKey, partialTxn, committedDeps, writes, result);
+        }
+
+        CheckStatusOk with(@Nonnull FoundKnownMap newMap)
+        {
+            if (newMap == this.map)
+                return this;
+            return new CheckStatusOkFull(newMap, maxKnowledgeSaveStatus, maxSaveStatus, promised, accepted, executeAt,
+                                     isCoordinating, durability, route, homeKey, partialTxn, committedDeps, writes, result);
+        }
+
         /**
          * This method assumes parameter is of the same type and has the same additional info (modulo partial replication).
          * If parameters have different info, it is undefined which properties will be returned.
@@ -383,7 +810,7 @@ public class CheckStatus extends AbstractEpochRequest<CheckStatus.CheckStatusRep
             CheckStatusOk minSrc = maxSrc == this ? that : this;
             if (!(minSrc instanceof CheckStatusOkFull))
             {
-                return new CheckStatusOkFull(max.truncated, max.invalidIfNotAtLeast, max.saveStatus, max.maxSaveStatus, max.promised, max.accepted, fullMax.executeAt, max.isCoordinating, max.durability, max.route,
+                return new CheckStatusOkFull(max.map, max.maxKnowledgeSaveStatus, max.maxSaveStatus, max.promised, max.accepted, fullMax.executeAt, max.isCoordinating, max.durability, max.route,
                                              max.homeKey, fullMax.partialTxn, fullMax.committedDeps, fullMax.writes, fullMax.result);
             }
 
@@ -397,71 +824,35 @@ public class CheckStatus extends AbstractEpochRequest<CheckStatus.CheckStatusRep
             Writes writes = (fullMax.writes != null ? fullMax : fullMin).writes;
             Result result = (fullMax.result != null ? fullMax : fullMin).result;
 
-            return new CheckStatusOkFull(max.truncated, max.invalidIfNotAtLeast, max.saveStatus, max.maxSaveStatus, max.promised, max.accepted, max.executeAt, max.isCoordinating, max.durability, max.route,
+            return new CheckStatusOkFull(max.map, max.maxKnowledgeSaveStatus, max.maxSaveStatus, max.promised, max.accepted, max.executeAt, max.isCoordinating, max.durability, max.route,
                                          max.homeKey, partialTxn, committedDeps, writes, result);
         }
 
-        public Known sufficientFor(Participants<?> participants, WithQuorum withQuorum)
-        {
-            if (inferInvalidated(withQuorum))
-                return Known.Invalidated;
-
-            return sufficientFor(participants, saveStatus, route, partialTxn, committedDeps, writes, result);
-        }
-
-        private static Known sufficientFor(Participants<?> participants, SaveStatus saveStatus, Route<?> route, PartialTxn partialTxn, PartialDeps committedDeps, Writes writes, Result result)
+        /**
+         * Reduce what is Known about all intersecting shards into a summary Known. This will be the maximal knowledge
+         * we have, i.e. if we have some outcome/decision on one shard but it is truncated on another intersecting shard,
+         * we will get the outcome/decision; if we only have it truncated on one shard and unknown on another, it will
+         * be shown as truncated.
+         *
+         * If a non-intersecting shard has information that can be propagated to this shard, i.e. the executeAt or outcome,
+         * then this will be merged as though it were an intersecting shard, however no record of truncation will be so propagated,
+         * nor any knowledge that does not transfer (i.e. Definition or Deps).
+         */
+        @Override
+        public Known knownFor(Routables<?> participants)
         {
-            Status.Definition definition = saveStatus.known.definition;
-            switch (definition)
-            {
-                default: throw new AssertionError();
-                case DefinitionKnown:
-                    if (partialTxn != null && partialTxn.covers(participants) && route.kind().isFullRoute())
-                        break;
-                    definition = Definition.DefinitionUnknown;
-                case DefinitionUnknown:
-                case NoOp:
-            }
-
-            KnownExecuteAt executeAt = saveStatus.known.executeAt;
-            KnownDeps deps = saveStatus.known.deps;
-            switch (deps)
-            {
-                default: throw new AssertionError();
-                case DepsKnown:
-                    if (committedDeps != null && committedDeps.covers(participants))
-                        break;
-                case DepsProposed:
-                case NoDeps:
-                    deps = KnownDeps.DepsUnknown;
-                case DepsUnknown:
-            }
-
-            Status.Outcome outcome = saveStatus.known.outcome;
-            switch (outcome)
-            {
-                default: throw new AssertionError();
-                case WasApply:
-                    if (writes != null && result != null)
-                        outcome = Outcome.Apply;
-
-                case Apply:
-                    if (writes == null || result == null)
-                        outcome = Outcome.WasApply;
-
-                case Invalidated:
-                case Unknown:
-                case Erased:
-            }
-
-            return new Known(definition, executeAt, deps, outcome);
+            Known known = super.knownFor(participants);
+            Invariants.checkState(!known.hasDefinition() || (partialTxn != null && partialTxn.covering().containsAll(participants)));
+            Invariants.checkState(!known.hasDecidedDeps() || (committedDeps != null && committedDeps.covering.containsAll(participants)));
+            return known;
         }
 
         @Override
         public String toString()
         {
             return "CheckStatusOk{" +
-                   "status:" + saveStatus +
+                   "map:" + map +
+                   ", maxSaveStatus:" + maxSaveStatus +
                    ", promised:" + promised +
                    ", accepted:" + accepted +
                    ", executeAt:" + executeAt +
diff --git a/accord-core/src/main/java/accord/messages/Commit.java b/accord-core/src/main/java/accord/messages/Commit.java
index a1b0404b..6f4bfd76 100644
--- a/accord-core/src/main/java/accord/messages/Commit.java
+++ b/accord-core/src/main/java/accord/messages/Commit.java
@@ -96,7 +96,7 @@ public class Commit extends TxnRequest<ReadNack>
         if (kind == Kind.Maximal)
         {
 //            boolean isHome = coordinateTopology.rangesForNode(to).contains(route.homeKey());
-            // TODO (expected): only includeQuery if isHome; this affects state eviction and is low priority given size in C*
+            // TODO (desired): only includeQuery if isHome; this affects state eviction and is low priority given size in C*
             partialTxn = txn.slice(scope.covering(), true);
             sendRoute = route;
         }
diff --git a/accord-core/src/main/java/accord/messages/GetDeps.java b/accord-core/src/main/java/accord/messages/GetDeps.java
index 083c15c8..28348ef2 100644
--- a/accord-core/src/main/java/accord/messages/GetDeps.java
+++ b/accord-core/src/main/java/accord/messages/GetDeps.java
@@ -33,6 +33,7 @@ import accord.topology.Topologies;
 import accord.utils.Invariants;
 
 import static accord.messages.PreAccept.calculatePartialDeps;
+import static accord.primitives.EpochSupplier.constant;
 
 public class GetDeps extends TxnRequest.WithUnsynced<PartialDeps>
 {
@@ -71,7 +72,7 @@ public class GetDeps extends TxnRequest.WithUnsynced<PartialDeps>
     public PartialDeps apply(SafeCommandStore instance)
     {
         Ranges ranges = instance.ranges().allBetween(minUnsyncedEpoch, executeAt);
-        return calculatePartialDeps(instance, txnId, keys, executeAt, ranges);
+        return calculatePartialDeps(instance, txnId, keys, constant(minUnsyncedEpoch), executeAt, ranges);
     }
 
     @Override
diff --git a/accord-core/src/main/java/accord/messages/MessageType.java b/accord-core/src/main/java/accord/messages/MessageType.java
index a3fe592a..56ea2ed4 100644
--- a/accord-core/src/main/java/accord/messages/MessageType.java
+++ b/accord-core/src/main/java/accord/messages/MessageType.java
@@ -66,7 +66,7 @@ public class MessageType
     public static final MessageType SET_GLOBALLY_DURABLE_REQ         = mt(REMOTE, true );
     public static final MessageType QUERY_DURABLE_BEFORE_REQ         = mt(REMOTE, false);
     public static final MessageType QUERY_DURABLE_BEFORE_RSP         = mt(REMOTE, false);
-    public static final MessageType APPLY_AND_WAIT_UNTIL_APPLIED_REQ = mt(REMOTE, true );
+    public static final MessageType APPLY_THEN_WAIT_UNTIL_APPLIED_REQ= mt(REMOTE, true );
 
     public static final MessageType PROPAGATE_PRE_ACCEPT_MSG         = mt(LOCAL,  true );
     public static final MessageType PROPAGATE_COMMIT_MSG             = mt(LOCAL,  true );
diff --git a/accord-core/src/main/java/accord/messages/PreAccept.java b/accord-core/src/main/java/accord/messages/PreAccept.java
index ce72cebc..45e9df5a 100644
--- a/accord-core/src/main/java/accord/messages/PreAccept.java
+++ b/accord-core/src/main/java/accord/messages/PreAccept.java
@@ -20,6 +20,7 @@ package accord.messages;
 
 import java.util.List;
 import java.util.Objects;
+import java.util.function.Function;
 import javax.annotation.Nullable;
 
 import org.slf4j.Logger;
@@ -30,7 +31,6 @@ import accord.local.Commands;
 import accord.local.Node.Id;
 import accord.local.SafeCommand;
 import accord.local.SafeCommandStore;
-import accord.local.SafeCommandStore.TestKind;
 import accord.messages.TxnRequest.WithUnsynced;
 import accord.primitives.Deps;
 import accord.primitives.EpochSupplier;
@@ -105,10 +105,12 @@ public class PreAccept extends WithUnsynced<PreAccept.PreAcceptReply> implements
     protected PreAcceptReply applyIfDoesNotCoordinate(SafeCommandStore safeStore)
     {
         // we only preaccept in the coordination epoch, but we might contact other epochs for dependencies
+        // TODO (required): for recovery we need to update CommandsForKeys et al, even if we don't participate in coordination.
+        //      must consider cleanup though. (alternative is to make recovery more complicated)
         Ranges ranges = safeStore.ranges().allBetween(minUnsyncedEpoch, txnId);
         if (txnId.rw() == ExclusiveSyncPoint)
             safeStore.commandStore().markExclusiveSyncPoint(safeStore, txnId, ranges);
-        return new PreAcceptOk(txnId, txnId, calculatePartialDeps(safeStore, txnId, partialTxn.keys(), txnId, ranges));
+        return new PreAcceptOk(txnId, txnId, calculatePartialDeps(safeStore, txnId, partialTxn.keys(), EpochSupplier.constant(minUnsyncedEpoch), txnId, ranges));
     }
 
     @Override
@@ -126,15 +128,18 @@ public class PreAccept extends WithUnsynced<PreAccept.PreAcceptReply> implements
         {
             default:
             case Success:
-            case Redundant: // we might hit 'Redundant' if we have to contact later epochs and partially re-contact a node we already contacted
+                // we might hit 'Redundant' if we have to contact later epochs and partially re-contact a node we already contacted
+                // TODO (expected): consider dedicated special case, or rename
+            case Redundant:
                 Command command = safeCommand.current();
                 // for efficiency, we don't usually return dependencies newer than txnId as they aren't necessarily needed
                 // for recovery, and it's better to persist less data than more. However, for exclusive sync points we
                 // don't need to perform an Accept round, nor do we need to persist this state to aid recovery. We just
                 // want the issuer of the sync point to know which transactions to wait for before it can safely treat
                 // all transactions with lower txnId as expired.
+                Ranges ranges = safeStore.ranges().allBetween(minUnsyncedEpoch, txnId);
                 return new PreAcceptOk(txnId, command.executeAt(),
-                        calculatePartialDeps(safeStore, txnId, partialTxn.keys(), txnId, safeStore.ranges().allBetween(minUnsyncedEpoch, txnId)));
+                                       calculatePartialDeps(safeStore, txnId, partialTxn.keys(), EpochSupplier.constant(minUnsyncedEpoch), txnId, ranges));
 
             case Truncated:
             case RejectedBallot:
@@ -247,29 +252,31 @@ public class PreAccept extends WithUnsynced<PreAccept.PreAcceptReply> implements
         }
     }
 
-    static PartialDeps calculatePartialDeps(SafeCommandStore commandStore, TxnId txnId, Seekables<?, ?> keys, Timestamp executeAt, Ranges ranges)
+    static PartialDeps calculatePartialDeps(SafeCommandStore safeStore, TxnId txnId, Seekables<?, ?> keys, EpochSupplier minEpoch, Timestamp executeAt, Ranges ranges)
     {
-        try (PartialDeps.Builder builder = PartialDeps.builder(ranges))
-        {
-            return calculateDeps(commandStore, txnId, keys, executeAt, ranges, builder);
-        }
+        // TODO (expected): do not build covering ranges; no longer especially valuable given use of FullRoute
+        return calculateDeps(safeStore, txnId, keys, minEpoch, executeAt, ranges, PartialDeps::builder);
     }
 
-    private static <T extends Deps> T calculateDeps(SafeCommandStore commandStore, TxnId txnId, Seekables<?, ?> keys, Timestamp executeAt, Ranges ranges, Deps.AbstractBuilder<T> builder)
+    private static <T extends Deps> T calculateDeps(SafeCommandStore safeStore, TxnId txnId, Seekables<?, ?> keys, EpochSupplier minEpoch, Timestamp executeAt, Ranges ranges, Function<Ranges, Deps.AbstractBuilder<T>> builderFactory)
     {
-        TestKind testKind = TestKind.conflicts(txnId.rw());
         // could use MAY_EXECUTE_BEFORE to prune those we know execute later.
         // NOTE: ExclusiveSyncPoint *relies* on STARTED_BEFORE to ensure it reports a dependency on *every* earlier TxnId that may execute after it.
         //       This is necessary for reporting to a bootstrapping replica which TxnId it must not prune from dependencies
         //       i.e. the source replica reports to the target replica those TxnId that STARTED_BEFORE and EXECUTES_AFTER.
-        commandStore.mapReduce(keys, ranges, testKind, STARTED_BEFORE, executeAt, ANY_DEPS, null, null, null,
-                (keyOrRange, testTxnId, testExecuteAt, saveStatus, in) -> {
-                    // TODO (easy, efficiency): either pass txnId as parameter or encode this behaviour in a specialised builder to avoid extra allocations
-                    if (!testTxnId.equals(txnId))
-                        in.add(keyOrRange, testTxnId);
-                    return in;
-                }, builder, null);
-        return builder.build();
+        try (Deps.AbstractBuilder<T> depBuilder = builderFactory.apply(ranges); Deps.AbstractBuilder<T> redundantBuilder = builderFactory.apply(ranges))
+        {
+            // TODO (expected): this can be neater
+            safeStore.mapReduce(keys, ranges, txnId.rw().witnesses(), STARTED_BEFORE, executeAt, ANY_DEPS, null, null, null,
+                                (txnId0, keyOrRange, testTxnId, testExecuteAt, testStatus, in) -> {
+                                    if (!testTxnId.equals(txnId0))
+                                        in.add(keyOrRange, testTxnId);
+                                    return in;
+                                }, txnId, depBuilder);
+            T deps = depBuilder.build();
+            T redundant = safeStore.commandStore().redundantBefore().collectDeps(keys, redundantBuilder, minEpoch, executeAt).build();
+            return (T)deps.with(redundant);
+        }
     }
 
     /**
diff --git a/accord-core/src/main/java/accord/messages/Propagate.java b/accord-core/src/main/java/accord/messages/Propagate.java
index 68a63df1..a0821bcc 100644
--- a/accord-core/src/main/java/accord/messages/Propagate.java
+++ b/accord-core/src/main/java/accord/messages/Propagate.java
@@ -27,6 +27,8 @@ import accord.local.SafeCommand;
 import accord.local.SafeCommandStore;
 import accord.local.SaveStatus;
 import accord.local.Status;
+import accord.messages.CheckStatus.FoundKnownMap;
+import accord.messages.CheckStatus.WithQuorum;
 import accord.primitives.EpochSupplier;
 import accord.primitives.Keys;
 import accord.primitives.PartialDeps;
@@ -44,34 +46,40 @@ import accord.utils.MapReduceConsume;
 import javax.annotation.Nullable;
 import java.util.function.BiConsumer;
 
+import static accord.coordinate.Infer.InvalidIfNot.NotKnownToBeInvalid;
+import static accord.local.RedundantStatus.LOCALLY_REDUNDANT;
 import static accord.local.Status.NotDefined;
 import static accord.local.Status.Phase.Cleanup;
 import static accord.local.Status.PreApplied;
+import static accord.messages.CheckStatus.WithQuorum.HasQuorum;
 import static accord.primitives.Routables.Slice.Minimal;
 
 public class Propagate implements MapReduceConsume<SafeCommandStore, Void>, EpochSupplier, LocalMessage
 {
     public static class SerializerSupport
     {
-        public static Propagate create(TxnId txnId, Route<?> route, SaveStatus saveStatus, SaveStatus maxSaveStatus, Status.Durability durability, RoutingKey homeKey, RoutingKey progressKey, Status.Known achieved, PartialTxn partialTxn, PartialDeps partialDeps, long toEpoch, Timestamp executeAt, Writes writes, Result result)
+        public static Propagate create(TxnId txnId, Route<?> route, SaveStatus maxKnowledgeSaveStatus, SaveStatus maxSaveStatus, Status.Durability durability, RoutingKey homeKey, RoutingKey progressKey, Status.Known achieved, FoundKnownMap known, boolean isTruncated, PartialTxn partialTxn, PartialDeps committedDeps, long toEpoch, Timestamp executeAt, Writes writes, Result result)
         {
-            return new Propagate(txnId, route, saveStatus, maxSaveStatus, durability, homeKey, progressKey, achieved, partialTxn, partialDeps, toEpoch, executeAt, writes, result, null);
+            return new Propagate(txnId, route, maxKnowledgeSaveStatus, maxSaveStatus, durability, homeKey, progressKey, achieved, known, isTruncated, partialTxn, committedDeps, toEpoch, executeAt, writes, result, null);
         }
     }
 
     public final TxnId txnId;
     public final Route<?> route;
-    public final SaveStatus saveStatus;
+    // TODO (expected): remove dependency on these two SaveStatus
+    public final SaveStatus maxKnowledgeSaveStatus;
     public final SaveStatus maxSaveStatus;
     public final Status.Durability durability;
     @Nullable public final RoutingKey homeKey;
     @Nullable public final RoutingKey progressKey;
     // this is a WHOLE NODE measure, so if commit epoch has more ranges we do not count as committed if we can only commit in coordination epoch
     public final Status.Known achieved;
+    public final FoundKnownMap known;
+    public final boolean isTruncated;
     @Nullable public final PartialTxn partialTxn;
-    @Nullable public final PartialDeps partialDeps;
+    @Nullable public final PartialDeps committedDeps;
     public final long toEpoch;
-    @Nullable public final Timestamp executeAt;
+    @Nullable public final Timestamp committedExecuteAt;
     @Nullable public final Writes writes;
     @Nullable public final Result result;
 
@@ -80,104 +88,107 @@ public class Propagate implements MapReduceConsume<SafeCommandStore, Void>, Epoc
     Propagate(
         TxnId txnId,
         Route<?> route,
-        SaveStatus saveStatus,
+        SaveStatus maxKnowledgeSaveStatus,
         SaveStatus maxSaveStatus,
         Status.Durability durability,
         @Nullable RoutingKey homeKey,
         @Nullable RoutingKey progressKey,
         Status.Known achieved,
+        FoundKnownMap known,
+        boolean isTruncated,
         @Nullable PartialTxn partialTxn,
-        @Nullable PartialDeps partialDeps,
+        @Nullable PartialDeps committedDeps,
         long toEpoch,
-        @Nullable Timestamp executeAt,
+        @Nullable Timestamp committedExecuteAt,
         @Nullable Writes writes,
         @Nullable Result result,
         BiConsumer<Status.Known, Throwable> callback)
     {
         this.txnId = txnId;
         this.route = route;
-        this.saveStatus = saveStatus;
+        this.maxKnowledgeSaveStatus = maxKnowledgeSaveStatus;
         this.maxSaveStatus = maxSaveStatus;
         this.durability = durability;
         this.homeKey = homeKey;
         this.progressKey = progressKey;
         this.achieved = achieved;
+        this.known = known;
+        this.isTruncated = isTruncated;
         this.partialTxn = partialTxn;
-        this.partialDeps = partialDeps;
+        this.committedDeps = committedDeps;
         this.toEpoch = toEpoch;
-        this.executeAt = executeAt;
+        this.committedExecuteAt = committedExecuteAt;
         this.writes = writes;
         this.result = result;
         this.callback = callback;
     }
 
     @SuppressWarnings({"rawtypes", "unchecked"})
-    public static void propagate(Node node, TxnId txnId, long sourceEpoch, CheckStatus.WithQuorum withQuorum, Route route, @Nullable Status.Known target, CheckStatus.CheckStatusOkFull full, BiConsumer<Status.Known, Throwable> callback)
+    public static void propagate(Node node, TxnId txnId, long sourceEpoch, WithQuorum withQuorum, Route route, @Nullable Status.Known target, CheckStatus.CheckStatusOkFull full, BiConsumer<Status.Known, Throwable> callback)
     {
-        if (full.saveStatus.status == NotDefined && full.invalidIfNotAtLeast == NotDefined)
+        if (full.maxKnowledgeSaveStatus.status == NotDefined && full.maxInvalidIfNot() == NotKnownToBeInvalid)
         {
             callback.accept(Status.Known.Nothing, null);
             return;
         }
 
-        Invariants.checkState(sourceEpoch == txnId.epoch() || (full.executeAt != null && sourceEpoch == full.executeAt.epoch()));
-        Route<?> maxRoute = Route.merge(route, full.route);
+        Invariants.checkState(sourceEpoch == txnId.epoch() || (full.executeAt != null && sourceEpoch == full.executeAt.epoch()) || full.maxKnowledgeSaveStatus == SaveStatus.Erased);
+
+        full = full.finish(route, withQuorum);
+        route = Invariants.nonNull(full.route);
 
         // TODO (required): permit individual shards that are behind to catch up by themselves
         long toEpoch = sourceEpoch;
         Ranges sliceRanges = node.topology().localRangesForEpochs(txnId.epoch(), toEpoch);
-        if (!maxRoute.covers(sliceRanges))
-        {
-            callback.accept(Status.Known.Nothing, null);
-            return;
-        }
 
-        RoutingKey progressKey = node.trySelectProgressKey(txnId, maxRoute);
+        RoutingKey progressKey = node.trySelectProgressKey(txnId, route);
 
-        Ranges covering = maxRoute.sliceCovering(sliceRanges, Minimal);
-        Participants<?> participatingKeys = maxRoute.participants().slice(covering, Minimal);
-        Status.Known achieved = full.sufficientFor(participatingKeys, withQuorum);
-        if (achieved.executeAt.hasDecidedExecuteAt() && full.executeAt.epoch() > toEpoch)
+        Ranges covering = route.sliceCovering(sliceRanges, Minimal);
+        Participants<?> participatingKeys = route.participants().slice(covering, Minimal);
+        Status.Known achieved = full.knownFor(participatingKeys);
+        if (achieved.executeAt.isDecidedAndKnownToExecute() && full.executeAt.epoch() > toEpoch)
         {
             Ranges acceptRanges;
             if (!node.topology().hasEpoch(full.executeAt.epoch()) ||
-                    (!maxRoute.covers(acceptRanges = node.topology().localRangesForEpochs(txnId.epoch(), full.executeAt.epoch()))))
+                (!route.covers(acceptRanges = node.topology().localRangesForEpochs(txnId.epoch(), full.executeAt.epoch()))))
             {
                 // we don't know what the execution epoch requires, so we cannot be sure we can replicate it locally
                 // we *could* wait until we have the local epoch before running this
                 Status.Outcome outcome = achieved.outcome.propagatesBetweenShards() ? achieved.outcome : Status.Outcome.Unknown;
-                achieved = new Status.Known(achieved.definition, achieved.executeAt, Status.KnownDeps.DepsUnknown, outcome);
+                achieved = new Status.Known(achieved.route, achieved.definition, achieved.executeAt, Status.KnownDeps.DepsUnknown, outcome);
             }
             else
             {
                 // TODO (expected): this should only be the two precise epochs, not the full range of epochs
                 sliceRanges = acceptRanges;
-                covering = maxRoute.sliceCovering(sliceRanges, Minimal);
-                participatingKeys = maxRoute.participants().slice(covering, Minimal);
-                Status.Known knownForExecution = full.sufficientFor(participatingKeys, withQuorum);
-                if ((target != null && target.isSatisfiedBy(knownForExecution)) || knownForExecution.isSatisfiedBy(achieved))
+                covering = route.sliceCovering(sliceRanges, Minimal);
+                participatingKeys = route.participants().slice(covering, Minimal);
+                Status.Known knownForExecution = full.knownFor(participatingKeys);
+                if ((target != null && target.isSatisfiedBy(knownForExecution)) || achieved.isSatisfiedBy(knownForExecution))
                 {
                     achieved = knownForExecution;
                     toEpoch = full.executeAt.epoch();
                 }
                 else
-                {
+                {   // TODO (expected): does downgrading this ever block progress?
                     Invariants.checkState(sourceEpoch == txnId.epoch(), "%d != %d", sourceEpoch, txnId.epoch());
-                    achieved = new Status.Known(achieved.definition, achieved.executeAt, knownForExecution.deps, knownForExecution.outcome);
+                    achieved = new Status.Known(achieved.route, achieved.definition, achieved.executeAt, knownForExecution.deps, knownForExecution.outcome);
                 }
             }
         }
 
-        PartialTxn partialTxn = null;
+        boolean isTruncated = withQuorum == HasQuorum && full.isTruncatedResponse(covering);
+
+        PartialTxn partialTxn = full.partialTxn;
         if (achieved.definition.isKnown())
             partialTxn = full.partialTxn.slice(sliceRanges, true).reconstitutePartial(covering);
 
-        PartialDeps partialDeps = null;
+        PartialDeps committedDeps = full.committedDeps;
         if (achieved.deps.hasDecidedDeps())
-            partialDeps = full.committedDeps.slice(sliceRanges).reconstitutePartial(covering);
+            committedDeps = full.committedDeps.slice(sliceRanges).reconstitutePartial(covering);
 
         Propagate propagate =
-            new Propagate(txnId, maxRoute, full.saveStatus, full.maxSaveStatus, full.durability, full.homeKey, progressKey, achieved, partialTxn, partialDeps, toEpoch, full.executeAt, full.writes, full.result, callback);
+            new Propagate(txnId, route, full.maxKnowledgeSaveStatus, full.maxSaveStatus, full.durability, full.homeKey, progressKey, achieved, full.map, isTruncated, partialTxn, committedDeps, toEpoch, full.executeAtIfKnown(), full.writes, full.result, callback);
 
         node.localMessage(propagate);
     }
@@ -191,10 +202,12 @@ public class Propagate implements MapReduceConsume<SafeCommandStore, Void>, Epoc
     @Override
     public Seekables<?, ?> keys()
     {
+        // TODO (now): this is insufficient, as we may applyOrUpgradeTruncated
+        // since we don't calculateDeps, we might not need to load CFK to register with it
         if (achieved.definition.isKnown())
             return partialTxn.keys();
         else if (achieved.deps.hasProposedOrDecidedDeps())
-            return partialDeps.keyDeps.keys();
+            return committedDeps.keyDeps.keys();
         else
             return Keys.EMPTY;
     }
@@ -210,40 +223,61 @@ public class Propagate implements MapReduceConsume<SafeCommandStore, Void>, Epoc
     {
         SafeCommand safeCommand = safeStore.get(txnId, this, route);
         Command command = safeCommand.current();
-        if (command.saveStatus().phase.compareTo(Status.Phase.Persist) >= 0)
-            return null;
 
-        Status propagate = achieved.propagate();
+        PartialTxn partialTxn = achieved.hasDefinition() ? this.partialTxn : null;
+        PartialDeps committedDeps = achieved.hasDecidedDeps() ? this.committedDeps : null;
+        switch (command.saveStatus().phase)
+        {
+            // Already know the outcome, waiting on durability so maybe update with new durability information which can also trigger cleanup
+            case Persist: return updateDurability(safeStore, safeCommand);
+            case Cleanup: return null;
+        }
+
+        Timestamp executeAtIfKnown = command.executeAtIfKnown(committedExecuteAt);
+        Status.Known achieved = this.achieved;
+        if (isTruncated)
+        {
+            // TODO (required): do not markShardStale for reads; in general optimise handling of case where we cannot recover a known no-op transaction
+
+            achieved = applyOrUpgradeTruncated(safeStore, safeCommand, command, executeAtIfKnown);
+            if (achieved == null)
+                return null;
+
+            Ranges needed = safeStore.ranges().allBetween(txnId.epoch(), (executeAtIfKnown == null ? txnId : executeAtIfKnown).epoch());
+            if (achieved.isDefinitionKnown() && partialTxn == null)
+            {
+                PartialTxn existing = command.partialTxn();
+                Ranges neededForDefinition = existing == null ? needed : needed.subtract(existing.covering());
+                partialTxn = this.partialTxn.slice(needed, true).reconstitutePartial(neededForDefinition);
+            }
+
+            if (achieved.hasDecidedDeps() && committedDeps == null)
+            {
+                Invariants.checkState(executeAtIfKnown != null);
+                // we don't subtract existing partialDeps, as they cannot be committed deps; we only permit committing deps covering all participating ranges
+                committedDeps = this.committedDeps.slice(needed).reconstitutePartial(needed);
+            }
+        }
+
+        Status propagate = achieved.atLeast(command.known()).propagatesStatus();
         if (command.hasBeen(propagate))
         {
-            if (maxSaveStatus.phase == Cleanup && durability.isDurableOrInvalidated() && Infer.safeToCleanup(safeStore, command, route, executeAt))
+            if (maxSaveStatus.phase == Cleanup && durability.isDurableOrInvalidated() && Infer.safeToCleanup(safeStore, command, route, executeAtIfKnown))
                 Commands.setTruncatedApply(safeStore, safeCommand);
-            return null;
+
+            // TODO (expected): maybe stale?
+            return updateDurability(safeStore, safeCommand);
         }
 
         switch (propagate)
         {
             default: throw new IllegalStateException("Unexpected status: " + propagate);
+            case Truncated: throw new IllegalStateException("Status expected to be handled elsewhere: " + propagate);
             case Accepted:
             case AcceptedInvalidate:
                 // we never "propagate" accepted statuses as these are essentially votes,
                 // and contribute nothing to our local state machine
-                throw new IllegalStateException("Invalid states to propagate: " + achieved.propagate());
-
-            case Truncated:
-                // if our peers have truncated this command, then either:
-                // 1) we have already applied it locally; 2) the command doesn't apply locally; 3) we are stale; or 4) the command is invalidated
-                if (command.hasBeen(PreApplied) || command.saveStatus().isUninitialised())
-                    break;
-
-                if (Infer.safeToCleanup(safeStore, command, route, executeAt))
-                {
-                    Commands.setErased(safeStore, safeCommand);
-                    break;
-                }
-
-                // TODO (required): check if we are stale
-                // otherwise we are either stale, or the command didn't reach consensus
+                throw new IllegalStateException("Invalid states to propagate: " + propagate);
 
             case Invalidated:
                 Commands.commitInvalidate(safeStore, safeCommand, route);
@@ -251,20 +285,21 @@ public class Propagate implements MapReduceConsume<SafeCommandStore, Void>, Epoc
 
             case Applied:
             case PreApplied:
-                Invariants.checkState(executeAt != null);
-                if (toEpoch >= executeAt.epoch())
+                Invariants.checkState(executeAtIfKnown != null);
+                if (toEpoch >= executeAtIfKnown.epoch())
                 {
-                    confirm(Commands.apply(safeStore, safeCommand, txnId, route, progressKey, executeAt, partialDeps, partialTxn, writes, result));
+                    confirm(Commands.apply(safeStore, safeCommand, txnId, route, progressKey, executeAtIfKnown, committedDeps, partialTxn, writes, result));
                     break;
                 }
 
             case Committed:
             case ReadyToExecute:
-                confirm(Commands.commit(safeStore, safeCommand, txnId, route, progressKey, partialTxn, executeAt, partialDeps));
+                confirm(Commands.commit(safeStore, safeCommand, txnId, route, progressKey, partialTxn, executeAtIfKnown, committedDeps));
                 break;
 
             case PreCommitted:
-                Commands.precommit(safeStore, safeCommand, txnId, executeAt, route);
+                confirm(Commands.precommit(safeStore, safeCommand, txnId, executeAtIfKnown, route));
+                // TODO (desired): would it be clearer to yield a SaveStatus so we can have PreCommittedWithDefinition
                 if (!achieved.definition.isKnown())
                     break;
 
@@ -278,15 +313,119 @@ public class Propagate implements MapReduceConsume<SafeCommandStore, Void>, Epoc
                 break;
         }
 
+        return updateDurability(safeStore, safeCommand);
+    }
 
-        if (!durability.isDurable() || homeKey == null)
+    // if can only propagate Truncated, we might be stale; try to upgrade for this command store only, even partially if necessary
+    // note: this is invoked if the command is truncated for ANY local command store - we might
+    private Status.Known applyOrUpgradeTruncated(SafeCommandStore safeStore, SafeCommand safeCommand, Command command, Timestamp executeAtIfKnown)
+    {
+        Invariants.checkState(!maxKnowledgeSaveStatus.is(Status.Invalidated));
+
+        if (Infer.safeToCleanup(safeStore, command, route, executeAtIfKnown))
+        {
+            // don't create a new Erased record if we're already cleaned up
+            Commands.setErased(safeStore, safeCommand);
             return null;
+        }
+
+        Ranges ranges = safeStore.ranges().allBetween(txnId.epoch(), (executeAtIfKnown == null ? txnId : executeAtIfKnown).epoch());
+        Participants<?> participants = route.participants(ranges, Minimal);
+        Invariants.checkState(!participants.isEmpty()); // we shouldn't be fetching data for transactions we only coordinate
+        boolean isTruncatedForLocalRanges = known.hasTruncated(participants);
 
-        if (!safeStore.ranges().coordinates(txnId).contains(homeKey))
+        if (!isTruncatedForLocalRanges)
+        {
+            // we're truncated *somewhere* but not locally; whether we have the executeAt is immaterial to this calculus,
+            // as we're either ready to go or we're waiting on the coordinating shard to complete this transaction, so pick
+            // the maximum we can achieve and return that
+            return known.knownFor(participants);
+        }
+
+        // if our peers have truncated this command, then either:
+        // 1) we have already applied it locally; 2) the command doesn't apply locally; 3) we are stale; or 4) the command is invalidated
+        // we're now at least partially stale, but let's first see if we can progress this shard, or we can do so in part
+        if (executeAtIfKnown == null)
+        {
+            // TODO (desired): merge this with the executeAt known path where possible, as a lot of duplication
+            if (safeStore.commandStore().redundantBefore().status(txnId, txnId, participants).compareTo(LOCALLY_REDUNDANT) >= 0)
+            {
+                // invalidated
+                Commands.setErased(safeStore, safeCommand);
+                return null;
+            }
+
+            ranges = safeStore.commandStore().redundantBefore().everExpectToExecute(txnId, ranges);
+            if (!ranges.isEmpty())
+            {
+                // we don't even know the execution time, so we cannot possibly proceed besides erasing the command state and marking ourselves stale
+                // TODO (required): we could in principle be stale for future epochs we haven't witnessed yet. Ensure up to date epochs before finalising this application, or else fetch a maximum possible epoch
+                safeStore.commandStore().markShardStale(safeStore, txnId, participants.toRanges().slice(ranges, Minimal), false);
+            }
+            Commands.setErased(safeStore, safeCommand);
+            return null;
+        }
+
+        if (safeStore.commandStore().redundantBefore().status(txnId, executeAtIfKnown, participants).compareTo(LOCALLY_REDUNDANT) >= 0)
+        {
+            // invalidated
+            Commands.setErased(safeStore, safeCommand);
+            return null;
+        }
+
+        // compute the ranges we expect to execute - i.e. those we own, and are not stale or pre-bootstrap
+        ranges = safeStore.commandStore().redundantBefore().expectToExecute(txnId, executeAtIfKnown, ranges);
+        if (ranges.isEmpty())
+        {
+            // TODO (expected): we might prefer to adopt Redundant status, and permit ourselves to later accept the result of the execution and/or definition
+            Commands.setTruncatedApply(safeStore, safeCommand, route);
+            return null;
+        }
+
+        // if the command has been truncated globally, then we should expect to apply it
+        // if we cannot obtain enough information from a majority to do so then we have been left behind
+        Status.Known required = PreApplied.minKnown;
+        Status.Known requireExtra = required.subtract(command.known()); // the extra information we need to reach pre-applied
+        Ranges achieveRanges = known.knownFor(requireExtra, ranges); // the ranges for which we can successfully achieve this
+
+        if (participants.isEmpty())
+        {
+            // we only coordinate this transaction, so being unable to retrieve its state does not imply any staleness
+            // TODO (now): double check this doesn't stop us coordinating the transaction (it shouldn't, as doesn't imply durability)
+            Commands.setTruncatedApply(safeStore, safeCommand, route);
+            return null;
+        }
+
+        // any ranges we execute but cannot achieve the pre-applied status for have been left behind and are stale
+        Ranges staleRanges = ranges.subtract(achieveRanges);
+        Participants<?> staleParticipants = participants.slice(staleRanges, Minimal);
+        staleRanges = staleParticipants.toRanges();
+
+        if (staleRanges.isEmpty())
+        {
+            Invariants.checkState(achieveRanges.containsAll(participants));
+            return required;
+        }
+
+        safeStore.commandStore().markShardStale(safeStore, executeAtIfKnown, staleRanges, true);
+        if (!staleRanges.containsAll(participants))
+            return required;
+
+        // TODO (expected): we might prefer to adopt Redundant status, and permit ourselves to later accept the result of the execution and/or definition
+        Commands.setTruncatedApply(safeStore, safeCommand, route);
+        return null;
+    }
+
+    /*
+     *  If there is new information about the command being durable and we are in the coordination shard in the coordination epoch then update the durability information and possibly cleanup
+     */
+    private Void updateDurability(SafeCommandStore safeStore, SafeCommand safeCommand)
+    {
+        // TODO (expected): Infer durability status from cleanup/truncation
+        if (!durability.isDurable() || homeKey == null)
             return null;
 
-        Timestamp executeAt = saveStatus.known.executeAt.hasDecidedExecuteAt() ? this.executeAt : null;
-        Commands.setDurability(safeStore, safeCommand, durability, route, executeAt);
+        Commands.setDurability(safeStore, safeCommand, durability, route, committedExecuteAt);
         return null;
     }
 
@@ -306,11 +445,12 @@ public class Propagate implements MapReduceConsume<SafeCommandStore, Void>, Epoc
     @Override
     public MessageType type()
     {
-        switch (achieved.propagate())
+        // TODO (now): this logic doesn't work now we permit upgrading; need to pick the maximum *possible* status we might propagate
+        switch (achieved.propagatesStatus())
         {
             case Applied:
             case PreApplied:
-                if (toEpoch >= executeAt.epoch())
+                if (toEpoch >= committedExecuteAt.epoch())
                     return MessageType.PROPAGATE_APPLY_MSG;
             case Committed:
             case ReadyToExecute:
@@ -359,13 +499,14 @@ public class Propagate implements MapReduceConsume<SafeCommandStore, Void>, Epoc
     public String toString()
     {
         return "Propagate{type:" + type() +
-                ", txnId: " + txnId +
-                ", saveStatus: " + saveStatus +
-                ", deps: " + partialDeps +
-                ", txn: " + partialTxn +
-                ", executeAt: " + executeAt +
-                ", writes:" + writes +
-                ", result:" + result +
-                '}';
+               ", txnId: " + txnId +
+               ", saveStatus: " + maxKnowledgeSaveStatus +
+               ", deps: " + committedDeps +
+               ", txn: " + partialTxn +
+               ", executeAt: " + committedExecuteAt +
+               ", writes:" + writes +
+               ", result:" + result +
+               '}';
     }
+
 }
diff --git a/accord-core/src/main/java/accord/messages/ReadData.java b/accord-core/src/main/java/accord/messages/ReadData.java
index 889b7067..b2c73948 100644
--- a/accord-core/src/main/java/accord/messages/ReadData.java
+++ b/accord-core/src/main/java/accord/messages/ReadData.java
@@ -182,20 +182,19 @@ public abstract class ReadData extends AbstractEpochRequest<ReadNack>
         ack(unavailable);
     }
 
-    protected AsyncChain<Data> execute(SafeCommandStore safeStore, Timestamp executeAt, PartialTxn txn)
+    protected AsyncChain<Data> execute(SafeCommandStore safeStore, Timestamp executeAt, PartialTxn txn, Ranges unavailable)
     {
-        return txn.read(safeStore, executeAt);
+        return txn.read(safeStore, executeAt, unavailable);
     }
 
     void read(SafeCommandStore safeStore, Timestamp executeAt, PartialTxn txn)
     {
         CommandStore unsafeStore = safeStore.commandStore();
         Ranges unavailable = safeStore.ranges().unsafeToReadAt(executeAt);
-
-        execute(safeStore, executeAt, txn).begin((next, throwable) -> {
+        // TODO (required): do we need to check unavailable again on completion, or throughout execution?
+        execute(safeStore, executeAt, txn, unavailable).begin((next, throwable) -> {
             if (throwable != null)
             {
-                // TODO (expected, exceptions): should send exception to client, and consistency handle/propagate locally
                 logger.trace("{}: read failed for {}: {}", txnId, unsafeStore, throwable);
                 node.reply(replyTo, replyContext, null, throwable);
                 cancel();
diff --git a/accord-core/src/main/java/accord/primitives/AbstractKeys.java b/accord-core/src/main/java/accord/primitives/AbstractKeys.java
index 8f3fcddf..d14acea4 100644
--- a/accord-core/src/main/java/accord/primitives/AbstractKeys.java
+++ b/accord-core/src/main/java/accord/primitives/AbstractKeys.java
@@ -28,6 +28,7 @@ import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import accord.api.RoutingKey;
+import accord.utils.ArrayBuffers.ObjectBuffers;
 import accord.utils.IndexedFold;
 import accord.utils.IndexedFoldToLong;
 import accord.utils.Invariants;
@@ -183,11 +184,21 @@ public abstract class AbstractKeys<K extends RoutableKey> implements Iterable<K>
         return SortedArrays.sliceWithMultipleMatches(keys, ranges.ranges, factory, (k, r) -> -r.compareTo(k), Range::compareTo);
     }
 
-    protected K[] subtract(Ranges ranges, IntFunction<K[]> factory)
+    protected K[] subtract(AbstractRanges ranges, IntFunction<K[]> factory)
     {
         return SortedArrays.subtractWithMultipleMatches(keys, ranges.ranges, factory, (k, r) -> -r.compareTo(k), Range::compareTo);
     }
 
+    protected K[] intersect(AbstractKeys<K> that, ObjectBuffers<K> buffers)
+    {
+        return SortedArrays.linearIntersection(this.keys, that.keys, buffers);
+    }
+
+    protected K[] intersect(AbstractRanges ranges, ObjectBuffers<K> buffers)
+    {
+        return SortedArrays.intersectWithMultipleMatches(keys, keys.length, ranges.ranges, ranges.ranges.length, (k, r) -> -r.compareTo(k), buffers);
+    }
+
     public boolean any(Ranges ranges, Predicate<? super K> predicate)
     {
         return 1 == foldl(ranges, (key, p2, prev, index) -> predicate.test(key) ? 1 : 0, 0, 0, 1);
diff --git a/accord-core/src/main/java/accord/primitives/AbstractRanges.java b/accord-core/src/main/java/accord/primitives/AbstractRanges.java
index 751f79b9..35ecc569 100644
--- a/accord-core/src/main/java/accord/primitives/AbstractRanges.java
+++ b/accord-core/src/main/java/accord/primitives/AbstractRanges.java
@@ -157,6 +157,30 @@ public abstract class AbstractRanges implements Iterable<Range>, Routables<Range
         return indexOf(that) >= 0;
     }
 
+    public boolean intersects(Routable key)
+    {
+        switch (key.domain())
+        {
+            default: throw new AssertionError();
+            case Range: return intersects((Range)key);
+            case Key: return contains((RoutableKey) key);
+        }
+    }
+
+    public Ranges subtract(Unseekables<?> keysOrRanges)
+    {
+        if (keysOrRanges.domain() == Routable.Domain.Key)
+            keysOrRanges = ((AbstractUnseekableKeys)keysOrRanges).toRanges();
+        return subtract((AbstractRanges) keysOrRanges);
+    }
+
+    public Ranges intersect(Unseekables<?> keysOrRanges)
+    {
+        if (keysOrRanges.domain() == Routable.Domain.Key)
+            keysOrRanges = ((AbstractUnseekableKeys)keysOrRanges).toRanges();
+        return sliceMinimal((AbstractRanges)keysOrRanges, this, (AbstractRanges)keysOrRanges, (i1, i2, rs) -> i2.ranges == rs && i2 instanceof Ranges ? (Ranges)i2 : Ranges.ofSortedAndDeoverlapped(rs));
+    }
+
     // returns ri in low 32 bits, ki in top, or -1 if no match found
     @Override
     public final long findNextIntersection(int ri, AbstractKeys<?> keys, int ki)
@@ -193,6 +217,14 @@ public abstract class AbstractRanges implements Iterable<Range>, Routables<Range
      * Subtracts the given set of ranges from this
      */
     public Ranges subtract(Ranges that)
+    {
+        return subtract((AbstractRanges) that);
+    }
+
+    /**
+     * Subtracts the given set of ranges from this
+     */
+    private Ranges subtract(AbstractRanges that)
     {
         if (that.isEmpty())
             return this instanceof Ranges ? (Ranges)this : ofSortedAndDeoverlappedUnchecked(ranges);
@@ -272,7 +304,7 @@ public abstract class AbstractRanges implements Iterable<Range>, Routables<Range
             case Key:
             {
                 AbstractKeys<?> that = (AbstractKeys<?>) keysOrRanges;
-                Range[] result = SortedArrays.asymmetricLinearIntersectionWithOverlaps(input.ranges, input.ranges.length, that.keys, that.keys.length, Range::compareTo, cachedRanges());
+                Range[] result = SortedArrays.intersectWithMultipleMatches(input.ranges, input.ranges.length, that.keys, that.keys.length, Range::compareTo, cachedRanges());
                 return result == input.ranges ? input : constructor.construct(input, param, result);
             }
         }
@@ -296,7 +328,7 @@ public abstract class AbstractRanges implements Iterable<Range>, Routables<Range
 
     static <C extends AbstractRanges, P, O> O sliceOverlapping(C covering, AbstractRanges input, P param, SliceConstructor<? super C, P, O> constructor)
     {
-        Range[] result = SortedArrays.asymmetricLinearIntersectionWithOverlaps(input.ranges, input.ranges.length, covering.ranges, covering.ranges.length, Range::compareIntersecting, cachedRanges());
+        Range[] result = SortedArrays.intersectWithMultipleMatches(input.ranges, input.ranges.length, covering.ranges, covering.ranges.length, Range::compareIntersecting, cachedRanges());
         return constructor.construct(covering, param, result);
     }
 
diff --git a/accord-core/src/main/java/accord/primitives/AbstractUnseekableKeys.java b/accord-core/src/main/java/accord/primitives/AbstractUnseekableKeys.java
index 80903cd3..0b1d1b31 100644
--- a/accord-core/src/main/java/accord/primitives/AbstractUnseekableKeys.java
+++ b/accord-core/src/main/java/accord/primitives/AbstractUnseekableKeys.java
@@ -19,6 +19,8 @@
 package accord.primitives;
 
 import accord.api.RoutingKey;
+import accord.utils.ArrayBuffers;
+import accord.utils.SortedArrays;
 
 import java.util.Arrays;
 
@@ -38,9 +40,69 @@ implements Iterable<RoutingKey>, Unseekables<RoutingKey>, Participants<RoutingKe
     }
 
     @Override
-    public Unseekables<RoutingKey> subtract(Ranges ranges)
+    public final AbstractUnseekableKeys intersect(Unseekables<?> keysOrRanges)
+    {
+        switch (keysOrRanges.domain())
+        {
+            default: throw new AssertionError("Unhandled domain: " + keysOrRanges.domain());
+            case Key:
+            {
+                AbstractUnseekableKeys that = (AbstractUnseekableKeys) keysOrRanges;
+                return weakWrap(intersect(that, ArrayBuffers.cachedRoutingKeys()), that);
+            }
+            case Range:
+            {
+                AbstractRanges that = (AbstractRanges) keysOrRanges;
+                return wrap(intersect(that, ArrayBuffers.cachedRoutingKeys()));
+            }
+        }
+    }
+
+    @Override
+    public final Participants<RoutingKey> subtract(Unseekables<?> keysOrRanges)
+    {
+        switch (keysOrRanges.domain())
+        {
+            default: throw new AssertionError("Unhandled domain: " + keysOrRanges.domain());
+            case Key:
+            {
+                AbstractUnseekableKeys that = (AbstractUnseekableKeys) keysOrRanges;
+                return weakWrap(SortedArrays.linearSubtract(this.keys, that.keys, RoutingKey[]::new), that);
+            }
+            case Range:
+            {
+                return subtract((AbstractRanges)keysOrRanges);
+            }
+        }
+    }
+
+    @Override
+    public Participants<RoutingKey> subtract(Ranges ranges)
+    {
+        return subtract((AbstractRanges) ranges);
+    }
+
+    private Participants<RoutingKey> subtract(AbstractRanges ranges)
     {
         RoutingKey[] output = subtract(ranges, RoutingKey[]::new);
         return output == keys ? this : new RoutingKeys(output);
     }
+
+    public Ranges toRanges()
+    {
+        Range[] ranges = new Range[keys.length];
+        for (int i = 0 ; i < keys.length ; ++i)
+            ranges[i] = keys[i].asRange();
+        return Ranges.ofSortedAndDeoverlapped(ranges);
+    }
+
+    private AbstractUnseekableKeys weakWrap(RoutingKey[] wrap, AbstractUnseekableKeys that)
+    {
+        return wrap == keys ? this : wrap == that.keys ? that : new RoutingKeys(wrap);
+    }
+
+    private AbstractUnseekableKeys wrap(RoutingKey[] wrap)
+    {
+        return wrap == keys ? this : new RoutingKeys(wrap);
+    }
 }
diff --git a/accord-core/src/main/java/accord/primitives/Deps.java b/accord-core/src/main/java/accord/primitives/Deps.java
index 69476ed2..d8e134d0 100644
--- a/accord-core/src/main/java/accord/primitives/Deps.java
+++ b/accord-core/src/main/java/accord/primitives/Deps.java
@@ -109,6 +109,11 @@ public class Deps
         return keyDeps.contains(txnId) || rangeDeps.contains(txnId);
     }
 
+    public boolean intersects(TxnId txnId, Ranges ranges)
+    {
+        return keyDeps.intersects(txnId, ranges) || rangeDeps.intersects(txnId, ranges);
+    }
+
     public Deps with(Deps that)
     {
         return new Deps(this.keyDeps.with(that.keyDeps), this.rangeDeps.with(that.rangeDeps));
diff --git a/accord-core/src/main/java/accord/primitives/EpochSupplier.java b/accord-core/src/main/java/accord/primitives/EpochSupplier.java
index 2d43e7a3..d6ad196e 100644
--- a/accord-core/src/main/java/accord/primitives/EpochSupplier.java
+++ b/accord-core/src/main/java/accord/primitives/EpochSupplier.java
@@ -21,4 +21,9 @@ package accord.primitives;
 public interface EpochSupplier
 {
     long epoch();
+
+    static EpochSupplier constant(long epoch)
+    {
+        return () -> epoch;
+    }
 }
diff --git a/accord-core/src/main/java/accord/primitives/KeyDeps.java b/accord-core/src/main/java/accord/primitives/KeyDeps.java
index 3b92b941..dae6729c 100644
--- a/accord-core/src/main/java/accord/primitives/KeyDeps.java
+++ b/accord-core/src/main/java/accord/primitives/KeyDeps.java
@@ -237,6 +237,30 @@ public class KeyDeps implements Iterable<Map.Entry<Key, TxnId>>
         return Arrays.binarySearch(txnIds, txnId) >= 0;
     }
 
+    public boolean intersects(TxnId txnId, Ranges ranges)
+    {
+        int txnIdx = Arrays.binarySearch(txnIds, txnId);
+        if (txnIdx < 0)
+            return false;
+
+        int[] txnIdsToKeys = txnIdsToKeys();
+
+        int start = txnIdx == 0 ? txnIds.length : txnIdsToKeys[txnIdx - 1];
+        int end = txnIdsToKeys[txnIdx];
+        if (start == end)
+            return false;
+
+        int li = start, ri = 0;
+        while (li < end && ri < ranges.size())
+        {
+            ri = ranges.findNext(ri, keys.get(txnIdsToKeys[li]), FAST);
+            if (ri >= 0) return true;
+            ri = -1 - ri;
+            ++li;
+        }
+        return false;
+    }
+
     // return true iff we map any keys to any txnId
     // if the mapping is empty we return false, whether or not we have any keys or txnId by themselves
     public boolean isEmpty()
diff --git a/accord-core/src/main/java/accord/primitives/KeyRoute.java b/accord-core/src/main/java/accord/primitives/KeyRoute.java
index 4373f72c..6d413b81 100644
--- a/accord-core/src/main/java/accord/primitives/KeyRoute.java
+++ b/accord-core/src/main/java/accord/primitives/KeyRoute.java
@@ -96,8 +96,8 @@ public abstract class KeyRoute extends AbstractUnseekableKeys implements Route<R
     public Participants<RoutingKey> participants(Ranges ranges)
     {
         RoutingKey[] keys = slice(ranges, RoutingKey[]::new);
-        if (keys == this.keys && !isParticipatingHomeKey)
-            return this;
+        if (isParticipatingHomeKey)
+            return keys == this.keys ? this : new RoutingKeys(keys);
 
         int removePos = Arrays.binarySearch(keys, homeKey);
         if (removePos < 0)
@@ -110,6 +110,18 @@ public abstract class KeyRoute extends AbstractUnseekableKeys implements Route<R
         return new RoutingKeys(result);
     }
 
+    @Override
+    public Participants<RoutingKey> participants(Ranges ranges, Slice slice)
+    {
+        return participants(ranges);
+    }
+
+    public Ranges toRanges()
+    {
+        Invariants.checkState(isParticipatingHomeKey);
+        return super.toRanges();
+    }
+
     @Override
     public RoutingKey homeKey()
     {
diff --git a/accord-core/src/main/java/accord/primitives/Keys.java b/accord-core/src/main/java/accord/primitives/Keys.java
index ee358d66..74db63a4 100644
--- a/accord-core/src/main/java/accord/primitives/Keys.java
+++ b/accord-core/src/main/java/accord/primitives/Keys.java
@@ -84,6 +84,11 @@ public class Keys extends AbstractKeys<Key> implements Seekables<Key, Keys>
         return wrap(slice(ranges, Key[]::new));
     }
 
+    public final Keys intersect(Keys that)
+    {
+        return wrap(SortedArrays.linearIntersection(this.keys, that.keys, ArrayBuffers.cachedKeys()), that);
+    }
+
     public Keys with(Key key)
     {
         int insertPos = Arrays.binarySearch(keys, key);
@@ -218,6 +223,11 @@ public class Keys extends AbstractKeys<Key> implements Seekables<Key, Keys>
         return wrap == keys ? this : wrap == that.keys && that instanceof Keys ? (Keys)that : new Keys(wrap);
     }
 
+    private AbstractKeys<Key> weakWrap(Key[] wrap, AbstractKeys<Key> that)
+    {
+        return wrap == keys ? this : wrap == that.keys ? that : new Keys(wrap);
+    }
+
     private Keys wrap(Key[] wrap)
     {
         return wrap == keys ? this : new Keys(wrap);
diff --git a/accord-core/src/main/java/accord/primitives/PartialDeps.java b/accord-core/src/main/java/accord/primitives/PartialDeps.java
index f3770062..7a9b09f8 100644
--- a/accord-core/src/main/java/accord/primitives/PartialDeps.java
+++ b/accord-core/src/main/java/accord/primitives/PartialDeps.java
@@ -66,6 +66,13 @@ public class PartialDeps extends Deps
             && rangeDeps.txnIds().containsAll(that.rangeDeps.txnIds());
     }
 
+    public Deps with(Deps that)
+    {
+        if (that instanceof PartialDeps)
+            return with((PartialDeps) that);
+        return super.with(that);
+    }
+
     public PartialDeps with(PartialDeps that)
     {
         Invariants.checkArgument((this.rangeDeps == null) == (that.rangeDeps == null));
diff --git a/accord-core/src/main/java/accord/primitives/Participants.java b/accord-core/src/main/java/accord/primitives/Participants.java
index e7b6ed7b..1f84ae3b 100644
--- a/accord-core/src/main/java/accord/primitives/Participants.java
+++ b/accord-core/src/main/java/accord/primitives/Participants.java
@@ -26,11 +26,52 @@ package accord.primitives;
  */
 public interface Participants<K extends Unseekable> extends Unseekables<K>
 {
+    Participants<K> with(Participants<K> with);
+
+    @Override
+    Participants<K> intersect(Unseekables<?> with);
     @Override
     Participants<K> slice(Ranges ranges);
     @Override
     Participants<K> slice(Ranges ranges, Slice slice);
-    Participants<K> with(Participants<K> with);
+    @Override
+    Participants<K> subtract(Ranges ranges);
+    Participants<K> subtract(Unseekables<?> without);
+
+    Ranges toRanges();
+
+    static boolean isParticipants(Unseekables<?> unseekables)
+    {
+        switch (unseekables.kind())
+        {
+            default: throw new AssertionError("Unhandled Unseekables.Kind: " + unseekables.kind());
+            case FullKeyRoute:
+            case FullRangeRoute:
+            case PartialKeyRoute:
+            case PartialRangeRoute:
+                return Route.castToRoute(unseekables).isParticipatingHomeKey();
+            case RoutingRanges:
+            case RoutingKeys:
+                return true;
+        }
+    }
+
+    static Participants<?> participants(Unseekables<?> unseekables)
+    {
+        switch (unseekables.kind())
+        {
+            default: throw new AssertionError("Unhandled Unseekables.Kind: " + unseekables.kind());
+            case FullKeyRoute:
+            case FullRangeRoute:
+            case PartialKeyRoute:
+            case PartialRangeRoute:
+                return Route.castToRoute(unseekables).participants();
+            case RoutingRanges:
+                return (Ranges) unseekables;
+            case RoutingKeys:
+                return (RoutingKeys) unseekables;
+        }
+    }
 
     /**
      * If both left and right are a Route, invoke {@link Route#union} on them. Otherwise invoke {@link #with}.
diff --git a/accord-core/src/main/java/accord/primitives/ProgressToken.java b/accord-core/src/main/java/accord/primitives/ProgressToken.java
index d80ac22b..d1dd50a7 100644
--- a/accord-core/src/main/java/accord/primitives/ProgressToken.java
+++ b/accord-core/src/main/java/accord/primitives/ProgressToken.java
@@ -31,10 +31,10 @@ import javax.annotation.Nonnull;
 public class ProgressToken implements Comparable<ProgressToken>, Outcome
 {
     public static final ProgressToken NONE = new ProgressToken(Durability.NotDurable, Status.NotDefined, Ballot.ZERO, false);
-    public static final ProgressToken INVALIDATED = new ProgressToken(Durability.DurableOrInvalidated, Status.Invalidated, Ballot.ZERO, false);
+    public static final ProgressToken INVALIDATED = new ProgressToken(Durability.MajorityOrInvalidated, Status.Invalidated, Ballot.ZERO, false);
     public static final ProgressToken APPLIED = new ProgressToken(Durability.NotDurable, Status.PreApplied, Ballot.ZERO, false);
     public static final ProgressToken DURABLE = new ProgressToken(Durability.Majority, Status.PreApplied, Ballot.ZERO, false);
-    public static final ProgressToken TRUNCATED = new ProgressToken(Durability.DurableOrInvalidated, Status.Truncated, Ballot.ZERO, false);
+    public static final ProgressToken TRUNCATED = new ProgressToken(Durability.MajorityOrInvalidated, Status.Truncated, Ballot.ZERO, false);
 
     public final Durability durability;
     public final Status status;
diff --git a/accord-core/src/main/java/accord/primitives/Range.java b/accord-core/src/main/java/accord/primitives/Range.java
index 65d88bdb..fd091676 100644
--- a/accord-core/src/main/java/accord/primitives/Range.java
+++ b/accord-core/src/main/java/accord/primitives/Range.java
@@ -53,6 +53,20 @@ public abstract class Range implements Comparable<RoutableKey>, Unseekable, Seek
             return 0;
         }
 
+        @Override
+        public int compareStartTo(RoutableKey key)
+        {
+            int c = start().compareTo(key);
+            if (c == 0) c = 1;
+            return c;
+        }
+
+        @Override
+        public int compareEndTo(RoutableKey key)
+        {
+            return end().compareTo(key);
+        }
+
         @Override
         public boolean startInclusive()
         {
@@ -101,6 +115,20 @@ public abstract class Range implements Comparable<RoutableKey>, Unseekable, Seek
             return false;
         }
 
+        @Override
+        public int compareStartTo(RoutableKey key)
+        {
+            return start().compareTo(key);
+        }
+
+        @Override
+        public int compareEndTo(RoutableKey key)
+        {
+            int c = end().compareTo(key);
+            if (c == 0) c = -1;
+            return c;
+        }
+
         @Override
         public Range newRange(RoutingKey start, RoutingKey end)
         {
@@ -155,6 +183,22 @@ public abstract class Range implements Comparable<RoutableKey>, Unseekable, Seek
                 }
                 return 0;
             }
+
+            @Override
+            public int compareStartTo(RoutableKey key)
+            {
+                int c = start().compareTo(key);
+                if (c == 0 && !startInclusive) c = 1;
+                return c;
+            }
+
+            @Override
+            public int compareEndTo(RoutableKey key)
+            {
+                int c = end().compareTo(key);
+                if (c == 0 && !endInclusive) c = -1;
+                return c;
+            }
         };
     }
 
@@ -223,6 +267,22 @@ public abstract class Range implements Comparable<RoutableKey>, Unseekable, Seek
     @Override
     public abstract int compareTo(RoutableKey key);
 
+    /**
+     * Returns a negative integer, zero, or a positive integer as the provided key is greater than, equal to,
+     * or less than the start of this range. This comparison is informed by the inclusivity of the start, i.e.
+     * if the raw keys are equal but the start is exclusive, the start is considered to sort after the provided key,
+     * so that it falls outside of the range.
+     */
+    public abstract int compareStartTo(RoutableKey key);
+
+    /**
+     * Returns a negative integer, zero, or a positive integer as the provided key is greater than, contained by,
+     * or less than the end of this range. This comparison is informed by the inclusivity of the end, i.e.
+     * if the raw keys are equal but the end is exclusive, the end is considered to sort before the provided key,
+     * so that it falls outside of the range.
+     */
+    public abstract int compareEndTo(RoutableKey key);
+
     public boolean contains(RoutableKey key)
     {
         return compareTo(key) == 0;
diff --git a/accord-core/src/main/java/accord/primitives/RangeDeps.java b/accord-core/src/main/java/accord/primitives/RangeDeps.java
index 1deb644c..cbff7aa3 100644
--- a/accord-core/src/main/java/accord/primitives/RangeDeps.java
+++ b/accord-core/src/main/java/accord/primitives/RangeDeps.java
@@ -28,6 +28,7 @@ import net.nicoulaj.compilecommand.annotations.Inline;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import java.util.*;
+import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.Predicate;
@@ -35,12 +36,15 @@ import java.util.function.Predicate;
 import static accord.utils.ArrayBuffers.*;
 import static accord.utils.RelationMultiMap.*;
 import static accord.utils.SortedArrays.Search.CEIL;
+import static accord.utils.SortedArrays.Search.FAST;
 
 /**
  * <p>Maintains a lazily-constructed, bidirectional map between Range and TxnId.
  * <p>Ranges are stored sorted by start then end, and indexed by a secondary {@link SearchableRangeList} structure.
  * <p>The relationship between Range and TxnId is maintained via {@code int[]} utilising {@link RelationMultiMap}
  * functionality.
+ *
+ * TODO (expected): de-overlap ranges per txnId if possible cheaply, or else reduce use of partial ranges where possible
  */
 public class RangeDeps implements Iterable<Map.Entry<Range, TxnId>>
 {
@@ -123,98 +127,103 @@ public class RangeDeps implements Iterable<Map.Entry<Range, TxnId>>
     }
 
     @Inline
-    public <P1, P2, P3> int forEach(RoutableKey key, IndexedTriConsumer<P1, P2, P3> forEachScanOrCheckpoint, IndexedRangeTriConsumer<P1, P2, P3> forEachRange, P1 p1, P2 p2, P3 p3, int minIndex)
+    public <P1, P2, P3, P4> int forEach(RoutableKey key, IndexedQuadConsumer<P1, P2, P3, P4> forEachScanOrCheckpoint, IndexedRangeQuadConsumer<P1, P2, P3, P4> forEachRange, P1 p1, P2 p2, P3 p3, P4 p4, int minIndex)
     {
-        return ensureSearchable().forEach(key, forEachScanOrCheckpoint, forEachRange, p1, p2, p3, minIndex);
+        return ensureSearchable().forEach(key, forEachScanOrCheckpoint, forEachRange, p1, p2, p3, p4, minIndex);
     }
 
-    private int forEach(RoutableKey key, Consumer<TxnId> forEach, int minIndex, @Nullable BitSet visited)
+    private <P1> int forEach(RoutableKey key, BiConsumer<P1, TxnId> forEach, P1 p1, int minIndex, @Nullable BitSet visited)
     {
         return forEach(key, RangeDeps::visitTxnIdsForRangeIndex, RangeDeps::visitTxnIdsForRangeIndex,
-                this, forEach, visited, minIndex);
+                this, forEach, p1, visited, minIndex);
     }
 
     private <P> int forEach(RoutableKey key, IndexedConsumer<P> forEach, P param, int minIndex)
+    {
+        return forEach(key, IndexedConsumer::accept, forEach, param, minIndex);
+    }
+
+    private <P1, P2> int forEach(RoutableKey key, IndexedBiConsumer<P1, P2> forEach, P1 p1, P2 p2, int minIndex)
     {
         return forEach(key, RangeDeps::visitTxnIdxsForRangeIndex, RangeDeps::visitTxnIdxsForRangeIndex,
-                this, forEach, param, minIndex);
+                this, forEach, p1, p2, minIndex);
     }
 
     @Inline
-    public <P1, P2, P3> int forEach(Range range, IndexedTriConsumer<P1, P2, P3> forEachScanOrCheckpoint, IndexedRangeTriConsumer<P1, P2, P3> forEachRange, P1 p1, P2 p2, P3 p3, int minIndex)
+    public <P1, P2, P3, P4> int forEach(Range range, IndexedQuadConsumer<P1, P2, P3, P4> forEachScanOrCheckpoint, IndexedRangeQuadConsumer<P1, P2, P3, P4> forEachRange, P1 p1, P2 p2, P3 p3, P4 p4, int minIndex)
     {
-        return ensureSearchable().forEach(range, forEachScanOrCheckpoint, forEachRange, p1, p2, p3, minIndex);
+        return ensureSearchable().forEach(range, forEachScanOrCheckpoint, forEachRange, p1, p2, p3, p4, minIndex);
     }
 
     @Inline
-    public <P1, P2, P3> int forEach(RoutingKey start, RoutingKey end, IndexedTriConsumer<P1, P2, P3> forEachScanOrCheckpoint, IndexedRangeTriConsumer<P1, P2, P3> forEachRange, P1 p1, P2 p2, P3 p3, int minIndex)
+    public <P1, P2, P3, P4> int forEach(RoutingKey start, RoutingKey end, IndexedQuadConsumer<P1, P2, P3, P4> forEachScanOrCheckpoint, IndexedRangeQuadConsumer<P1, P2, P3, P4> forEachRange, P1 p1, P2 p2, P3 p3, P4 p4, int minIndex)
     {
-        return ensureSearchable().forEach(start, end, forEachScanOrCheckpoint, forEachRange, p1, p2, p3, minIndex);
+        return ensureSearchable().forEach(start, end, forEachScanOrCheckpoint, forEachRange, p1, p2, p3, p4, minIndex);
     }
 
-    private <P1, P2, P3> void forEach(Ranges ranges, IndexedTriConsumer<P1, P2, P3> forEachScanOrCheckpoint, IndexedRangeTriConsumer<P1, P2, P3> forEachRange, P1 p1, P2 p2, P3 p3)
+    private <P1, P2, P3, P4> void forEach(Ranges ranges, IndexedQuadConsumer<P1, P2, P3, P4> forEachScanOrCheckpoint, IndexedRangeQuadConsumer<P1, P2, P3, P4> forEachRange, P1 p1, P2 p2, P3 p3, P4 p4)
     {
         int minIndex = 0;
         for (int i = 0; i < ranges.size() ; ++i)
-            minIndex = forEach(ranges.get(i), forEachScanOrCheckpoint, forEachRange, p1, p2, p3, minIndex);
+            minIndex = forEach(ranges.get(i), forEachScanOrCheckpoint, forEachRange, p1, p2, p3, p4, minIndex);
     }
 
-    private int forEach(Range range, Consumer<TxnId> forEach, int minIndex, @Nullable BitSet visited)
+    private <P1> int forEach(Range range, BiConsumer<P1, TxnId> forEach, P1 p1, int minIndex, @Nullable BitSet visited)
     {
         return forEach(range, RangeDeps::visitTxnIdsForRangeIndex, RangeDeps::visitTxnIdsForRangeIndex,
-                this, forEach, visited, minIndex);
+                this, forEach, p1, visited, minIndex);
     }
 
-    public <P> int forEach(Range range, IndexedConsumer<P> forEach, P param, int minIndex)
+    public <P1, P2> int forEach(Range range, IndexedBiConsumer<P1, P2> forEach, P1 p1, P2 p2, int minIndex)
     {
         return forEach(range, RangeDeps::visitTxnIdxsForRangeIndex, RangeDeps::visitTxnIdxsForRangeIndex,
-                this, forEach, param, minIndex);
+                this, forEach, p1, p2, minIndex);
     }
 
-    public <P> int forEach(RoutingKey start, RoutingKey end, IndexedConsumer<P> forEach, P param, int minIndex)
+    public <P1, P2> int forEach(RoutingKey start, RoutingKey end, IndexedBiConsumer<P1, P2> forEach, P1 p1, P2 p2, int minIndex)
     {
         return forEach(start, end, RangeDeps::visitTxnIdxsForRangeIndex, RangeDeps::visitTxnIdxsForRangeIndex,
-                this, forEach, param, minIndex);
+                this, forEach, p1, p2, minIndex);
     }
 
-    private void visitTxnIdsForRangeIndex(Consumer<TxnId> forEach, @Nullable BitSet visited, int rangeIndex)
+    private <P1> void visitTxnIdsForRangeIndex(BiConsumer<P1, TxnId> forEach, P1 p1, @Nullable BitSet visited, int rangeIndex)
     {
         for (int i = startOffset(ranges, rangesToTxnIds, rangeIndex), end = endOffset(rangesToTxnIds, rangeIndex) ; i < end ; ++i)
-            visitTxnId(rangesToTxnIds[i], forEach, visited);
+            visitTxnId(rangesToTxnIds[i], forEach, p1, visited);
     }
 
-    private void visitTxnIdsForRangeIndex(Consumer<TxnId> forEach, @Nullable BitSet visited, int start, int end)
+    private <P1> void visitTxnIdsForRangeIndex(BiConsumer<P1, TxnId> forEach, P1 p1, @Nullable BitSet visited, int start, int end)
     {
         if (end == 0)
             return;
         for (int i = startOffset(ranges, rangesToTxnIds, start) ; i < endOffset(rangesToTxnIds, end - 1) ; ++i)
-            visitTxnId(rangesToTxnIds[i], forEach, visited);
+            visitTxnId(rangesToTxnIds[i], forEach, p1, visited);
     }
 
     // TODO (low priority, efficiency): ideally we would accept something like a BitHashSet or IntegerTrie
     //   as O(N) space needed for BitSet here (but with a very low constant multiplier)
-    private void visitTxnId(int txnIdx, Consumer<TxnId> forEach, @Nullable BitSet visited)
+    private <P1> void visitTxnId(int txnIdx, BiConsumer<P1, TxnId> forEach, P1 p1, @Nullable BitSet visited)
     {
         if (visited == null || !visited.get(txnIdx))
         {
             if (visited != null)
                 visited.set(txnIdx);
-            forEach.accept(txnIds[txnIdx]);
+            forEach.accept(p1, txnIds[txnIdx]);
         }
     }
 
-    private <P> void visitTxnIdxsForRangeIndex(IndexedConsumer<P> forEach, P param, int rangeIndex)
+    private <P1, P2> void visitTxnIdxsForRangeIndex(IndexedBiConsumer<P1, P2> forEach, P1 p1, P2 p2, int rangeIndex)
     {
         for (int i = startOffset(ranges, rangesToTxnIds, rangeIndex), end = endOffset(rangesToTxnIds, rangeIndex) ; i < end ; ++i)
-            forEach.accept(param, rangesToTxnIds[i]);
+            forEach.accept(p1, p2, rangesToTxnIds[i]);
     }
 
-    private <P> void visitTxnIdxsForRangeIndex(IndexedConsumer<P> forEach, P param, int start, int end)
+    private <P1, P2> void visitTxnIdxsForRangeIndex(IndexedBiConsumer<P1, P2> forEach, P1 p1, P2 p2, int start, int end)
     {
         if (end == 0)
             return;
         for (int i = startOffset(ranges, rangesToTxnIds, start) ; i < endOffset(rangesToTxnIds, end - 1) ; ++i)
-            forEach.accept(param, rangesToTxnIds[i]);
+            forEach.accept(p1, p2, rangesToTxnIds[i]);
     }
 
     /**
@@ -222,7 +231,15 @@ public class RangeDeps implements Iterable<Map.Entry<Range, TxnId>>
      */
     public void forEachUniqueTxnId(RoutableKey key, Consumer<TxnId> forEach)
     {
-        forEach(key, forEach, 0, new BitSet());
+        forEachUniqueTxnId(key, forEach, Consumer::accept);
+    }
+
+    /**
+     * Each matching TxnId will be provided precisely once
+     */
+    public <P1> void forEachUniqueTxnId(RoutableKey key, P1 p1, BiConsumer<P1, TxnId> forEach)
+    {
+        forEach(key, forEach, p1, 0, new BitSet());
     }
 
     /**
@@ -236,27 +253,35 @@ public class RangeDeps implements Iterable<Map.Entry<Range, TxnId>>
     /**
      * The same TxnId may be provided as a parameter multiple times
      */
-    public <P> void forEach(Range range, IndexedConsumer<P> forEach, P param)
+    public <P1, P2> void forEach(Range range, P1 p1, P2 p2, IndexedBiConsumer<P1, P2> forEach)
     {
-        forEach(range, forEach, param, 0);
+        forEach(range, forEach, p1, p2, 0);
     }
 
     /**
      * The same TxnId may be provided as a parameter multiple times
      */
-    public void forEach(AbstractRanges ranges, Consumer<TxnId> forEach)
+    public <P1> void forEach(AbstractRanges ranges, P1 p1, BiConsumer<P1, TxnId> forEach)
     {
         int minIndex = 0;
         for (int i = 0; i < ranges.size() ; ++i)
-            minIndex = forEach(ranges.get(i), forEach, minIndex, null);
+            minIndex = forEach(ranges.get(i), forEach, p1, minIndex, null);
+    }
+
+    /**
+     * The same TxnId may be provided as a parameter multiple times
+     */
+    public <P1> void forEach(Unseekables<?> unseekables, P1 p1, IndexedConsumer<P1> forEach)
+    {
+        forEach(unseekables, forEach, p1, IndexedConsumer::accept);
     }
 
     /**
      * The same TxnId may be provided as a parameter multiple times
      */
-    public <P> void forEach(Unseekables<?> unseekables, P param, IndexedConsumer<P> forEach)
+    public <P1, P2> void forEach(Unseekables<?> unseekables, P1 p1, P2 p2, IndexedBiConsumer<P1, P2> forEach)
     {
-        forEach(unseekables, null, 0, unseekables.size(), param, forEach);
+        forEach(unseekables, null, 0, unseekables.size(), p1, p2, forEach);
     }
 
     /**
@@ -264,16 +289,16 @@ public class RangeDeps implements Iterable<Map.Entry<Range, TxnId>>
      * @param slice is only useful in case unseekables is a Ranges, in which case we only visit the intersection of the slice and the ranges we walk
      *              for keys it is expected that the caller has already sliced in this manner
      */
-    public <P> void forEach(Unseekables<?> unseekables, @Nullable Range slice, int from, int to, P param, IndexedConsumer<P> forEach)
+    public <P1, P2> void forEach(Unseekables<?> unseekables, @Nullable Range slice, int from, int to, P1 p1, P2 p2, IndexedBiConsumer<P1, P2> forEach)
     {
         switch (unseekables.domain())
         {
             default: throw new AssertionError("Unknown domain: " + unseekables.domain());
             case Key:
-                forEach((AbstractKeys<?>) unseekables, from, to, param, forEach);
+                forEach((AbstractKeys<?>) unseekables, from, to, p1, p2, forEach);
                 break;
             case Range:
-                forEach((AbstractRanges) unseekables, slice, from, to, param, forEach);
+                forEach((AbstractRanges) unseekables, slice, from, to, p1, p2, forEach);
                 break;
         }
     }
@@ -281,33 +306,33 @@ public class RangeDeps implements Iterable<Map.Entry<Range, TxnId>>
     /**
      * The same TxnId may be provided as a parameter multiple times
      */
-    public <P> void forEach(AbstractKeys<?> keys, P param, IndexedConsumer<P> forEach)
+    public <P1, P2> void forEach(AbstractKeys<?> keys, P1 p1, P2 p2, IndexedBiConsumer<P1, P2> forEach)
     {
-        forEach(keys, 0, keys.size(), param, forEach);
+        forEach(keys, 0, keys.size(), p1, p2, forEach);
     }
 
     /**
      * The same TxnId may be provided as a parameter multiple times
      */
-    public <P> void forEach(AbstractRanges ranges, P param, IndexedConsumer<P> forEach)
+    public <P1, P2> void forEach(AbstractRanges ranges, P1 p1, P2 p2, IndexedBiConsumer<P1, P2> forEach)
     {
-        forEach(ranges, null, 0, ranges.size(), param, forEach);
+        forEach(ranges, null, 0, ranges.size(), p1, p2, forEach);
     }
 
     /**
      * The same TxnId may be provided as a parameter multiple times
      */
-    public <P> void forEach(AbstractKeys<?> keys, int from, int to, P param, IndexedConsumer<P> forEach)
+    public <P1, P2> void forEach(AbstractKeys<?> keys, int from, int to, P1 p1, P2 p2, IndexedBiConsumer<P1, P2> forEach)
     {
         int minIndex = 0;
         for (int i = from ; i < to ; ++i)
-            minIndex = forEach(keys.get(i), forEach, param, minIndex);
+            minIndex = forEach(keys.get(i), forEach, p1, p2, minIndex);
     }
 
     /**
      * The same TxnId may be provided as a parameter multiple times
      */
-    public <P> void forEach(AbstractRanges ranges, @Nullable Range slice, int from, int to, P param, IndexedConsumer<P> forEach)
+    public <P1, P2> void forEach(AbstractRanges ranges, @Nullable Range slice, int from, int to, P1 p1, P2 p2, IndexedBiConsumer<P1, P2> forEach)
     {
         int minIndex = 0;
         for (int i = from; i < to ; ++i)
@@ -322,7 +347,7 @@ public class RangeDeps implements Iterable<Map.Entry<Range, TxnId>>
                 if (end.compareTo(start) <= 0)
                     continue;
             }
-            minIndex = forEach(start, end, forEach, param, minIndex);
+            minIndex = forEach(start, end, forEach, p1, p2, minIndex);
         }
     }
 
@@ -331,7 +356,7 @@ public class RangeDeps implements Iterable<Map.Entry<Range, TxnId>>
      */
     public void forEachUniqueTxnId(Range range, Consumer<TxnId> forEach)
     {
-        forEach(range, forEach, 0, new BitSet());
+        forEach(range, Consumer::accept, forEach, 0, new BitSet());
     }
 
     /**
@@ -341,10 +366,21 @@ public class RangeDeps implements Iterable<Map.Entry<Range, TxnId>>
      * @param forEach function to call on each unique {@link TxnId}
      */
     public void forEachUniqueTxnId(AbstractRanges ranges, Consumer<TxnId> forEach)
+    {
+        forEachUniqueTxnId(ranges, forEach, Consumer::accept);
+    }
+
+    /**
+     * Each matching TxnId will be provided precisely once
+     *
+     * @param ranges to match on
+     * @param forEach function to call on each unique {@link TxnId}
+     */
+    public <P1> void forEachUniqueTxnId(AbstractRanges ranges, P1 p1, BiConsumer<P1, TxnId> forEach)
     {
         int minIndex = 0;
         for (int i = 0; i < ranges.size() ; ++i)
-            minIndex = forEach(ranges.get(i), forEach, minIndex, new BitSet());
+            minIndex = forEach(ranges.get(i), forEach, p1, minIndex, new BitSet());
     }
 
     // return true iff we map any ranges to any txnId
@@ -361,14 +397,19 @@ public class RangeDeps implements Iterable<Map.Entry<Range, TxnId>>
 
     public Ranges ranges(TxnId txnId)
     {
-        int txnIdIndex = Arrays.binarySearch(txnIds, txnId);
-        if (txnIdIndex < 0)
-            throw new IllegalArgumentException("Key not found");
+        int txnIdx = Arrays.binarySearch(txnIds, txnId);
+        if (txnIdx < 0)
+            return Ranges.EMPTY;
 
+        return ranges(txnIdx);
+    }
+
+    public Ranges ranges(int txnIdx)
+    {
         ensureTxnIdToRange();
 
-        int start = txnIdIndex == 0 ? txnIds.length : txnIdsToRanges[txnIdIndex - 1];
-        int end = txnIdsToRanges[txnIdIndex];
+        int start = txnIdx == 0 ? txnIds.length : txnIdsToRanges[txnIdx - 1];
+        int end = txnIdsToRanges[txnIdx];
         if (start == end)
             return Ranges.EMPTY;
 
@@ -391,6 +432,70 @@ public class RangeDeps implements Iterable<Map.Entry<Range, TxnId>>
         return Ranges.ofSorted(result);
     }
 
+    public boolean intersects(TxnId txnId, Ranges ranges)
+    {
+        int txnIdx = Arrays.binarySearch(txnIds, txnId);
+        if (txnIdx < 0)
+            return false;
+
+        return intersects(txnIdx, ranges);
+    }
+
+    public boolean intersects(int txnIdx, Ranges intersects)
+    {
+        ensureTxnIdToRange();
+
+        int start = txnIdx == 0 ? txnIds.length : txnIdsToRanges[txnIdx - 1];
+        int end = txnIdsToRanges[txnIdx];
+        if (start == end)
+            return false;
+
+        int li = start, ri = 0;
+        while (li < end && ri < intersects.size())
+        {
+            ri = intersects.findNext(ri, ranges[txnIdsToRanges[li]], FAST);
+            if (ri >= 0) return true;
+            ri = -1 - ri;
+            ++li;
+        }
+        return false;
+    }
+
+    public boolean intersects(TxnId txnId, RoutableKey key)
+    {
+        int txnIdx = Arrays.binarySearch(txnIds, txnId);
+        if (txnIdx < 0)
+            throw new IllegalArgumentException("Key not found");
+
+        return intersects(txnIdx, key);
+    }
+
+    public boolean intersects(int txnIdx, RoutableKey key)
+    {
+        ensureTxnIdToRange();
+
+        int start = txnIdx == 0 ? txnIds.length : txnIdsToRanges[txnIdx - 1];
+        int end = txnIdsToRanges[txnIdx];
+        for (int i = start ; i < end ; ++i)
+        {
+            int c = ranges[i].compareTo(key);
+            if (c == 0) return true;
+            if (c > 0) return false;
+        }
+        return false;
+    }
+
+    public <P1, V> V foldEachRange(int txnIdx, P1 p1, V accumulate, TriFunction<P1, Range, V, V> fold)
+    {
+        ensureTxnIdToRange();
+
+        int start = txnIdx == 0 ? txnIds.length : txnIdsToRanges[txnIdx - 1];
+        int end = txnIdsToRanges[txnIdx];
+        for (int i = start; i < end ; ++i)
+            accumulate = fold.apply(p1, ranges[txnIdsToRanges[i++]], accumulate);
+        return accumulate;
+    }
+
     void ensureTxnIdToRange()
     {
         if (txnIdsToRanges != null)
@@ -406,7 +511,7 @@ public class RangeDeps implements Iterable<Map.Entry<Range, TxnId>>
 
         try (RangeAndMapCollector collector = new RangeAndMapCollector(ensureSearchable().maxScanAndCheckpointMatches))
         {
-            forEach(select, collector, collector, ranges, rangesToTxnIds, null);
+            forEach(select, collector, collector, ranges, rangesToTxnIds, null, null);
 
             if (collector.rangesCount == 0)
                 return new RangeDeps(NO_RANGES, NO_TXNIDS, NO_INTS);
@@ -577,8 +682,8 @@ public class RangeDeps implements Iterable<Map.Entry<Range, TxnId>>
     }
 
     static class RangeCollector implements
-            IndexedRangeTriConsumer<Range[], int[], Object>,
-            IndexedTriConsumer<Range[], int[], Object>,
+            IndexedRangeQuadConsumer<Range[], int[], Object, Object>,
+            IndexedQuadConsumer<Range[], int[], Object, Object>,
             AutoCloseable
     {
         int[] oooBuffer;
@@ -592,13 +697,13 @@ public class RangeDeps implements Iterable<Map.Entry<Range, TxnId>>
         }
 
         @Override
-        public void accept(Range[] o, int[] o2, Object o3, int index)
+        public void accept(Range[] o, int[] o2, Object o3, Object o4, int index)
         {
             oooBuffer[oooCount++] = index;
         }
 
         @Override
-        public void accept(Range[] ranges, int[] rangesToTxnIds, Object o3, int fromIndex, int toIndex)
+        public void accept(Range[] ranges, int[] rangesToTxnIds, Object o3, Object o4, int fromIndex, int toIndex)
         {
             if (oooCount > 0)
             {
diff --git a/accord-core/src/main/java/accord/primitives/RangeRoute.java b/accord-core/src/main/java/accord/primitives/RangeRoute.java
index b3c905b2..39c41c56 100644
--- a/accord-core/src/main/java/accord/primitives/RangeRoute.java
+++ b/accord-core/src/main/java/accord/primitives/RangeRoute.java
@@ -118,13 +118,18 @@ public abstract class RangeRoute extends AbstractRanges implements Route<Range>,
 
         // TODO (desired): efficiency (lots of unnecessary allocations)
         // TODO (expected): this should return a PartialRangeRoute, but we need to remove Route.covering()
-        return ranges().subtract(Ranges.of(homeKey().asRange()));
+        return toRanges().subtract(Ranges.of(homeKey().asRange()));
     }
 
     @Override
     public Participants<Range> participants(Ranges slice)
     {
-        Range[] ranges = slice(slice, Overlapping, this, null, (i1, i2, rs) -> rs);
+        return participants(slice, Overlapping);
+    }
+
+    public Participants<Range> participants(Ranges slice, Slice kind)
+    {
+        Range[] ranges = slice(slice, kind, this, null, (i1, i2, rs) -> rs);
         if (ranges == this.ranges && isParticipatingHomeKey)
             return this;
 
@@ -137,7 +142,7 @@ public abstract class RangeRoute extends AbstractRanges implements Route<Range>,
         return result.subtract(Ranges.of(homeKey().asRange()));
     }
 
-    public Ranges ranges()
+    public Ranges toRanges()
     {
         return Ranges.ofSortedAndDeoverlapped(ranges);
     }
diff --git a/accord-core/src/main/java/accord/primitives/Ranges.java b/accord-core/src/main/java/accord/primitives/Ranges.java
index 87e66010..5cc07c80 100644
--- a/accord-core/src/main/java/accord/primitives/Ranges.java
+++ b/accord-core/src/main/java/accord/primitives/Ranges.java
@@ -144,6 +144,12 @@ public class Ranges extends AbstractRanges implements Iterable<Range>, Seekables
         return this;
     }
 
+    @Override
+    public Ranges toRanges()
+    {
+        return this;
+    }
+
     @Override
     public FullRangeRoute toRoute(RoutingKey homeKey)
     {
diff --git a/accord-core/src/main/java/accord/primitives/Routable.java b/accord-core/src/main/java/accord/primitives/Routable.java
index a4d5270f..1072df68 100644
--- a/accord-core/src/main/java/accord/primitives/Routable.java
+++ b/accord-core/src/main/java/accord/primitives/Routable.java
@@ -46,6 +46,11 @@ public interface Routable
         {
             return VALUES[ordinal];
         }
+
+        public char shortName()
+        {
+            return this == Key ? 'K' : 'R';
+        }
     }
 
     Domain domain();
diff --git a/accord-core/src/main/java/accord/primitives/Routables.java b/accord-core/src/main/java/accord/primitives/Routables.java
index b4460b21..64806423 100644
--- a/accord-core/src/main/java/accord/primitives/Routables.java
+++ b/accord-core/src/main/java/accord/primitives/Routables.java
@@ -58,6 +58,7 @@ public interface Routables<K extends Routable> extends Iterable<K>
             case Range: return intersects((AbstractRanges) routables);
         }
     }
+
     boolean contains(RoutableKey key);
     boolean containsAll(Routables<?> keysOrRanges);
 
diff --git a/accord-core/src/main/java/accord/primitives/Route.java b/accord-core/src/main/java/accord/primitives/Route.java
index 70fdb745..775c2b3b 100644
--- a/accord-core/src/main/java/accord/primitives/Route.java
+++ b/accord-core/src/main/java/accord/primitives/Route.java
@@ -27,7 +27,7 @@ public interface Route<K extends Unseekable> extends Unseekables<K>
     RoutingKey homeKey();
 
     /**
-     * @return true iff homeKey() is not involved in the transaction, only in its coordination (i.e. !txn.keys().contains(homeKey())
+     * @return true iff homeKey() is involved in the transaction, not only in its coordination (i.e. txn.keys().contains(homeKey())
      */
     boolean isParticipatingHomeKey();
     RoutingKey someParticipatingKey();
@@ -68,6 +68,11 @@ public interface Route<K extends Unseekable> extends Unseekables<K>
      */
     Participants<K> participants(Ranges ranges);
 
+    /**
+     * Return the unseekables excluding any coordination-only home key, that intersect the provided ranges
+     */
+    Participants<K> participants(Ranges ranges, Slice slice);
+
     default boolean hasParticipants()
     {
         return size() > (isParticipatingHomeKey() || !contains(homeKey()) ? 0 : 1);
@@ -87,7 +92,30 @@ public interface Route<K extends Unseekable> extends Unseekables<K>
 
         switch (unseekables.domain())
         {
-            default: throw new AssertionError();
+            default: throw new ClassCastException(unseekables + " is not a FullRoute");
+            case Key: return (FullKeyRoute) unseekables;
+            case Range: return (FullRangeRoute) unseekables;
+        }
+    }
+
+    // this method exists solely to circumvent JDK bug with testing and casting interfaces
+    static FullRoute<?> castToNonNullFullRoute(@Nullable Unseekables<?> unseekables)
+    {
+        FullRoute<?> route = tryCastToFullRoute(unseekables);
+        if (route == null)
+            throw new ClassCastException(unseekables + " is not a FullRoute");
+        return route;
+    }
+
+    // this method exists solely to circumvent JDK bug with testing and casting interfaces
+    static FullRoute<?> tryCastToFullRoute(@Nullable Unseekables<?> unseekables)
+    {
+        if (unseekables == null)
+            return null;
+
+        switch (unseekables.domain())
+        {
+            default: return null;
             case Key: return (FullKeyRoute) unseekables;
             case Range: return (FullRangeRoute) unseekables;
         }
diff --git a/accord-core/src/main/java/accord/primitives/Txn.java b/accord-core/src/main/java/accord/primitives/Txn.java
index 5a792918..85cb0cbe 100644
--- a/accord-core/src/main/java/accord/primitives/Txn.java
+++ b/accord-core/src/main/java/accord/primitives/Txn.java
@@ -32,6 +32,14 @@ import accord.api.Update;
 import accord.local.SafeCommandStore;
 import accord.utils.async.AsyncChain;
 import accord.utils.async.AsyncChains;
+import java.util.function.Predicate;
+
+import static accord.primitives.Txn.Kind.Kinds.Any;
+import static accord.primitives.Txn.Kind.Kinds.AnyGlobal;
+import static accord.primitives.Txn.Kind.Kinds.RorWs;
+import static accord.primitives.Txn.Kind.Kinds.SyncPoints;
+import static accord.primitives.Txn.Kind.Kinds.Ws;
+import static accord.primitives.Txn.Kind.Kinds.WsOrSyncPoint;
 
 public interface Txn
 {
@@ -93,6 +101,26 @@ public interface Txn
         LocalOnly
         ;
 
+        public enum Kinds implements Predicate<Kind>
+        {
+            Ws, RorWs, WsOrSyncPoint, SyncPoints, AnyGlobal, Any;
+
+            @Override
+            public boolean test(Kind kind)
+            {
+                switch (this)
+                {
+                    default: throw new AssertionError();
+                    case Any: return true;
+                    case AnyGlobal: return !kind.isLocal();
+                    case WsOrSyncPoint: return kind == Write || kind == Kind.SyncPoint || kind == ExclusiveSyncPoint;
+                    case SyncPoints: return kind == Kind.SyncPoint || kind == ExclusiveSyncPoint;
+                    case Ws: return kind == Write;
+                    case RorWs: return kind == Read || kind == Write;
+                }
+            }
+        }
+
         // in future: BlindWrite, Interactive?
 
         private static final Kind[] VALUES = Kind.values();
@@ -138,6 +166,43 @@ public interface Txn
         {
             return VALUES[ordinal];
         }
+
+        public Kinds witnesses()
+        {
+            switch (this)
+            {
+                default: throw new AssertionError();
+                case Read:
+                case NoOp:
+                    return Ws;
+                case Write:
+                    return RorWs;
+                case SyncPoint:
+                case ExclusiveSyncPoint:
+                    return AnyGlobal;
+            }
+        }
+
+        public Kinds witnessedBy()
+        {
+            switch (this)
+            {
+                default: throw new AssertionError();
+                case Read:
+                    return WsOrSyncPoint;
+                case Write:
+                    return AnyGlobal;
+                case SyncPoint:
+                case ExclusiveSyncPoint:
+                case NoOp:
+                    return SyncPoints;
+            }
+        }
+
+        public char shortName()
+        {
+            return toString().charAt(0);
+        }
     }
 
     class InMemory implements Txn
@@ -267,10 +332,10 @@ public interface Txn
         return new Writes(txnId, executeAt, update.keys(), update.apply(executeAt, data));
     }
 
-    default AsyncChain<Data> read(SafeCommandStore safeStore, Timestamp executeAt)
+    default AsyncChain<Data> read(SafeCommandStore safeStore, Timestamp executeAt, Ranges unavailable)
     {
-        Ranges ranges = safeStore.ranges().safeToReadAt(executeAt);
-        List<AsyncChain<Data>> chains = Routables.foldlMinimal(read().keys(), ranges, (key, accumulate, index) -> {
+        Ranges ranges = safeStore.ranges().allAt(executeAt).subtract(unavailable);
+        List<AsyncChain<Data>> chains = Routables.foldlMinimal(keys(), ranges, (key, accumulate, index) -> {
             AsyncChain<Data> result = read().read(key, safeStore, executeAt, safeStore.dataStore());
             accumulate.add(result);
             return accumulate;
diff --git a/accord-core/src/main/java/accord/primitives/TxnId.java b/accord-core/src/main/java/accord/primitives/TxnId.java
index dc2a0986..7e153970 100644
--- a/accord-core/src/main/java/accord/primitives/TxnId.java
+++ b/accord-core/src/main/java/accord/primitives/TxnId.java
@@ -18,12 +18,16 @@
 
 package accord.primitives;
 
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
 import accord.local.Node.Id;
 import accord.primitives.Routable.Domain;
 import accord.primitives.Txn.Kind;
 
 import static accord.primitives.Txn.Kind.Read;
 import static accord.primitives.Txn.Kind.Write;
+import static accord.utils.Invariants.illegalArgument;
 
 public class TxnId extends Timestamp
 {
@@ -111,6 +115,12 @@ public class TxnId extends Timestamp
         return merge(this, that, TxnId::fromBits);
     }
 
+    @Override
+    public String toString()
+    {
+        return "[" + epoch() + ',' + hlc() + ',' + flags() + '(' + domain().shortName() + rw().shortName() + ')' + ',' + node + ']';
+    }
+
     private static int flags(Kind rw, Domain domain)
     {
         return flags(rw) | flags(domain);
@@ -155,4 +165,13 @@ public class TxnId extends Timestamp
     {
         return new TxnId(epochMsb(epoch), 0, Id.NONE);
     }
+
+    private static final Pattern PARSE = Pattern.compile("\\[(?<epoch>[0-9]+),(?<hlc>[0-9]+),(?<flags>[0-9]+),(?<node>[0-9]+)]");
+    public static TxnId parse(String txnIdString)
+    {
+        Matcher m = PARSE.matcher(txnIdString);
+        if (!m.matches())
+            throw illegalArgument("Invalid TxnId string: " + txnIdString);
+        return fromValues(Long.parseLong(m.group("epoch")), Long.parseLong(m.group("hlc")), Integer.parseInt(m.group("flags")), new Id(Integer.parseInt(m.group("node"))));
+    }
 }
diff --git a/accord-core/src/main/java/accord/primitives/Unseekables.java b/accord-core/src/main/java/accord/primitives/Unseekables.java
index e373a4d9..51b80e7d 100644
--- a/accord-core/src/main/java/accord/primitives/Unseekables.java
+++ b/accord-core/src/main/java/accord/primitives/Unseekables.java
@@ -47,6 +47,8 @@ public interface Unseekables<K extends Unseekable> extends Iterable<K>, Routable
     Unseekables<K> slice(Ranges ranges, Slice slice);
 
     Unseekables<K> subtract(Ranges ranges);
+    Unseekables<K> subtract(Unseekables<?> subtract);
+    Unseekables<K> intersect(Unseekables<?> intersect);
 
     /**
      * Return an object containing any {@code K} present in either of the original collections.
diff --git a/accord-core/src/main/java/accord/utils/Invariants.java b/accord-core/src/main/java/accord/utils/Invariants.java
index cd5ba6bd..361c29a8 100644
--- a/accord-core/src/main/java/accord/utils/Invariants.java
+++ b/accord-core/src/main/java/accord/utils/Invariants.java
@@ -39,7 +39,7 @@ public class Invariants
         return DEBUG;
     }
 
-    private static void illegalState(String msg)
+    public static void illegalState(String msg)
     {
         throw new IllegalStateException(msg);
     }
@@ -49,7 +49,7 @@ public class Invariants
         illegalState(null);
     }
 
-    private static void illegalArgument(String msg)
+    public static RuntimeException illegalArgument(String msg)
     {
         throw new IllegalArgumentException(msg);
     }
diff --git a/accord-core/src/main/java/accord/utils/RandomSource.java b/accord-core/src/main/java/accord/utils/RandomSource.java
index 3d4861e5..8528acf8 100644
--- a/accord-core/src/main/java/accord/utils/RandomSource.java
+++ b/accord-core/src/main/java/accord/utils/RandomSource.java
@@ -18,16 +18,15 @@
 
 package accord.utils;
 
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.List;
-import java.util.NavigableSet;
 import java.util.Random;
-import java.util.Set;
-import java.util.stream.DoubleStream;
-import java.util.stream.IntStream;
-import java.util.stream.LongStream;
+import java.util.function.BooleanSupplier;
+import java.util.function.IntSupplier;
+import java.util.function.LongSupplier;
+import java.util.function.Supplier;
 
+import accord.utils.random.Picker;
+
+// TODO (expected): merge with C* RandomSource
 public interface RandomSource
 {
     static RandomSource wrap(Random random)
@@ -38,14 +37,40 @@ public interface RandomSource
     void nextBytes(byte[] bytes);
 
     boolean nextBoolean();
+    default BooleanSupplier uniformBools() { return this::nextBoolean; }
+    default BooleanSupplier biasedUniformBools(float chance) { return () -> decide(chance); }
+    default Supplier<BooleanSupplier> biasedUniformBoolsSupplier(float minChance)
+    {
+        return () -> {
+            float chance = minChance + (1 - minChance)*nextFloat();
+            return () -> decide(chance);
+        };
+    }
 
-    int nextInt();
+    /**
+     * Returns true with a probability of {@code chance}. This is logically the same as
+     * <pre>{@code nextFloat() < chance}</pre>
+     *
+     * @param chance cumulative probability in range [0..1]
+     */
+    default boolean decide(float chance)
+    {
+        return nextFloat() < chance;
+    }
 
-    default int nextInt(int maxExclusive)
+    /**
+     * Returns true with a probability of {@code chance}. This is logically the same as
+     * <pre>{@code nextDouble() < chance}</pre>
+     *
+     * @param chance cumulative probability in range [0..1]
+     */
+    default boolean decide(double chance)
     {
-        return nextInt(0, maxExclusive);
+        return nextDouble() < chance;
     }
 
+    int nextInt();
+    default int nextInt(int maxExclusive) { return nextInt(0, maxExclusive); }
     default int nextInt(int minInclusive, int maxExclusive)
     {
         // this is diff behavior than ThreadLocalRandom, which returns nextInt
@@ -74,29 +99,47 @@ public interface RandomSource
         }
         return result;
     }
-
-    default IntStream ints()
+    default int nextBiasedInt(int minInclusive, int median, int maxExclusive)
     {
-        return IntStream.generate(this::nextInt);
+        checkBiasedUniform(minInclusive, median, maxExclusive);
+
+        int range = Math.max(maxExclusive - median, median - minInclusive) * 2;
+        int next = nextInt(range) - range/2;
+        next += median;
+        return next >= median ? next <  maxExclusive ? next : nextInt(median, maxExclusive)
+                              : next >= minInclusive ? next : minInclusive == median ? median : nextInt(minInclusive, median);
     }
 
-    default IntStream ints(int maxExclusive)
+    default IntSupplier uniformInts(int minInclusive, int maxExclusive) { return () -> nextInt(minInclusive, maxExclusive); }
+    default IntSupplier biasedUniformInts(int minInclusive, int median, int maxExclusive)
     {
-        return IntStream.generate(() -> nextInt(maxExclusive));
+        checkBiasedUniform(minInclusive, median, maxExclusive);
+        return () -> nextBiasedInt(minInclusive, median, maxExclusive);
     }
-
-    default IntStream ints(int minInclusive, int maxExclusive)
+    default Supplier<IntSupplier> biasedUniformIntsSupplier(int absoluteMinInclusive, int absoluteMaxExclusive, int minMedian, int maxMedian, int minRange, int maxRange)
     {
-        return IntStream.generate(() -> nextInt(minInclusive, maxExclusive));
+        return biasedUniformIntsSupplier(absoluteMinInclusive, absoluteMaxExclusive, minMedian, (minMedian+maxMedian)/2, maxMedian, minRange, (minRange+maxRange)/2, maxRange);
     }
-
-    long nextLong();
-
-    default long nextLong(long maxExclusive)
+    default Supplier<IntSupplier> biasedUniformIntsSupplier(int absoluteMinInclusive, int absoluteMaxExclusive, int minMedian, int medianMedian, int maxMedian, int minRange, int medianRange, int maxRange)
     {
-        return nextLong(0, maxExclusive);
+        checkBiasedUniform(minMedian, medianMedian, maxMedian);
+        checkBiasedUniform(minRange, medianRange, maxRange);
+        if (minMedian < absoluteMinInclusive)
+            throw new IllegalArgumentException(String.format("absoluteMin (%s) should be less than or equal to minMedian (%s)", absoluteMinInclusive, minMedian));
+        if (maxMedian > absoluteMaxExclusive)
+            throw new IllegalArgumentException(String.format("absoluteMax (%s) should be greater than or equal to maxMedian (%s)", absoluteMaxExclusive, maxMedian));
+        if (minRange < 1)
+            throw new IllegalArgumentException(String.format("minRange (%s) should be greater than or equal to 1", minRange));
+        return () -> {
+            int median = nextBiasedInt(minMedian, medianMedian, maxMedian);
+            int minInclusive = Math.max(absoluteMinInclusive, median - nextBiasedInt(minRange, medianRange, maxRange)/2);
+            int maxExclusive = Math.min(absoluteMaxExclusive, median + (nextBiasedInt(minRange, medianRange, maxRange)+1)/2);
+            return biasedUniformInts(minInclusive, median, maxExclusive);
+        };
     }
 
+    long nextLong();
+    default long nextLong(long maxExclusive) { return nextLong(0, maxExclusive); }
     default long nextLong(long minInclusive, long maxExclusive)
     {
         // this is diff behavior than ThreadLocalRandom, which returns nextLong
@@ -125,31 +168,57 @@ public interface RandomSource
         }
         return result;
     }
-
-    default LongStream longs()
+    default long nextBiasedLong(long minInclusive, long median, long maxExclusive)
     {
-        return LongStream.generate(this::nextLong);
+        checkBiasedUniform(minInclusive, median, maxExclusive);
+
+        long range = Math.max(maxExclusive - median, median - minInclusive) * 2;
+        long next = nextLong(range) - range/2;
+        next += median;
+        return next >= median ? next <  maxExclusive ? next : nextLong(median, maxExclusive)
+                              : next >= minInclusive ? next : minInclusive == median ? median : nextLong(minInclusive, median);
     }
 
-    default LongStream longs(long maxExclusive)
+    default LongSupplier uniformLongs(long minInclusive, long maxExclusive) { return () -> nextLong(minInclusive, maxExclusive); }
+    default LongSupplier biasedUniformLongs(long minInclusive, long median, long maxExclusive)
+    {
+        checkBiasedUniform(minInclusive, median, maxExclusive);
+        return () -> nextBiasedLong(minInclusive, median, maxExclusive);
+    }
+    default Supplier<LongSupplier> biasedUniformLongsSupplier(long absoluteMinInclusive, long absoluteMaxExclusive, long minMedian, long maxMedian, long minRange, long maxRange)
+    {
+        return biasedUniformLongsSupplier(absoluteMinInclusive, absoluteMaxExclusive, minMedian, (minMedian+maxMedian)/2, maxRange, minRange, (minRange+maxRange)/2, maxRange);
+    }
+    default Supplier<LongSupplier> biasedUniformLongsSupplier(long absoluteMinInclusive, long absoluteMaxExclusive, long minMedian, long medianMedian, long maxMedian, long minRange, long medianRange, long maxRange)
     {
-        return LongStream.generate(() -> nextLong(maxExclusive));
+        checkBiasedUniform(minMedian, medianMedian, maxMedian);
+        checkBiasedUniform(minRange, medianRange, maxRange);
+        if (minMedian < absoluteMinInclusive)
+            throw new IllegalArgumentException(String.format("absoluteMin (%s) should be less than or equal to minMedian (%s)", absoluteMinInclusive, minMedian));
+        if (maxMedian > absoluteMaxExclusive)
+            throw new IllegalArgumentException(String.format("absoluteMax (%s) should be greater than or equal to maxMedian (%s)", absoluteMaxExclusive, maxMedian));
+        if (minRange < 1)
+            throw new IllegalArgumentException(String.format("minRange (%s) should be greater than or equal to 1", minRange));
+        return () -> {
+            long median = nextBiasedLong(minMedian, medianMedian, maxMedian);
+            long minInclusive = Math.max(absoluteMinInclusive, median - nextBiasedLong(minRange, medianRange, maxRange)/2);
+            long maxExclusive = Math.min(absoluteMaxExclusive, median + (1+nextBiasedLong(minRange, medianRange, maxRange))/2);
+            return biasedUniformLongs(minInclusive, median, maxExclusive);
+        };
     }
 
-    default LongStream longs(long minInclusive, long maxExclusive)
+    static void checkBiasedUniform(long minInclusive, long median, long maxExclusive)
     {
-        return LongStream.generate(() -> nextLong(minInclusive, maxExclusive));
+        if (minInclusive > median)
+            throw new IllegalArgumentException(String.format("Min (%s) should be equal to or less than median (%d).", minInclusive, median));
+        if (median >= maxExclusive)
+            throw new IllegalArgumentException(String.format("Median (%s) should be less than max (%d).", median, maxExclusive));
     }
 
     float nextFloat();
 
     double nextDouble();
-
-    default double nextDouble(double maxExclusive)
-    {
-        return nextDouble(0, maxExclusive);
-    }
-
+    default double nextDouble(double maxExclusive) { return nextDouble(0, maxExclusive); }
     default double nextDouble(double minInclusive, double maxExclusive)
     {
         if (minInclusive >= maxExclusive)
@@ -162,136 +231,15 @@ public interface RandomSource
         return result;
     }
 
-    default DoubleStream doubles()
-    {
-        return DoubleStream.generate(this::nextDouble);
-    }
-
-    default DoubleStream doubles(double maxExclusive)
-    {
-        return DoubleStream.generate(() -> nextDouble(maxExclusive));
-    }
-
-    default DoubleStream doubles(double minInclusive, double maxExclusive)
-    {
-        return DoubleStream.generate(() -> nextDouble(minInclusive, maxExclusive));
-    }
-
     double nextGaussian();
 
-    default int pickInt(int first, int second, int... rest)
-    {
-        int offset = nextInt(0, rest.length + 2);
-        switch (offset)
-        {
-            case 0:  return first;
-            case 1:  return second;
-            default: return rest[offset - 2];
-        }
-    }
-
-    default int pickInt(int[] array)
-    {
-        return pickInt(array, 0, array.length);
-    }
-
-    default int pickInt(int[] array, int offset, int length)
-    {
-        Invariants.checkIndexInBounds(array.length, offset, length);
-        if (length == 1)
-            return array[offset];
-        return array[nextInt(offset, offset + length)];
-    }
-
-    default long pickLong(long first, long second, long... rest)
-    {
-        int offset = nextInt(0, rest.length + 2);
-        switch (offset)
-        {
-            case 0:  return first;
-            case 1:  return second;
-            default: return rest[offset - 2];
-        }
-    }
-
-    default long pickLong(long[] array)
-    {
-        return pickLong(array, 0, array.length);
-    }
-
-    default long pickLong(long[] array, int offset, int length)
-    {
-        Invariants.checkIndexInBounds(array.length, offset, length);
-        if (length == 1)
-            return array[offset];
-        return array[nextInt(offset, offset + length)];
-    }
-
-    default <T extends Comparable<T>> T pick(Set<T> set)
-    {
-        List<T> values = new ArrayList<>(set);
-        // Non-ordered sets may have different iteration order on different environments, which would make a seed produce different histories!
-        // To avoid such a problem, make sure to apply a deterministic function (sort).
-        if (!(set instanceof NavigableSet))
-            values.sort(Comparator.naturalOrder());
-        return pick(values);
-    }
-
-    default <T> T pick(T first, T second, T... rest)
-    {
-        int offset = nextInt(0, rest.length + 2);
-        switch (offset)
-        {
-            case 0:  return first;
-            case 1:  return second;
-            default: return rest[offset - 2];
-        }
-    }
-
-    default <T> T pick(T[] array)
-    {
-        return array[nextInt(array.length)];
-    }
-
-    default <T> T pick(List<T> values)
-    {
-        return pick(values, 0, values.size());
-    }
-
-    default <T> T pick(List<T> values, int offset, int length)
-    {
-        Invariants.checkIndexInBounds(values.size(), offset, length);
-        if (length == 1)
-            return values.get(offset);
-        return values.get(nextInt(offset, offset + length));
-    }
+    default <T> Supplier<T> randomWeightedPicker(T[] objects) { return Picker.WeightedObjectPicker.randomWeighted(this, objects); }
+    default <T> Supplier<T> randomWeightedPicker(T[] objects, float[] bias) { return Picker.WeightedObjectPicker.randomWeighted(this, objects, bias); }
+    default <T> Supplier<T> weightedPicker(T[] objects, float[] proportionalWeights) { return Picker.WeightedObjectPicker.weighted(this, objects, proportionalWeights); }
 
     void setSeed(long seed);
-
     RandomSource fork();
 
-    /**
-     * Returns true with a probability of {@code chance}.  This logic is logically the same as
-     * <pre>{@code nextFloat() < chance}</pre>
-     *
-     * @param chance cumulative probability in range [0..1]
-     */
-    default boolean decide(float chance)
-    {
-        return nextFloat() < chance;
-    }
-
-    /**
-     * Returns true with a probability of {@code chance}.  This logic is logically the same as
-     * <pre>{@code nextDouble() < chance}</pre>
-     *
-     * @param chance cumulative probability in range [0..1]
-     */
-    default boolean decide(double chance)
-    {
-        return nextDouble() < chance;
-    }
-
     default long reset()
     {
         long seed = nextLong();
diff --git a/accord-core/src/main/java/accord/utils/ReducingIntervalMap.java b/accord-core/src/main/java/accord/utils/ReducingIntervalMap.java
index c4c0fab4..9c3184c6 100644
--- a/accord-core/src/main/java/accord/utils/ReducingIntervalMap.java
+++ b/accord-core/src/main/java/accord/utils/ReducingIntervalMap.java
@@ -30,11 +30,14 @@ import javax.annotation.Nullable;
 import com.google.common.annotations.VisibleForTesting;
 
 /**
- * Mostly copied/adapted from Cassandra's PaxosRepairHistory class
- *
  * Represents a map of ranges where precisely one value is bound to each point in the continuum of ranges,
  * and a simple function is sufficient to merge values inserted to overlapping ranges.
  *
+ * Copied/adapted from Cassandra's PaxosRepairHistory class, however has a major distinction: applies only to the
+ * covered ranges, i.e. the first start bound is the lower bound, and the last start bound is the upper bound,
+ * and everything else is considered unknown. This is in contrast to the C* version where every logical range has
+ * some associated information, with the first and last entries applying to everything either side of the start/end bound.
+ *
  * A simple sorted array of bounds is sufficient to represent the state and perform efficient lookups.
  *
  * TODO (desired): use a mutable b-tree instead
@@ -47,8 +50,8 @@ public class ReducingIntervalMap<K extends Comparable<? super K>, V>
     // for simplicity at construction, we permit this to be overridden by the first insertion
     final boolean inclusiveEnds;
     // starts is 1 longer than values, so that starts[0] == start of values[0]
-    final K[] starts;
-    final V[] values;
+    protected final K[] starts;
+    protected final V[] values;
 
     public ReducingIntervalMap()
     {
@@ -83,6 +86,35 @@ public class ReducingIntervalMap<K extends Comparable<? super K>, V>
         return result;
     }
 
+    public <V2> V2 foldl(BiFunction<V, V2, V2> reduce, V2 accumulator, Predicate<V2> terminate)
+    {
+        for (V value : values)
+        {
+            if (value != null)
+            {
+                accumulator = reduce.apply(value, accumulator);
+                if (terminate.test(accumulator))
+                    break;
+            }
+        }
+        return accumulator;
+    }
+
+    public <V2> V2 foldlWithBounds(QuadFunction<V, V2, K, K, V2> fold, V2 accumulator, Predicate<V2> terminate)
+    {
+        for (int i = 0 ; i < values.length ; ++i)
+        {
+            V value = values[i];
+            if (value != null)
+            {
+                accumulator = fold.apply(value, accumulator, starts[i], starts[i+1]);
+                if (terminate.test(accumulator))
+                    break;
+            }
+        }
+        return accumulator;
+    }
+
     public String toString()
     {
         return toString(v -> true);
@@ -329,6 +361,10 @@ public class ReducingIntervalMap<K extends Comparable<? super K>, V>
             return a;
         }
 
+        /**
+         * null is a valid value to represent no knowledge, and is the *expected* final value, representing
+         * the bound of our knowledge (any higher key will find no associated information)
+         */
         public void append(K start, @Nullable V value, BiFunction<V, V, V> reduce)
         {
             int tailIdx = starts.size() - 1;
diff --git a/accord-core/src/main/java/accord/utils/ReducingRangeMap.java b/accord-core/src/main/java/accord/utils/ReducingRangeMap.java
index a762fdc2..190a99e1 100644
--- a/accord-core/src/main/java/accord/utils/ReducingRangeMap.java
+++ b/accord-core/src/main/java/accord/utils/ReducingRangeMap.java
@@ -28,6 +28,11 @@ import static accord.utils.SortedArrays.exponentialSearch;
 
 public class ReducingRangeMap<V> extends ReducingIntervalMap<RoutingKey, V>
 {
+    public interface ReduceFunction<V, V2, P1, P2>
+    {
+        V2 apply(V v, V2 v2, P1 p1, P2 p2, int startMatchingInput, int endMatchingInput, int matchingStartBound);
+    }
+
     public static class SerializerSupport
     {
         public static <V> ReducingRangeMap<V> create(boolean inclusiveEnds, RoutingKey[] ends, V[] values)
@@ -41,9 +46,9 @@ public class ReducingRangeMap<V> extends ReducingIntervalMap<RoutingKey, V>
         super();
     }
 
-    protected ReducingRangeMap(boolean inclusiveEnds, RoutingKey[] ends, V[] values)
+    protected ReducingRangeMap(boolean inclusiveEnds, RoutingKey[] starts, V[] values)
     {
-        super(inclusiveEnds, ends, values);
+        super(inclusiveEnds, starts, values);
     }
 
     public V foldl(Routables<?> routables, BiFunction<V, V, V> fold, V accumulator)
@@ -56,6 +61,16 @@ public class ReducingRangeMap<V> extends ReducingIntervalMap<RoutingKey, V>
         return foldl(routables, (a, b, f, ignore) -> f.apply(a, b), accumulator, fold, null, terminate);
     }
 
+    public <V2> V2 foldlWithBounds(Routables<?> routables, QuadFunction<V, V2, RoutingKey, RoutingKey, V2> fold, V2 accumulator, Predicate<V2> terminate)
+    {
+        return foldl(routables, (a, b, f, self, i, j, k) -> f.apply(a, b, self.starts[k], self.starts[k+1]), accumulator, fold, this, terminate);
+    }
+
+    public <R extends Routable, V2> V2 foldlWithInputAndBounds(Routables<R> routables, IndexedRangeQuadFunction<V, V2, RoutingKey, RoutingKey, V2> fold, V2 accumulator, Predicate<V2> terminate)
+    {
+        return foldl(routables, (a, b, f, self, i, j, k) -> f.apply(a, b, self.starts[k], self.starts[k+1], i, j), accumulator, fold, this, terminate);
+    }
+
     public <V2, P1> V2 foldl(Routables<?> routables, TriFunction<V, V2, P1, V2> fold, V2 accumulator, P1 p1, Predicate<V2> terminate)
     {
         return foldl(routables, (a, b, f, p) -> f.apply(a, b, p), accumulator, fold, p1, terminate);
@@ -63,10 +78,35 @@ public class ReducingRangeMap<V> extends ReducingIntervalMap<RoutingKey, V>
 
     public <V2, P1, P2> V2 foldl(Routables<?> routables, QuadFunction<V, V2, P1, P2, V2> fold, V2 accumulator, P1 p1, P2 p2, Predicate<V2> terminate)
     {
-        return foldl(routables, (v, v2, param1, param2, i, j) -> fold.apply(v, v2, param1, param2), accumulator, p1, p2, terminate);
+        return foldl(routables, (v, v2, param1, param2, i, j, k) -> fold.apply(v, v2, param1, param2), accumulator, p1, p2, terminate);
     }
 
     public <V2, P1, P2> V2 foldl(Routables<?> routables, IndexedRangeQuadFunction<V, V2, P1, P2, V2> fold, V2 accumulator, P1 p1, P2 p2, Predicate<V2> terminate)
+    {
+        return foldl(routables, (v, v2, param1, param2, i, j, k) -> fold.apply(v, v2, param1, param2, i, j), accumulator, p1, p2, terminate);
+    }
+
+    public V foldlWithDefault(Routables<?> routables, BiFunction<V, V, V> fold, V defaultValue, V accumulator)
+    {
+        return foldlWithDefault(routables, (a, b, f, ignore) -> f.apply(a, b), defaultValue, accumulator, fold, null, ignore -> false);
+    }
+
+    public <V2> V2 foldlWithDefault(Routables<?> routables, BiFunction<V, V2, V2> fold, V defaultValue, V2 accumulator, Predicate<V2> terminate)
+    {
+        return foldlWithDefault(routables, (a, b, f, ignore) -> f.apply(a, b), defaultValue, accumulator, fold, null, terminate);
+    }
+
+    public <V2, P1> V2 foldlWithDefault(Routables<?> routables, TriFunction<V, V2, P1, V2> fold, V defaultValue, V2 accumulator, P1 p1, Predicate<V2> terminate)
+    {
+        return foldlWithDefault(routables, (a, b, f, p) -> f.apply(a, b, p), defaultValue, accumulator, fold, p1, terminate);
+    }
+
+    public <V2, P1, P2> V2 foldlWithDefault(Routables<?> routables, QuadFunction<V, V2, P1, P2, V2> fold, V defaultValue, V2 accumulator, P1 p1, P2 p2, Predicate<V2> terminate)
+    {
+        return foldlWithDefault(routables, (v, v2, param1, param2, i, j) -> fold.apply(v, v2, param1, param2), defaultValue, accumulator, p1, p2, terminate);
+    }
+
+    private <V2, P1, P2> V2 foldl(Routables<?> routables, ReduceFunction<V, V2, P1, P2> fold, V2 accumulator, P1 p1, P2 p2, Predicate<V2> terminate)
     {
         switch (routables.domain())
         {
@@ -76,7 +116,7 @@ public class ReducingRangeMap<V> extends ReducingIntervalMap<RoutingKey, V>
         }
     }
 
-    public <V2, P1, P2> V2 foldl(AbstractKeys<?> keys, IndexedRangeQuadFunction<V, V2, P1, P2, V2> fold, V2 accumulator, P1 p1, P2 p2, Predicate<V2> terminate)
+    private <V2, P1, P2> V2 foldl(AbstractKeys<?> keys, ReduceFunction<V, V2, P1, P2> fold, V2 accumulator, P1 p1, P2 p2, Predicate<V2> terminate)
     {
         if (values.length == 0)
             return accumulator;
@@ -100,7 +140,7 @@ public class ReducingRangeMap<V> extends ReducingIntervalMap<RoutingKey, V>
 
             if (j != nextj && values[i] != null)
             {
-                accumulator = fold.apply(values[i], accumulator, p1, p2, j, nextj);
+                accumulator = fold.apply(values[i], accumulator, p1, p2, j, nextj, i);
                 if (terminate.test(accumulator))
                     return accumulator;
             }
@@ -110,7 +150,7 @@ public class ReducingRangeMap<V> extends ReducingIntervalMap<RoutingKey, V>
         return accumulator;
     }
 
-    public <V2, P1, P2> V2 foldl(AbstractRanges ranges, IndexedRangeQuadFunction<V, V2, P1, P2, V2> fold, V2 accumulator, P1 p1, P2 p2, Predicate<V2> terminate)
+    private <V2, P1, P2> V2 foldl(AbstractRanges ranges, ReduceFunction<V, V2, P1, P2> fold, V2 accumulator, P1 p1, P2 p2, Predicate<V2> terminate)
     {
         if (values.length == 0)
             return accumulator;
@@ -144,7 +184,58 @@ public class ReducingRangeMap<V> extends ReducingIntervalMap<RoutingKey, V>
 
             if (toj > j && values[i] != null)
             {
-                accumulator = fold.apply(values[i], accumulator, p1, p2, j, toj);
+                accumulator = fold.apply(values[i], accumulator, p1, p2, j, toj, i);
+                if (terminate.test(accumulator))
+                    return accumulator;
+            }
+            ++i;
+            j = nextj;
+        }
+        return accumulator;
+    }
+
+    public <V2, P1, P2> V2 foldlWithDefault(Routables<?> routables, IndexedRangeQuadFunction<V, V2, P1, P2, V2> fold, V defaultValue, V2 accumulator, P1 p1, P2 p2, Predicate<V2> terminate)
+    {
+        switch (routables.domain())
+        {
+            default: throw new AssertionError("Unknown domain: " + routables.domain());
+            case Key: return foldlWithDefault((AbstractKeys<?>) routables, fold, defaultValue, accumulator, p1, p2, terminate);
+            case Range: return foldlWithDefault((AbstractRanges) routables, fold, defaultValue, accumulator, p1, p2, terminate);
+        }
+    }
+
+    private <V2, P1, P2> V2 foldlWithDefault(AbstractKeys<?> keys, IndexedRangeQuadFunction<V, V2, P1, P2, V2> fold, V defaultValue, V2 accumulator, P1 p1, P2 p2, Predicate<V2> terminate)
+    {
+        if (values.length == 0 || keys.isEmpty())
+            return fold.apply(defaultValue, accumulator, p1, p2, 0, keys.size());
+
+        int i = 0, j = keys.findNext(0, starts[0], FAST);
+        if (j < 0) j = -1 - j;
+        else if (inclusiveEnds) ++j;
+
+        if (j > 0)
+            accumulator = fold.apply(defaultValue, accumulator, p1, p2, 0, j);
+
+        while (j < keys.size())
+        {
+            i = exponentialSearch(starts, i, starts.length, keys.get(j));
+            if (i < 0) i = -2 - i;
+            else if (inclusiveEnds) --i;
+
+            if (i >= values.length)
+                return fold.apply(defaultValue, accumulator, p1, p2, j, keys.size());
+
+            int nextj = keys.findNext(j, starts[i + 1], FAST);
+            if (nextj < 0) nextj = -1 -nextj;
+            else if (inclusiveEnds) ++nextj;
+
+            if (j != nextj)
+            {
+                V value = values[i];
+                if (value == null)
+                    value = defaultValue;
+
+                accumulator = fold.apply(value, accumulator, p1, p2, j, nextj);
                 if (terminate.test(accumulator))
                     return accumulator;
             }
@@ -154,6 +245,58 @@ public class ReducingRangeMap<V> extends ReducingIntervalMap<RoutingKey, V>
         return accumulator;
     }
 
+    private <V2, P1, P2> V2 foldlWithDefault(AbstractRanges ranges, IndexedRangeQuadFunction<V, V2, P1, P2, V2> fold, V defaultValue, V2 accumulator, P1 p1, P2 p2, Predicate<V2> terminate)
+    {
+        if (values.length == 0 || ranges.isEmpty())
+            return fold.apply(defaultValue, accumulator, p1, p2, 0, ranges.size());
+
+        // TODO (desired): first searches should be binarySearch
+        int j = ranges.findNext(0, starts[0], FAST);
+        if (j < 0) j = -1 - j;
+        else if (inclusiveEnds && ranges.get(j).end().equals(starts[0])) ++j;
+
+        if (j > 0 || starts[0].compareTo(ranges.get(0).start()) > 0)
+            accumulator = fold.apply(defaultValue, accumulator, p1, p2, 0, j);
+
+        int i = 0;
+        while (j < ranges.size())
+        {
+            Range range = ranges.get(j);
+            RoutingKey start = range.start();
+            int nexti = exponentialSearch(starts, i, starts.length, start);
+            if (nexti < 0) i = Math.max(i, -2 - nexti);
+            else if (nexti > i && !inclusiveStarts()) i = nexti - 1;
+            else i = nexti;
+
+            if (i >= values.length)
+                return fold.apply(defaultValue, accumulator, p1, p2, j, ranges.size());
+
+            int toj, nextj = ranges.findNext(j, starts[i + 1], FAST);
+            if (nextj < 0) toj = nextj = -1 -nextj;
+            else
+            {
+                toj = nextj + 1;
+                if (inclusiveEnds && ranges.get(nextj).end().equals(starts[i + 1]))
+                    ++nextj;
+            }
+
+            if (toj > j)
+            {
+                V value = values[i];
+                if (value == null)
+                    value = defaultValue;
+
+                accumulator = fold.apply(value, accumulator, p1, p2, j, toj);
+                if (terminate.test(accumulator))
+                    return accumulator;
+            }
+            ++i;
+            j = nextj;
+        }
+
+        return accumulator;
+    }
+
     public static <V> ReducingRangeMap<V> create(Ranges ranges, V value)
     {
         if (value == null)
diff --git a/accord-core/src/main/java/accord/utils/SearchableRangeList.java b/accord-core/src/main/java/accord/utils/SearchableRangeList.java
index 06027bb7..22aeb3a1 100644
--- a/accord-core/src/main/java/accord/utils/SearchableRangeList.java
+++ b/accord-core/src/main/java/accord/utils/SearchableRangeList.java
@@ -145,12 +145,12 @@ public class SearchableRangeList
     }
 
     @Inline
-    public <P1, P2, P3> int forEach(Range range, IndexedTriConsumer<P1, P2, P3> forEachScanOrCheckpoint, IndexedRangeTriConsumer<P1, P2, P3> forEachRange, P1 p1, P2 p2, P3 p3, int minIndex)
+    public <P1, P2, P3, P4> int forEach(Range range, IndexedQuadConsumer<P1, P2, P3, P4> forEachScanOrCheckpoint, IndexedRangeQuadConsumer<P1, P2, P3, P4> forEachRange, P1 p1, P2 p2, P3 p3, P4 p4, int minIndex)
     {
-        return forEach(range.start(), range.end(), forEachScanOrCheckpoint, forEachRange, p1, p2, p3, minIndex);
+        return forEach(range.start(), range.end(), forEachScanOrCheckpoint, forEachRange, p1, p2, p3, p4, minIndex);
     }
 
-    public <P1, P2, P3> int forEach(RoutingKey startKey, RoutingKey endKey, IndexedTriConsumer<P1, P2, P3> forEachScanOrCheckpoint, IndexedRangeTriConsumer<P1, P2, P3> forEachRange, P1 p1, P2 p2, P3 p3, int minIndex)
+    public <P1, P2, P3, P4> int forEach(RoutingKey startKey, RoutingKey endKey, IndexedQuadConsumer<P1, P2, P3, P4> forEachScanOrCheckpoint, IndexedRangeQuadConsumer<P1, P2, P3, P4> forEachRange, P1 p1, P2 p2, P3 p3, P4 p4, int minIndex)
     {
         if (ranges.length == 0 || minIndex == ranges.length)
             return minIndex;
@@ -175,20 +175,21 @@ public class SearchableRangeList
                 ++start;
         }
 
-        return forEach(start, end, floor, startKey, forEachScanOrCheckpoint, forEachRange, p1, p2, p3, minIndex);
+        // Since endInclusive() != startInclusive(), so no need to adjust start/end comparisons
+        return forEach(start, end, floor, startKey, 0, forEachScanOrCheckpoint, forEachRange, p1, p2, p3, p4, minIndex);
     }
 
     @Inline
-    public <P1, P2, P3> int forEach(RoutableKey key, IndexedTriConsumer<P1, P2, P3> forEachScanOrCheckpoint, IndexedRangeTriConsumer<P1, P2, P3> forEachRange, P1 p1, P2 p2, P3 p3, int minIndex)
+    public <P1, P2, P3, P4> int forEach(RoutableKey key, IndexedQuadConsumer<P1, P2, P3, P4> forEachScanOrCheckpoint, IndexedRangeQuadConsumer<P1, P2, P3, P4> forEachRange, P1 p1, P2 p2, P3 p3, P4 p4, int minIndex)
     {
         if (ranges.length == 0 || minIndex == ranges.length)
             return minIndex;
 
-        int end = SortedArrays.binarySearch(ranges, minIndex, ranges.length, key, (a, b) -> a.compareTo(b.start()), FLOOR);
+        int end = SortedArrays.binarySearch(ranges, minIndex, ranges.length, key, (a, b) -> -b.compareStartTo(a), FLOOR);
         if (end < 0) end = -1 - end;
         if (end <= minIndex) return minIndex;
 
-        int floor = SortedArrays.binarySearch(ranges, minIndex, ranges.length, key, (a, b) -> a.compareTo(b.start()), CEIL);
+        int floor = SortedArrays.binarySearch(ranges, minIndex, ranges.length, key, (a, b) -> -b.compareStartTo(a), CEIL);
         int start = floor;
         if (floor < 0)
         {
@@ -200,18 +201,18 @@ public class SearchableRangeList
             start = floor = -2 - floor;
             if (start < 0)
                 start = floor = 0;
-            else if (ranges[start].end().compareTo(key) <= 0)
+            else if (ranges[start].compareEndTo(key) < 0)
                 ++start;
         }
 
-        // Key cannot equal RoutingKey, so can ignore inclusivity/exclusivity of end bound
-        return forEach(start, end, floor, key, forEachScanOrCheckpoint, forEachRange, p1, p2, p3, minIndex);
+        int bound = ranges[0].endInclusive() ? -1 : 0;
+        return forEach(start, end, floor, key, bound, forEachScanOrCheckpoint, forEachRange, p1, p2, p3, p4, minIndex);
     }
 
     @Inline
-    private <P1, P2, P3> int forEach(int start, int end, int floor, RoutableKey startBound,
-                                     IndexedTriConsumer<P1, P2, P3> forEachScanOrCheckpoint, IndexedRangeTriConsumer<P1, P2, P3> forEachRange,
-                                     P1 p1, P2 p2, P3 p3, int minIndex)
+    private <P1, P2, P3, P4> int forEach(int start, int end, int floor, RoutableKey startBound, int cmpStartBoundWithEnd,
+                                     IndexedQuadConsumer<P1, P2, P3, P4> forEachScanOrCheckpoint, IndexedRangeQuadConsumer<P1, P2, P3, P4> forEachRange,
+                                     P1 p1, P2 p2, P3 p3, P4 p4, int minIndex)
     {
         if (start < minIndex) start = minIndex;
 
@@ -267,11 +268,11 @@ public class SearchableRangeList
                     if (ri < 0)
                         continue;
 
-                    if (ranges[ri].end().compareTo(startBound) <= 0)
+                    if (ranges[ri].end().compareTo(startBound) <= cmpStartBoundWithEnd)
                         break;
 
                     if (ri >= minIndex && ri < minScanIndex)
-                        forEachScanOrCheckpoint.accept(p1, p2, p3, ri);
+                        forEachScanOrCheckpoint.accept(p1, p2, p3, p4, ri);
                 }
             }
             else
@@ -279,21 +280,21 @@ public class SearchableRangeList
                 // if startBound is key, we cannot be equal to it;
                 // if startBound is a Range start, we also cannot be equal to it due to the requirement that
                 // endInclusive() != startInclusive(), so equality really means inequality
-                if (ranges[ri].end().compareTo(startBound) <= 0)
+                if (ranges[ri].end().compareTo(startBound) <= cmpStartBoundWithEnd)
                     break;
 
                 if (ri >= minIndex && ri < minScanIndex)
-                    forEachScanOrCheckpoint.accept(p1, p2, p3, ri);
+                    forEachScanOrCheckpoint.accept(p1, p2, p3, p4, ri);
             }
         }
 
         for (int i = minScanIndex; i < floor ; ++i)
         {
-            if (ranges[i].end().compareTo(startBound) > 0)
-                forEachScanOrCheckpoint.accept(p1, p2, p3, i);
+            if (ranges[i].end().compareTo(startBound) > cmpStartBoundWithEnd)
+                forEachScanOrCheckpoint.accept(p1, p2, p3, p4, i);
         }
 
-        forEachRange.accept(p1, p2, p3, start, end);
+        forEachRange.accept(p1, p2, p3, p4, start, end);
         return end;
     }
 
diff --git a/accord-core/src/main/java/accord/utils/SimpleBitSet.java b/accord-core/src/main/java/accord/utils/SimpleBitSet.java
index bff0b5c1..133a1f53 100644
--- a/accord-core/src/main/java/accord/utils/SimpleBitSet.java
+++ b/accord-core/src/main/java/accord/utils/SimpleBitSet.java
@@ -401,4 +401,10 @@ public class SimpleBitSet
     {
         return Arrays.equals(this.bits, other.bits);
     }
+
+    public void clear()
+    {
+        count = 0;
+        Arrays.fill(bits, 0L);
+    }
 }
diff --git a/accord-core/src/main/java/accord/utils/SortedArrays.java b/accord-core/src/main/java/accord/utils/SortedArrays.java
index 7602484c..255bc89e 100644
--- a/accord-core/src/main/java/accord/utils/SortedArrays.java
+++ b/accord-core/src/main/java/accord/utils/SortedArrays.java
@@ -403,15 +403,15 @@ public class SortedArrays
     /**
      * A linear intersection where we only want results from the left inputs, and the right inputs may either be a different type or otherwise only used for filtering
      */
-    public static <T1, T2> T1[] asymmetricLinearIntersectionWithOverlaps(T1[] left, int leftLength, T2[] right, int rightLength, AsymmetricComparator<? super T1, ? super T2> comparator, ObjectBuffers<T1> buffers)
+    public static <T1, T2> T1[] intersectWithMultipleMatches(T1[] left, int leftLength, T2[] right, int rightLength, AsymmetricComparator<? super T1, ? super T2> comparator, ObjectBuffers<T1> buffers)
     {
-        return (T1[])internalLinearIntersectionWithOverlaps(true, left, leftLength, right, rightLength, comparator, buffers, null);
+        return (T1[]) internalLinearIntersectionWithMultipleMatches(true, left, leftLength, right, rightLength, comparator, buffers, null);
     }
 
     /**
      * A linear intersection where we only want results from the left inputs, and the right inputs may either be a different type or otherwise only used for filtering
      */
-    private static <T1, T2> Object[] internalLinearIntersectionWithOverlaps(boolean preferLeft, T1[] left, int leftLength, T2[] right, int rightLength, AsymmetricComparator<? super T1, ? super T2> comparator, ObjectBuffers<T1> leftBuffers, @Nullable ObjectBuffers<T2> rightBuffers)
+    private static <T1, T2> Object[] internalLinearIntersectionWithMultipleMatches(boolean preferLeft, T1[] left, int leftLength, T2[] right, int rightLength, AsymmetricComparator<? super T1, ? super T2> comparator, ObjectBuffers<T1> leftBuffers, @Nullable ObjectBuffers<T2> rightBuffers)
     {
         int leftIdx = 0;
         int rightIdx = 0;
@@ -587,6 +587,8 @@ public class SortedArrays
     /**
      * Given two sorted arrays, return the elements present only in the first, preferentially returning the first array
      * itself if possible
+     *
+     * TODO (expected): use cachedBuffers
      */
     @SuppressWarnings("unused") // was used until recently, might be used again?
     public static <T extends Comparable<? super T>> T[] linearSubtract(T[] left, T[] right, IntFunction<T[]> allocate)
@@ -657,6 +659,8 @@ public class SortedArrays
      * Given two sorted arrays {@code slice} and {@code select}, where each array's contents is unique and non-overlapping
      * with itself, but may match multiple entries in the other array, return a new array containing the elements of {@code slice}
      * that match elements of {@code select} as per the provided comparators.
+     *
+     * TODO (expected): use buffers rather than factory
      */
     public static <A, R> A[] sliceWithMultipleMatches(A[] input, R[] select, IntFunction<A[]> factory, AsymmetricComparator<A, R> cmp1, AsymmetricComparator<R, A> cmp2)
     {
@@ -721,6 +725,8 @@ public class SortedArrays
      * Given two sorted arrays {@code slice} and {@code select}, where each array's contents is unique and non-overlapping
      * with itself, but may match multiple entries in the other array, return a new array containing the elements of {@code slice}
      * that match elements of {@code select} as per the provided comparators.
+     *
+     * TODO (expected): use buffers rather than factory
      */
     public static <A, R> A[] subtractWithMultipleMatches(A[] input, R[] subtract, IntFunction<A[]> factory, AsymmetricComparator<A, R> cmp1, AsymmetricComparator<R, A> cmp2)
     {
diff --git a/accord-core/src/main/java/accord/utils/random/Picker.java b/accord-core/src/main/java/accord/utils/random/Picker.java
new file mode 100644
index 00000000..f6f07d75
--- /dev/null
+++ b/accord-core/src/main/java/accord/utils/random/Picker.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package accord.utils.random;
+
+import java.util.Arrays;
+import java.util.function.Supplier;
+
+import accord.utils.Invariants;
+import accord.utils.RandomSource;
+
+public class Picker
+{
+    static abstract class Weighted
+    {
+        final RandomSource random;
+        final float[] weights;
+
+        public Weighted(RandomSource random, int length)
+        {
+            this(random, randomWeights(random, length));
+        }
+
+        public Weighted(RandomSource random, float[] weights)
+        {
+            this.random = random;
+            this.weights = weights;
+        }
+
+        static float[] randomWeights(RandomSource random, int length)
+        {
+            float[] weights = new float[length - 1];
+            float sum = 0;
+            for (int i = 0 ; i < weights.length ; ++i)
+                weights[i] = sum += random.nextFloat();
+            sum += random.nextFloat();
+            for (int i = 0 ; i < weights.length ; ++i)
+                weights[i] /= sum;
+            return weights;
+        }
+
+        static float[] randomWeights(RandomSource random, float[] bias)
+        {
+            float[] weights = new float[bias.length - 1];
+            float sum = 0;
+            for (int i = 0 ; i < weights.length ; ++i)
+                weights[i] = sum += random.nextFloat() * bias[i];
+            sum += random.nextFloat() * bias[weights.length];
+            for (int i = 0 ; i < weights.length ; ++i)
+                weights[i] /= sum;
+            return weights;
+        }
+
+        static float[] normaliseWeights(float[] input)
+        {
+            float[] output = new float[input.length - 1];
+            float sum = 0;
+            for (int i = 0 ; i < output.length ; ++i)
+                output[i] = sum += input[i];
+            sum += input[output.length];
+            for (int i = 0 ; i < output.length ; ++i)
+                output[i] /= sum;
+            return output;
+        }
+
+        int pickIndex()
+        {
+            int i = Arrays.binarySearch(weights, random.nextFloat());
+            if (i < 0) i = -1 - i;
+            return i;
+        }
+    }
+
+    public static class WeightedObjectPicker<T> extends Weighted implements Supplier<T>
+    {
+        final T[] values;
+
+        private WeightedObjectPicker(RandomSource random, T[] values, float[] weights)
+        {
+            super(random, values.length);
+            this.values = values;
+        }
+
+        @Override
+        public T get()
+        {
+            return values[pickIndex()];
+        }
+
+        public static <T> WeightedObjectPicker<T> randomWeighted(RandomSource random, T[] values)
+        {
+            return new WeightedObjectPicker<>(random, values, randomWeights(random, values.length));
+        }
+
+        public static <T> WeightedObjectPicker<T> randomWeighted(RandomSource random, T[] values, float[] bias)
+        {
+            Invariants.checkArgument(values.length == bias.length);
+            return new WeightedObjectPicker<>(random, values, randomWeights(random, bias));
+        }
+
+        public static <T> WeightedObjectPicker<T> weighted(RandomSource random, T[] values, float[] proportionalWeights)
+        {
+            Invariants.checkArgument(values.length == proportionalWeights.length);
+            return new WeightedObjectPicker<>(random, values, normaliseWeights(proportionalWeights));
+        }
+    }
+}
diff --git a/accord-core/src/main/java/accord/local/CommandListener.java b/accord-core/src/main/java/accord/utils/random/Suppliers.java
similarity index 51%
rename from accord-core/src/main/java/accord/local/CommandListener.java
rename to accord-core/src/main/java/accord/utils/random/Suppliers.java
index cb7a782a..e353308c 100644
--- a/accord-core/src/main/java/accord/local/CommandListener.java
+++ b/accord-core/src/main/java/accord/utils/random/Suppliers.java
@@ -16,24 +16,40 @@
  * limitations under the License.
  */
 
-package accord.local;
+package accord.utils.random;
 
-import accord.primitives.TxnId;
+import java.util.function.BooleanSupplier;
+import java.util.function.IntSupplier;
+import java.util.function.LongSupplier;
+import java.util.function.Supplier;
 
-public interface CommandListener
+public class Suppliers
 {
-    void onChange(SafeCommandStore safeStore, SafeCommand safeCommand);
+    public interface Bools extends BooleanSupplier
+    {
+        boolean get();
+        void cycle();
+        default boolean getAsBoolean() { return get(); }
+    }
+
+    public interface Ints extends IntSupplier
+    {
+        int get();
+        void cycle();
+        default int getAsInt() { return get(); }
+    }
 
-    /**
-     * Scope needed to run onChange
-     */
-    PreLoadContext listenerPreLoadContext(TxnId caller);
+    public interface Longs extends LongSupplier
+    {
+        long get();
+        void cycle();
+        default long getAsLong() { return get(); }
+    }
 
-    /**
-     * Indicates the command to listener relationship doesn't need to persist across restarts
-     */
-    default boolean isTransient()
+    public interface Objects<T> extends Supplier<T>
     {
-        return false;
+        T get();
+        void cycle();
     }
+
 }
diff --git a/accord-core/src/test/java/accord/burn/BurnTest.java b/accord-core/src/test/java/accord/burn/BurnTest.java
index 721cf030..8e45b464 100644
--- a/accord-core/src/test/java/accord/burn/BurnTest.java
+++ b/accord-core/src/test/java/accord/burn/BurnTest.java
@@ -34,6 +34,7 @@ import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.LongSupplier;
@@ -74,6 +75,7 @@ import accord.messages.MessageType;
 import accord.primitives.Keys;
 import accord.primitives.Range;
 import accord.primitives.Ranges;
+import accord.primitives.Timestamp;
 import accord.primitives.Txn;
 import accord.utils.DefaultRandom;
 import accord.utils.RandomSource;
@@ -116,8 +118,6 @@ public class BurnTest
                 ListQuery query = new ListQuery(client, count);
                 ListRequest request = new ListRequest(new Txn.InMemory(ranges, read, query, null), listener);
                 packets.add(new Packet(client, node, count, request));
-
-
             }
             else
             {
@@ -200,24 +200,25 @@ public class BurnTest
         RandomDelayQueue delayQueue = new Factory(random).get();
         PropagatingPendingQueue queue = new PropagatingPendingQueue(failures, delayQueue);
         RandomSource retryRandom = random.fork();
-        ListAgent agent = new ListAgent(1000L, failures::add, retry -> {
+        Function<BiConsumer<Timestamp, Ranges>, ListAgent> agentSupplier = onStale -> new ListAgent(1000L, failures::add, retry -> {
             long delay = retryRandom.nextInt(1, 15);
             queue.add((PendingRunnable)retry::run, delay, TimeUnit.SECONDS);
-        });
+        }, onStale);
 
         Supplier<LongSupplier> nowSupplier = () -> {
             RandomSource forked = random.fork();
+            // TODO (now): meta-randomise scale of clock drift
             return FrequentLargeRange.builder(forked)
                                                    .ratio(1, 5)
                                                    .small(50, 5000, TimeUnit.MICROSECONDS)
                                                    .large(1, 10, TimeUnit.MILLISECONDS)
                                                    .build()
-                    .mapAsLong(j -> Math.max(0, queue.nowInMillis() + j))
+                                                   .mapAsLong(j -> Math.max(0, queue.nowInMillis() + TimeUnit.NANOSECONDS.toMillis(j)))
                     .asLongSupplier(forked);
         };
 
         Verifier verifier = createVerifier(keyCount);
-        SimulatedDelayedExecutorService globalExecutor = new SimulatedDelayedExecutorService(queue, agent);
+        SimulatedDelayedExecutorService globalExecutor = new SimulatedDelayedExecutorService(queue, null);
 
         Function<CommandStore, AsyncExecutor> executor = ignore -> globalExecutor;
 
@@ -311,9 +312,10 @@ public class BurnTest
         Map<MessageType, Stats> messageStatsMap;
         try
         {
-            messageStatsMap = Cluster.run(toArray(nodes, Id[]::new), listener, () -> queue, queue::checkFailures,
-                                          responseSink, globalExecutor,
-                                          random::fork, nowSupplier,
+            messageStatsMap = Cluster.run(toArray(nodes, Id[]::new), listener, () -> queue,
+                                          (id, onStale) -> globalExecutor.withAgent(agentSupplier.apply(onStale)),
+                                          queue::checkFailures,
+                                          responseSink, random::fork, nowSupplier,
                                           topologyFactory, initialRequests::poll,
                                           onSubmitted::set
             );
diff --git a/accord-core/src/test/java/accord/burn/TopologyUpdates.java b/accord-core/src/test/java/accord/burn/TopologyUpdates.java
index ae3d3f0a..caaa7099 100644
--- a/accord-core/src/test/java/accord/burn/TopologyUpdates.java
+++ b/accord-core/src/test/java/accord/burn/TopologyUpdates.java
@@ -28,15 +28,16 @@ import accord.utils.MessageTask;
 import org.agrona.collections.Long2ObjectHashMap;
 
 import java.util.*;
+import java.util.function.Function;
 
 public class TopologyUpdates
 {
     private final Long2ObjectHashMap<Map<Node.Id, Ranges>> pendingTopologies = new Long2ObjectHashMap<>();
-    private final AgentExecutor executor;
 
-    public TopologyUpdates(AgentExecutor executor)
+    Function<Node.Id, AgentExecutor> executors;
+    public TopologyUpdates(Function<Node.Id, AgentExecutor> executors)
     {
-        this.executor = executor;
+        this.executors = executors;
     }
 
     public synchronized MessageTask notify(Node originator, Topology prev, Topology update)
@@ -50,7 +51,7 @@ public class TopologyUpdates
             nodeToNewRanges.put(node, newRanges);
         }
         pendingTopologies.put(update.epoch(), nodeToNewRanges);
-        return MessageTask.begin(originator, nodes, executor, "TopologyNotify:" + update.epoch(), (node, from, onDone) -> {
+        return MessageTask.begin(originator, nodes, executors.apply(originator.id()), "TopologyNotify:" + update.epoch(), (node, from, onDone) -> {
             long nodeEpoch = node.epoch();
             if (nodeEpoch + 1 < update.epoch())
                 onDone.accept(false);
@@ -71,7 +72,7 @@ public class TopologyUpdates
         if (pending.isEmpty())
             pendingTopologies.remove(epoch);
 
-        MessageTask.begin(originator, cluster, executor, "SyncComplete:" + epoch, (node, from, onDone) -> {
+        MessageTask.begin(originator, cluster, executors.apply(originator.id()), "SyncComplete:" + epoch, (node, from, onDone) -> {
             node.onRemoteSyncComplete(originator.id(), epoch);
             onDone.accept(true);
         });
@@ -79,8 +80,8 @@ public class TopologyUpdates
 
     public synchronized void epochClosed(Node originator, Collection<Node.Id> cluster, Ranges ranges, long epoch)
     {
-        executor.execute(() -> {
-            MessageTask.begin(originator, cluster, executor, "EpochClosed:" + epoch, (node, from, onDone) -> {
+        executors.apply(originator.id()).execute(() -> {
+            MessageTask.begin(originator, cluster, executors.apply(originator.id()), "EpochClosed:" + epoch, (node, from, onDone) -> {
                 node.onEpochClosed(ranges, epoch);
                 onDone.accept(true);
             });
@@ -89,8 +90,8 @@ public class TopologyUpdates
 
     public synchronized void epochRedundant(Node originator, Collection<Node.Id> cluster, Ranges ranges, long epoch)
     {
-        executor.execute(() -> {
-            MessageTask.begin(originator, cluster, executor, "EpochComplete:" + epoch, (node, from, onDone) -> {
+        executors.apply(originator.id()).execute(() -> {
+            MessageTask.begin(originator, cluster, executors.apply(originator.id()), "EpochComplete:" + epoch, (node, from, onDone) -> {
                 node.onEpochRedundant(ranges, epoch);
                 onDone.accept(true);
             });
diff --git a/accord-core/src/test/java/accord/coordinate/tracking/TrackerReconciler.java b/accord-core/src/test/java/accord/coordinate/tracking/TrackerReconciler.java
index 0c9bdea8..7fca1f82 100644
--- a/accord-core/src/test/java/accord/coordinate/tracking/TrackerReconciler.java
+++ b/accord-core/src/test/java/accord/coordinate/tracking/TrackerReconciler.java
@@ -123,7 +123,7 @@ public abstract class TrackerReconciler<ST extends ShardTracker, T extends Abstr
 
         Deque<Topology> topologies = new ArrayDeque<>();
         topologies.add(topology);
-        TopologyUpdates topologyUpdates = new TopologyUpdates(executor);
+        TopologyUpdates topologyUpdates = new TopologyUpdates(null);
         TopologyRandomizer configRandomizer = new TopologyRandomizer(() -> random, topology, topologyUpdates, null);
         while (--count > 0)
         {
diff --git a/accord-core/src/test/java/accord/impl/TestAgent.java b/accord-core/src/test/java/accord/impl/TestAgent.java
index d3e4832b..0624c4ea 100644
--- a/accord-core/src/test/java/accord/impl/TestAgent.java
+++ b/accord-core/src/test/java/accord/impl/TestAgent.java
@@ -95,6 +95,12 @@ public class TestAgent implements Agent
         retry.run();
     }
 
+    @Override
+    public void onStale(Timestamp staleSince, Ranges ranges)
+    {
+
+    }
+
     @Override
     public void onUncaughtException(Throwable t)
     {
@@ -119,7 +125,8 @@ public class TestAgent implements Agent
     }
 
     @Override
-    public void onLocalBarrier(@Nonnull Seekables<?, ?> keysOrRanges, @Nonnull Timestamp executeAt) {
+    public void onLocalBarrier(@Nonnull Seekables<?, ?> keysOrRanges, @Nonnull Timestamp executeAt)
+    {
         completedLocalBarriers.computeIfAbsent(executeAt, ignored -> new AtomicInteger()).incrementAndGet();
     }
 }
diff --git a/accord-core/src/test/java/accord/impl/basic/Cluster.java b/accord-core/src/test/java/accord/impl/basic/Cluster.java
index 8595f7a6..7455a413 100644
--- a/accord-core/src/test/java/accord/impl/basic/Cluster.java
+++ b/accord-core/src/test/java/accord/impl/basic/Cluster.java
@@ -22,15 +22,18 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import java.util.function.BiConsumer;
+import java.util.function.BiFunction;
+import java.util.function.BooleanSupplier;
 import java.util.function.Consumer;
 import java.util.function.Function;
+import java.util.function.IntSupplier;
 import java.util.function.LongSupplier;
 import java.util.function.Supplier;
 
@@ -46,6 +49,7 @@ import accord.burn.BurnTestConfigurationService;
 import accord.burn.TopologyUpdates;
 import accord.coordinate.TxnExecute;
 import accord.coordinate.TxnPersist;
+import accord.burn.random.FrequentLargeRange;
 import accord.impl.CoordinateDurabilityScheduling;
 import accord.impl.IntHashKey;
 import accord.impl.SimpleProgressLog;
@@ -65,12 +69,22 @@ import accord.messages.Message;
 import accord.messages.Reply;
 import accord.messages.Request;
 import accord.messages.SafeCallback;
+import accord.primitives.Ranges;
+import accord.primitives.Timestamp;
 import accord.topology.Topology;
 import accord.topology.TopologyRandomizer;
 import accord.utils.RandomSource;
 import accord.utils.async.AsyncChains;
 import accord.utils.async.AsyncResult;
 
+import static accord.impl.basic.Cluster.OverrideLinksKind.NONE;
+import static accord.impl.basic.Cluster.OverrideLinksKind.RANDOM_BIDIRECTIONAL;
+import static accord.impl.basic.NodeSink.Action.DELIVER;
+import static accord.impl.basic.NodeSink.Action.DROP;
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.singletonMap;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
 import static java.util.concurrent.TimeUnit.SECONDS;
 import static java.util.stream.Collectors.toList;
 
@@ -86,9 +100,34 @@ public class Cluster implements Scheduler
         public String toString() { return Integer.toString(count); }
     }
 
-    Map<MessageType, Stats> statsMap = new HashMap<>();
+    public static class LinkConfig
+    {
+        final Function<List<Id>, BiFunction<Id, Id, Link>> overrideLinks;
+        final BiFunction<Id, Id, Link> defaultLinks;
+
+        public LinkConfig(Function<List<Id>, BiFunction<Id, Id, Link>> overrideLinks, BiFunction<Id, Id, Link> defaultLinks)
+        {
+            this.overrideLinks = overrideLinks;
+            this.defaultLinks = defaultLinks;
+        }
+    }
 
-    final RandomSource randomSource;
+    static class Link
+    {
+        final Supplier<NodeSink.Action> action;
+        final LongSupplier latencyMicros;
+
+        Link(Supplier<NodeSink.Action> action, LongSupplier latencyMicros)
+        {
+            this.action = action;
+            this.latencyMicros = latencyMicros;
+        }
+    }
+
+    final Map<MessageType, Stats> statsMap = new HashMap<>();
+
+    final RandomSource random;
+    final LinkConfig linkConfig;
     final Function<Id, Node> lookup;
     final PendingQueue pending;
     final Runnable checkFailures;
@@ -98,17 +137,18 @@ public class Cluster implements Scheduler
     final MessageListener messageListener;
     int clock;
     int recurring;
-    Set<Id> partitionSet;
+    BiFunction<Id, Id, Link> links;
 
-    public Cluster(RandomSource randomSource, MessageListener messageListener, Supplier<PendingQueue> queueSupplier, Runnable checkFailures, Function<Id, Node> lookup, Consumer<Packet> responseSink)
+    public Cluster(RandomSource random, MessageListener messageListener, Supplier<PendingQueue> queueSupplier, Runnable checkFailures, Function<Id, Node> lookup, IntSupplier rf, Consumer<Packet> responseSink)
     {
-        this.randomSource = randomSource;
+        this.random = random;
         this.messageListener = messageListener;
         this.pending = queueSupplier.get();
         this.checkFailures = checkFailures;
         this.lookup = lookup;
         this.responseSink = responseSink;
-        this.partitionSet = new HashSet<>();
+        this.linkConfig = defaultLinkConfig(random, rf);
+        this.links = linkConfig.defaultLinks;
     }
 
     NodeSink create(Id self, RandomSource random)
@@ -218,47 +258,75 @@ public class Cluster implements Scheduler
         run.run();
     }
 
-    public static Map<MessageType, Stats> run(Id[] nodes, MessageListener messageListener, Supplier<PendingQueue> queueSupplier, Runnable checkFailures, Consumer<Packet> responseSink, AgentExecutor executor, Supplier<RandomSource> randomSupplier, Supplier<LongSupplier> nowSupplierSupplier, TopologyFactory topologyFactory, Supplier<Packet> in, Consumer<Runnable> noMoreWorkSignal)
+    public static Map<MessageType, Stats> run(Id[] nodes, MessageListener messageListener, Supplier<PendingQueue> queueSupplier,
+                                                  BiFunction<Id, BiConsumer<Timestamp, Ranges>, AgentExecutor> nodeExecutorSupplier,
+                                                  Runnable checkFailures, Consumer<Packet> responseSink,
+                                                  Supplier<RandomSource> randomSupplier, Supplier<LongSupplier> nowSupplierSupplier,
+                                                  TopologyFactory topologyFactory, Supplier<Packet> in, Consumer<Runnable> noMoreWorkSignal)
     {
         Topology topology = topologyFactory.toTopology(nodes);
-        Map<Id, Node> lookup = new LinkedHashMap<>();
+        Map<Id, Node> nodeMap = new LinkedHashMap<>();
+        Map<Id, AgentExecutor> executorMap = new LinkedHashMap<>();
         try
         {
-            Cluster sinks = new Cluster(randomSupplier.get(), messageListener, queueSupplier, checkFailures, lookup::get, responseSink);
-            TopologyUpdates topologyUpdates = new TopologyUpdates(executor);
-            TopologyRandomizer configRandomizer = new TopologyRandomizer(randomSupplier, topology, topologyUpdates, lookup::get);
+            RandomSource random = randomSupplier.get();
+            Cluster sinks = new Cluster(randomSupplier.get(), messageListener, queueSupplier, checkFailures, nodeMap::get, () -> topologyFactory.rf, responseSink);
+            TopologyUpdates topologyUpdates = new TopologyUpdates(executorMap::get);
+            TopologyRandomizer configRandomizer = new TopologyRandomizer(randomSupplier, topology, topologyUpdates, nodeMap::get);
             List<CoordinateDurabilityScheduling> durabilityScheduling = new ArrayList<>();
             for (Id id : nodes)
             {
                 MessageSink messageSink = sinks.create(id, randomSupplier.get());
                 LongSupplier nowSupplier = nowSupplierSupplier.get();
-                BurnTestConfigurationService configService = new BurnTestConfigurationService(id, executor, randomSupplier, topology, lookup::get, topologyUpdates);
                 LocalConfig localConfig = new MutableLocalConfig();
+                BiConsumer<Timestamp, Ranges> onStale = (sinceAtLeast, ranges) -> configRandomizer.onStale(id, sinceAtLeast, ranges);
+                AgentExecutor nodeExecutor = nodeExecutorSupplier.apply(id, onStale);
+                executorMap.put(id, nodeExecutor);
+                BurnTestConfigurationService configService = new BurnTestConfigurationService(id, nodeExecutor, randomSupplier, topology, nodeMap::get, topologyUpdates);
+                BooleanSupplier isLoadedCheck = random.biasedUniformBools(0.5f);
                 Node node = new Node(id, messageSink, LocalMessage::process, configService, nowSupplier, NodeTimeService.unixWrapper(TimeUnit.MILLISECONDS, nowSupplier),
                                      () -> new ListStore(id), new ShardDistributor.EvenSplit<>(8, ignore -> new IntHashKey.Splitter()),
-                                     executor.agent(),
+                                     nodeExecutor.agent(),
                                      randomSupplier.get(), sinks, SizeOfIntersectionSorter.SUPPLIER,
-                                     SimpleProgressLog::new, DelayedCommandStores.factory(sinks.pending), TxnExecute.FACTORY, TxnPersist.FACTORY, Apply.FACTORY,
+                                     SimpleProgressLog::new, DelayedCommandStores.factory(sinks.pending, isLoadedCheck), TxnExecute.FACTORY, TxnPersist.FACTORY, Apply.FACTORY,
                                      localConfig);
-                lookup.put(id, node);
                 CoordinateDurabilityScheduling durability = new CoordinateDurabilityScheduling(node);
                 // TODO (desired): randomise
                 durability.setFrequency(60, SECONDS);
                 durability.setGlobalCycleTime(180, SECONDS);
                 durabilityScheduling.add(durability);
+                nodeMap.put(id, node);
+                durabilityScheduling.add(new CoordinateDurabilityScheduling(node));
+            }
+
+            Runnable updateDurabilityRate;
+            {
+                IntSupplier frequencySeconds       = random.biasedUniformIntsSupplier( 1, 120, 10,  30, 10,  60).get();
+                IntSupplier shardCycleTimeSeconds  = random.biasedUniformIntsSupplier(5, 60, 10, 30, 1, 30).get();
+                IntSupplier globalCycleTimeSeconds = random.biasedUniformIntsSupplier( 1,  90, 10,  30, 10,  60).get();
+                updateDurabilityRate = () -> {
+                    int f = frequencySeconds.getAsInt();
+                    int s = shardCycleTimeSeconds.getAsInt();
+                    int g = globalCycleTimeSeconds.getAsInt();
+                    durabilityScheduling.forEach(d -> {
+                        d.setFrequency(f, SECONDS);
+                        d.setShardCycleTime(s, SECONDS);
+                        d.setGlobalCycleTime(g, SECONDS);
+                    });
+                };
             }
+            updateDurabilityRate.run();
 
             // startup
-            AsyncResult<?> startup = AsyncChains.reduce(lookup.values().stream().map(Node::unsafeStart).collect(toList()), (a, b) -> null).beginAsResult();
+            AsyncResult<?> startup = AsyncChains.reduce(nodeMap.values().stream().map(Node::unsafeStart).collect(toList()), (a, b) -> null).beginAsResult();
             while (sinks.processPending());
             Assertions.assertTrue(startup.isDone());
 
             List<Id> nodesList = new ArrayList<>(Arrays.asList(nodes));
-            RandomSource shuffleRandom = randomSupplier.get();
             Scheduled chaos = sinks.recurring(() -> {
-                Collections.shuffle(nodesList, shuffleRandom.asJdkRandom());
-                int partitionSize = shuffleRandom.nextInt((topologyFactory.rf+1)/2);
-                sinks.partitionSet = new LinkedHashSet<>(nodesList.subList(0, partitionSize));
+                sinks.links = sinks.linkConfig.overrideLinks.apply(nodesList);
+                if (random.decide(0.1f))
+                    updateDurabilityRate.run();
             }, 5L, SECONDS);
 
             Scheduled reconfigure = sinks.recurring(configRandomizer::maybeUpdateTopology, 1, SECONDS);
@@ -278,7 +346,7 @@ public class Cluster implements Scheduler
             chaos.cancel();
             reconfigure.cancel();
             durabilityScheduling.forEach(CoordinateDurabilityScheduling::stop);
-            sinks.partitionSet = Collections.emptySet();
+            sinks.links = sinks.linkConfig.defaultLinks;
 
             // give progress log et al a chance to finish
             // TODO (desired, testing): would be nice to make this more certain than an arbitrary number of additional rounds
@@ -300,7 +368,157 @@ public class Cluster implements Scheduler
         }
         finally
         {
-            lookup.values().forEach(Node::shutdown);
+            nodeMap.values().forEach(Node::shutdown);
         }
     }
+
+    private static BiFunction<Id, Id, Link> partition(List<Id> nodes, RandomSource random, int rf, BiFunction<Id, Id, Link> up)
+    {
+        Collections.shuffle(nodes, random.asJdkRandom());
+        int partitionSize = random.nextInt((rf+1)/2);
+        Set<Id> partition = new LinkedHashSet<>(nodes.subList(0, partitionSize));
+        BiFunction<Id, Id, Link> down = (from, to) -> new Link(() -> DROP, up.apply(from, to).latencyMicros);
+        return (from, to) -> (partition.contains(from) == partition.contains(to) ? up : down).apply(from, to);
+    }
+
+    /**
+     * pair every node with one other node in one direction with a network behaviour override
+     */
+    private static BiFunction<Id, Id, Link> pairedUnidirectionalOverrides(Function<Link, Link> linkOverride, List<Id> nodes, RandomSource random, BiFunction<Id, Id, Link> fallback)
+    {
+        Map<Id, Map<Id, Link>> map = new HashMap<>();
+        Collections.shuffle(nodes, random.asJdkRandom());
+        for (int i = 0 ; i + 1 < nodes.size() ; i += 2)
+        {
+            Id from = nodes.get(i);
+            Id to = nodes.get(i + 1);
+            Link link = linkOverride.apply(fallback.apply(from, to));
+            map.put(from, singletonMap(to, link));
+        }
+        return (from, to) -> nonNullOrGet(map.getOrDefault(from, emptyMap()).get(to), from, to, fallback);
+    }
+
+    private static BiFunction<Id, Id, Link> randomOverrides(boolean bidirectional, Function<Link, Link> linkOverride, int count, List<Id> nodes, RandomSource random, BiFunction<Id, Id, Link> fallback)
+    {
+        Map<Id, Map<Id, Link>> map = new HashMap<>();
+        while (count > 0)
+        {
+            Id from = nodes.get(random.nextInt(nodes.size()));
+            Id to = nodes.get(random.nextInt(nodes.size()));
+            Link fwd = linkOverride.apply(fallback.apply(from, to));
+            if (null == map.computeIfAbsent(from, ignore -> new HashMap<>()).putIfAbsent(to, fwd))
+            {
+                if (bidirectional)
+                {
+                    Link rev = linkOverride.apply(fallback.apply(to, from));
+                    map.computeIfAbsent(to, ignore -> new HashMap<>()).put(from, rev);
+                }
+                --count;
+            }
+        }
+        return (from, to) -> nonNullOrGet(map.getOrDefault(from, emptyMap()).get(to), from, to, fallback);
+    }
+
+    private static Link nonNullOrGet(Link ifNotNull, Id from, Id to, BiFunction<Id, Id, Link> function)
+    {
+        if (ifNotNull != null)
+            return ifNotNull;
+        return function.apply(from, to);
+    }
+
+    private static Link healthy(LongSupplier latency)
+    {
+        return new Link(() -> DELIVER, latency);
+    }
+
+    private static Link down(LongSupplier latency)
+    {
+        return new Link(() -> DROP, latency);
+    }
+
+    private LongSupplier defaultRandomWalkLatencyMicros(RandomSource random)
+    {
+        LongSupplier range = FrequentLargeRange.builder(random)
+                                               .ratio(1, 5)
+                                               .small(500, TimeUnit.MICROSECONDS, 5, MILLISECONDS)
+                                               .large(50, MILLISECONDS, 5, SECONDS)
+                                               .build().asLongSupplier(random);
+
+        return () -> NANOSECONDS.toMicros(range.getAsLong());
+    }
+
+    enum OverrideLinkKind { LATENCY, ACTION, BOTH }
+
+    private Supplier<Function<Link, Link>> linkOverrideSupplier(RandomSource random)
+    {
+        Supplier<OverrideLinkKind> nextKind = random.randomWeightedPicker(OverrideLinkKind.values());
+        Supplier<LongSupplier> latencySupplier = random.biasedUniformLongsSupplier(
+            MILLISECONDS.toMicros(1L), SECONDS.toMicros(2L),
+            MILLISECONDS.toMicros(1L), MILLISECONDS.toMicros(300L), SECONDS.toMicros(1L),
+            MILLISECONDS.toMicros(1L), MILLISECONDS.toMicros(300L), SECONDS.toMicros(1L)
+        );
+        NodeSink.Action[] actions = NodeSink.Action.values();
+        Supplier<Supplier<NodeSink.Action>> actionSupplier = () -> random.randomWeightedPicker(actions);
+        return () -> {
+            OverrideLinkKind kind = nextKind.get();
+            switch (kind)
+            {
+                default: throw new AssertionError("Unhandled: " + kind);
+                case BOTH: return ignore -> new Link(actionSupplier.get(), latencySupplier.get());
+                case ACTION: return override -> new Link(actionSupplier.get(), override.latencyMicros);
+                case LATENCY: return override -> new Link(override.action, latencySupplier.get());
+            }
+        };
+    }
+
+    enum OverrideLinksKind { NONE, PAIRED_UNIDIRECTIONAL, RANDOM_UNIDIRECTIONAL, RANDOM_BIDIRECTIONAL }
+
+    private Function<List<Id>, BiFunction<Id, Id, Link>> overrideLinks(RandomSource random, IntSupplier rf, BiFunction<Id, Id, Link> defaultLinks)
+    {
+        Supplier<Function<Link, Link>> linkOverrideSupplier = linkOverrideSupplier(random);
+        BooleanSupplier partitionChance = random.biasedUniformBools(random.nextFloat());
+        Supplier<OverrideLinksKind> nextKind = random.randomWeightedPicker(OverrideLinksKind.values());
+        return nodesList -> {
+            BiFunction<Id, Id, Link> links = defaultLinks;
+            if (partitionChance.getAsBoolean()) // 50% chance of a whole network partition
+                links = partition(nodesList, random, rf.getAsInt(), links);
+
+            OverrideLinksKind kind = nextKind.get();
+            if (kind == NONE)
+                return links;
+
+            Function<Link, Link> linkOverride = linkOverrideSupplier.get();
+            switch (kind)
+            {
+                default: throw new AssertionError("Unhandled: " + kind);
+                case PAIRED_UNIDIRECTIONAL:
+                    return pairedUnidirectionalOverrides(linkOverride, nodesList, random, defaultLinks);
+                case RANDOM_BIDIRECTIONAL:
+                case RANDOM_UNIDIRECTIONAL:
+                    boolean bidirectional = kind == RANDOM_BIDIRECTIONAL;
+                    int count = random.nextInt(bidirectional || random.nextBoolean() ? nodesList.size() : (nodesList.size() * nodesList.size())/2);
+                    return randomOverrides(bidirectional, linkOverride, count, nodesList, random, defaultLinks);
... 750 lines suppressed ...


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org