You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by be...@apache.org on 2024/10/13 11:14:19 UTC

(cassandra-accord) branch trunk updated: Do not advance the range we're processing for durability scheduling until success; clear Recover.recoverOks to free up memory once quorum reached

This is an automated email from the ASF dual-hosted git repository.

benedict pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git


The following commit(s) were added to refs/heads/trunk by this push:
     new 63f89692 Do not advance the range we're processing for durability scheduling until success; clear Recover.recoverOks to free up memory once quorum reached
63f89692 is described below

commit 63f896923fd67e7ac228f06bdcfbfe4f407be1d2
Author: Benedict Elliott Smith <be...@apache.org>
AuthorDate: Sun Oct 13 12:13:28 2024 +0100

    Do not advance the range we're processing for durability scheduling until success; clear Recover.recoverOks to free up memory once quorum reached
---
 .../src/main/java/accord/coordinate/Recover.java   |  36 +--
 .../impl/CoordinateDurabilityScheduling.java       | 273 +++++++++++----------
 .../src/main/java/accord/local/CommandStore.java   |  30 ++-
 .../src/main/java/accord/local/CommandStores.java  |   4 +-
 .../main/java/accord/messages/BeginRecovery.java   |   1 -
 .../main/java/accord/primitives/LatestDeps.java    |   5 +-
 6 files changed, 192 insertions(+), 157 deletions(-)

diff --git a/accord-core/src/main/java/accord/coordinate/Recover.java b/accord-core/src/main/java/accord/coordinate/Recover.java
index 5f89c835..09ec89e5 100644
--- a/accord-core/src/main/java/accord/coordinate/Recover.java
+++ b/accord-core/src/main/java/accord/coordinate/Recover.java
@@ -101,7 +101,7 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
     private final BiConsumer<Outcome, Throwable> callback;
     private boolean isDone;
 
-    private final SortedListMap<Id, RecoverOk> recoverOks;
+    private SortedListMap<Id, RecoverOk> recoverOks;
     private final RecoveryTracker tracker;
     private boolean isBallotPromised;
 
@@ -199,6 +199,8 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
         Invariants.checkState(!isBallotPromised);
         isBallotPromised = true;
 
+        SortedListMap<Id, RecoverOk> recoverOks = this.recoverOks;
+        if (!Invariants.debug()) this.recoverOks = null;
         List<RecoverOk> recoverOkList = recoverOks.valuesAsNullableList();
         RecoverOk acceptOrCommit = maxAccepted(recoverOkList);
         RecoverOk acceptOrCommitNotTruncated = acceptOrCommit == null || acceptOrCommit.status != Status.Truncated
@@ -213,14 +215,14 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
                 case Truncated: throw illegalState("Truncate should be filtered");
                 case Invalidated:
                 {
-                    commitInvalidate();
+                    commitInvalidate(invalidateUntil(recoverOks));
                     return;
                 }
 
                 case Applied:
                 case PreApplied:
                 {
-                    withCommittedDeps(executeAt, (stableDeps, withEpochFailure) -> {
+                    withCommittedDeps(recoverOkList, executeAt, (stableDeps, withEpochFailure) -> {
                         if (withEpochFailure != null)
                         {
                             node.agent().onUncaughtException(CoordinationFailed.wrap(withEpochFailure));
@@ -242,7 +244,7 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
                 {
                     // TODO (required): if we have to calculate deps for any shard, should we first ensure they are stable?
                     //   it should only be possible for a fast path decision, but we might have Stable in only one shard.
-                    withCommittedDeps(executeAt, (stableDeps, withEpochFailure) -> {
+                    withCommittedDeps(recoverOkList, executeAt, (stableDeps, withEpochFailure) -> {
                         if (withEpochFailure != null)
                         {
                             node.agent().onUncaughtException(CoordinationFailed.wrap(withEpochFailure));
@@ -257,7 +259,7 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
                 case Committed:
                 {
                     // TODO (expected): should we only ask for txns with t0 < t0' < t?
-                    withCommittedDeps(executeAt, (committedDeps, withEpochFailure) -> {
+                    withCommittedDeps(recoverOkList, executeAt, (committedDeps, withEpochFailure) -> {
                         if (withEpochFailure != null)
                         {
                             node.agent().onUncaughtException(CoordinationFailed.wrap(withEpochFailure));
@@ -280,7 +282,7 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
 
                 case AcceptedInvalidate:
                 {
-                    invalidate();
+                    invalidate(recoverOks);
                     return;
                 }
 
@@ -321,7 +323,7 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
 
         if (tracker.rejectsFastPath() || recoverOks.valuesAsNullableStream().anyMatch(ok -> ok != null && ok.rejectsFastPath))
         {
-            invalidate();
+            invalidate(recoverOks);
             return;
         }
 
@@ -348,9 +350,9 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
         propose(txnId, proposeDeps);
     }
 
-    private void withCommittedDeps(Timestamp executeAt, BiConsumer<Deps, Throwable> withDeps)
+    private void withCommittedDeps(List<RecoverOk> nullableRecoverOkList, Timestamp executeAt, BiConsumer<Deps, Throwable> withDeps)
     {
-        LatestDeps.MergedCommitResult merged = LatestDeps.mergeCommit(txnId, executeAt, recoverOks.valuesAsNullableList(), ok -> ok == null ? null : ok.deps);
+        LatestDeps.MergedCommitResult merged = LatestDeps.mergeCommit(txnId, executeAt, nullableRecoverOkList, ok -> ok == null ? null : ok.deps);
         node.withEpoch(executeAt.epoch(), (ignored, withEpochFailure) -> {
             if (withEpochFailure != null)
             {
@@ -379,22 +381,26 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
         });
     }
 
-    private void invalidate()
+    private void invalidate(SortedListMap<Id, RecoverOk> recoverOks)
     {
+        Timestamp invalidateUntil = invalidateUntil(recoverOks);
         proposeInvalidate(node, ballot, txnId, route.homeKey(), (success, fail) -> {
             if (fail != null) accept(null, fail);
-            else commitInvalidate();
+            else commitInvalidate(invalidateUntil);
         });
     }
 
-    private void commitInvalidate()
+    private Timestamp invalidateUntil(SortedListMap<Id, RecoverOk> recoverOks)
     {
         // If not accepted then the executeAt is not consistent cross the peers and likely different on every node.  There is also an edge case
         // when ranges are removed from the topology, during this case the executeAt won't know the ranges and the invalidate commit will fail.
-        Timestamp invalidateUntil = recoverOks.valuesAsNullableStream()
-                                              .map(ok -> ok == null ? null : ok.status.hasBeen(Status.Accepted) ? ok.executeAt : ok.txnId)
-                                              .reduce(txnId, Timestamp::nonNullOrMax);
+        return recoverOks.valuesAsNullableStream()
+                         .map(ok -> ok == null ? null : ok.status.hasBeen(Status.Accepted) ? ok.executeAt : ok.txnId)
+                         .reduce(txnId, Timestamp::nonNullOrMax);
+    }
 
+    private void commitInvalidate(Timestamp invalidateUntil)
+    {
         node.withEpoch(invalidateUntil.epoch(), (ignored, withEpochFailure) -> {
             if (withEpochFailure != null)
             {
diff --git a/accord-core/src/main/java/accord/impl/CoordinateDurabilityScheduling.java b/accord-core/src/main/java/accord/impl/CoordinateDurabilityScheduling.java
index bb6de074..3a224e94 100644
--- a/accord-core/src/main/java/accord/impl/CoordinateDurabilityScheduling.java
+++ b/accord-core/src/main/java/accord/impl/CoordinateDurabilityScheduling.java
@@ -23,11 +23,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
-import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.stream.Collectors;
 
 import com.google.common.primitives.Ints;
 import org.slf4j.Logger;
@@ -87,32 +83,30 @@ public class CoordinateDurabilityScheduling
 
     private final Node node;
     private Scheduler.Scheduled scheduled;
-    private final AtomicReference<Ranges> active = new AtomicReference<>(Ranges.EMPTY);
-    private final ConcurrentHashMap<TxnId, Ranges> coordinating = new ConcurrentHashMap<>();
 
     /*
      * In each round at each node wait this amount of time between initiating new CoordinateShardDurable
      */
-    private int frequencyMicros = Ints.saturatedCast(TimeUnit.MILLISECONDS.toMicros(500L));
+    private long frequencyMicros = TimeUnit.MILLISECONDS.toMicros(500L);
 
     /*
      * In each round at each node wait this amount of time between allocating a CoordinateShardDurable txnId
      * and coordinating the shard durability
      */
-    private int txnIdLagMicros = Ints.saturatedCast(TimeUnit.SECONDS.toMicros(1L));
+    private long txnIdLagMicros = TimeUnit.SECONDS.toMicros(1L);
 
     /*
      * In each round at each node wait this amount of time between allocating a CoordinateShardDurable txnId
      * and coordinating the shard durability
      */
-    private int durabilityLagMicros = Ints.saturatedCast(TimeUnit.MILLISECONDS.toMicros(500L));
+    private long durabilityLagMicros = TimeUnit.MILLISECONDS.toMicros(500L);
 
     /*
      * Target for how often the entire ring should be processed in microseconds. Every node will start at an offset in the current round that is based
      * on this value / by (total # replicas * its index in the current round). The current round is determined by dividing time since the epoch by this
      * duration.
      */
-    private int shardCycleTimeMicros = Ints.saturatedCast(TimeUnit.SECONDS.toMicros(30));
+    private long shardCycleTimeMicros = TimeUnit.SECONDS.toMicros(30);
 
     /*
      * Every node will independently attempt to invoke CoordinateGloballyDurable
@@ -127,13 +121,138 @@ public class CoordinateDurabilityScheduling
     private long globalCycleTimeMicros = TimeUnit.SECONDS.toMicros(30);
 
     private Topology currentGlobalTopology;
-    private final Map<Shard, Integer> localIndexes = new HashMap<>();
+    private final Map<Range, ShardScheduler> shardSchedulers = new HashMap<>();
     private int globalIndex;
 
     private long nextGlobalSyncTimeMicros;
-    private long prevShardSyncTimeMicros;
     volatile boolean stop;
 
+    private class ShardScheduler
+    {
+        Shard shard;
+
+        // based on ideal number of splits
+        int nodeOffset;
+
+        int index;
+        int numberOfSplits, desiredNumberOfSplits;
+        boolean defunct;
+        long shardCycleTimeMicros;
+        Scheduler.Scheduled scheduled;
+
+        private ShardScheduler()
+        {
+        }
+
+        synchronized void update(Shard shard, int offset)
+        {
+            this.shard = shard;
+            this.nodeOffset = offset;
+            this.shardCycleTimeMicros = Math.max(CoordinateDurabilityScheduling.this.shardCycleTimeMicros, shard.rf() * 3L * frequencyMicros);
+            this.desiredNumberOfSplits = (int) ((shardCycleTimeMicros + frequencyMicros - 1) / frequencyMicros);
+            if (numberOfSplits == 0 || numberOfSplits < desiredNumberOfSplits)
+            {
+                index = offset;
+                numberOfSplits = desiredNumberOfSplits;
+            }
+        }
+
+        synchronized void markDefunct()
+        {
+            defunct = true;
+        }
+
+        synchronized void schedule()
+        {
+            if (defunct)
+                return;
+
+            long nowMicros = node.elapsed(MICROSECONDS);
+            int cyclePosition = (nodeOffset + (((index * shard.rf()) + numberOfSplits - 1) / numberOfSplits)) % shard.rf();
+            long microsOffset = (cyclePosition * shardCycleTimeMicros) / shard.rf();
+            long scheduleAt = nowMicros - (nowMicros % shardCycleTimeMicros) + microsOffset;
+            if (nowMicros > scheduleAt)
+                scheduleAt += shardCycleTimeMicros;
+
+            ShardDistributor distributor = node.commandStores().shardDistributor();
+            Range range = distributor.splitRange(shard.range, index, index + 1, numberOfSplits);
+
+            scheduled = node.scheduler().once(() -> {
+                // TODO (required): allocate stale HLC from a reservation of HLCs for this purpose
+                TxnId syncId = node.nextTxnId(ExclusiveSyncPoint, Domain.Range);
+                startShardSync(syncId, Ranges.of(range));
+            }, scheduleAt - nowMicros, MICROSECONDS);
+        }
+
+        /**
+         * The first step for coordinating shard durable is to run an exclusive sync point
+         * the result of which can then be used to run
+         */
+        private void startShardSync(TxnId syncId, Ranges ranges)
+        {
+            scheduled = node.scheduler().once(() -> node.withEpoch(syncId.epoch(), (ignored, withEpochFailure) -> {
+                if (withEpochFailure != null)
+                {
+                    startShardSync(syncId, ranges);
+                    Throwable wrapped = CoordinationFailed.wrap(withEpochFailure);
+                    logger.trace("Exception waiting for epoch before coordinating exclusive sync point for local shard durability, epoch " + syncId.epoch(), wrapped);
+                    node.agent().onUncaughtException(wrapped);
+                    return;
+                }
+                scheduled = null;
+                FullRoute<Range> route = (FullRoute<Range>) node.computeRoute(syncId, ranges);
+                exclusiveSyncPoint(node, syncId, route)
+                .addCallback((success, fail) -> {
+                    if (fail != null)
+                    {
+                        synchronized (ShardScheduler.this)
+                        {
+                            index += numberOfSplits;
+                            numberOfSplits *= 2;
+                            // TODO (required): try to recover or invalidate prior sync point
+                            schedule();
+                            logger.warn("{}: Exception coordinating ExclusiveSyncPoint for {} durability. Increased numberOfSplits to " + numberOfSplits, syncId, ranges, fail);
+                        }
+                    }
+                    else
+                    {
+                        coordinateShardDurableAfterExclusiveSyncPoint(node, success);
+                        logger.trace("{}: Successfully coordinated ExclusiveSyncPoint for local shard durability of {}", syncId, ranges);
+                    }
+                });
+            }), txnIdLagMicros, MICROSECONDS);
+        }
+
+        private void coordinateShardDurableAfterExclusiveSyncPoint(Node node, SyncPoint<Range> exclusiveSyncPoint)
+        {
+            scheduled = node.scheduler().once(() -> {
+                scheduled = null;
+                node.commandStores().any().execute(() -> {
+                    CoordinateShardDurable.coordinate(node, exclusiveSyncPoint)
+                                          .addCallback((success, fail) -> {
+                                              if (fail != null && fail.getClass() != SyncPointErased.class)
+                                              {
+                                                  logger.trace("Exception coordinating local shard durability, will retry immediately", fail);
+                                                  coordinateShardDurableAfterExclusiveSyncPoint(node, exclusiveSyncPoint);
+                                              }
+                                              else
+                                              {
+                                                  synchronized (ShardScheduler.this)
+                                                  {
+                                                      if (++index == 0 && numberOfSplits > desiredNumberOfSplits)
+                                                          numberOfSplits = Math.max(desiredNumberOfSplits, (int)(numberOfSplits * 0.9));
+
+                                                      schedule();
+                                                  }
+                                              }
+                                          });
+                });
+            }, durabilityLagMicros, MICROSECONDS);
+        }
+
+    }
+
+
     public CoordinateDurabilityScheduling(Node node)
     {
         this.node = node;
@@ -171,7 +290,6 @@ public class CoordinateDurabilityScheduling
     {
         Invariants.checkState(!stop); // cannot currently restart safely
         long nowMicros = node.elapsed(MICROSECONDS);
-        prevShardSyncTimeMicros = nowMicros;
         setNextGlobalSyncTime(nowMicros);
         scheduled = node.scheduler().recurring(this::run, frequencyMicros, MICROSECONDS);
     }
@@ -184,106 +302,27 @@ public class CoordinateDurabilityScheduling
     }
 
     /**
-     * Schedule the first CoordinateShardDurable execution for the current round. Sub-steps will be scheduled after
-     * each sub-step completes, and once all are completed scheduleCoordinateShardDurable is called again.
+     * Update our topology information, and schedule any global syncs that may be pending.
      */
     private void run()
     {
         if (stop)
             return;
 
+        // TODO (expected): invoke this as soon as topology is updated in topology manager
         updateTopology();
         if (currentGlobalTopology == null || currentGlobalTopology.size() == 0)
             return;
 
+        // TODO (expected): schedule this directly based on the global sync frequency - this is an artefact of previously scheduling shard syncs as well
         long nowMicros = node.elapsed(MICROSECONDS);
         if (nextGlobalSyncTimeMicros <= nowMicros)
         {
             startGlobalSync();
             setNextGlobalSyncTime(nowMicros);
         }
-
-        List<Ranges> coordinate = rangesToShardSync(nowMicros);
-        prevShardSyncTimeMicros = nowMicros;
-        if (coordinate.isEmpty())
-        {
-            logger.trace("Nothing pending in schedule for time slot at {}", nowMicros);
-            return;
-        }
-
-        logger.trace("Scheduling CoordinateShardDurable for {} at {}", coordinate, nowMicros);
-        for (Ranges ranges : coordinate)
-        {
-            if (ranges.isEmpty())
-                continue;
-
-            Ranges inactiveRanges = ranges.without(active.get());
-            if (!inactiveRanges.equals(ranges))
-            {
-                String waitingOn = coordinating.entrySet().stream().filter(e -> e.getValue().intersects(ranges)).map(Objects::toString).collect(Collectors.joining(", ", "[", "]"));
-                logger.info("Not initiating new durability scheduling for {} as previous attempt(s) {} still in progress (scheduling {})", ranges.without(inactiveRanges), waitingOn, inactiveRanges);
-                if (inactiveRanges.isEmpty())
-                    continue;
-            }
-            active.accumulateAndGet(inactiveRanges, Ranges::with);
-            startShardSync(inactiveRanges);
-        }
-    }
-
-    /**
-     * The first step for coordinating shard durable is to run an exclusive sync point
-     * the result of which can then be used to run
-     */
-    private void startShardSync(Ranges ranges)
-    {
-        TxnId at = node.nextTxnId(ExclusiveSyncPoint, Domain.Range);
-        coordinating.put(at, ranges);
-        node.scheduler().once(() -> node.withEpoch(at.epoch(), (ignored, withEpochFailure) -> {
-                           FullRoute<Range> route = (FullRoute<Range>) node.computeRoute(at, ranges);
-                           if (withEpochFailure != null)
-                           {
-                               Throwable wrapped = CoordinationFailed.wrap(withEpochFailure);
-                               logger.trace("Exception waiting for epoch before coordinating exclusive sync point for local shard durability, epoch " + at.epoch(), wrapped);
-                               node.agent().onUncaughtException(wrapped);
-                               return;
-                           }
-                           exclusiveSyncPoint(node, at, route)
-                           .addCallback((success, fail) -> {
-                               if (fail != null)
-                               {
-                                   logger.trace("{}: Exception coordinating ExclusiveSyncPoint for local shard durability of {}", at, ranges, fail);
-                                   coordinating.remove(at);
-                                   active.accumulateAndGet(route.toRanges(), Ranges::without);
-                               }
-                               else
-                               {
-                                   coordinateShardDurableAfterExclusiveSyncPoint(node, success);
-                                   logger.trace("{}: Successfully coordinated ExclusiveSyncPoint for local shard durability of {}", at, ranges);
-                               }
-                           });
-        }), txnIdLagMicros, MICROSECONDS);
     }
 
-    private void coordinateShardDurableAfterExclusiveSyncPoint(Node node, SyncPoint<Range> exclusiveSyncPoint)
-    {
-        node.scheduler().once(() -> {
-            node.commandStores().any().execute(() -> {
-                CoordinateShardDurable.coordinate(node, exclusiveSyncPoint)
-                                      .addCallback((success, fail) -> {
-                                          if (fail != null && fail.getClass() != SyncPointErased.class)
-                                          {
-                                              logger.trace("Exception coordinating local shard durability, will retry immediately", fail);
-                                              coordinateShardDurableAfterExclusiveSyncPoint(node, exclusiveSyncPoint);
-                                          }
-                                          else
-                                          {
-                                              coordinating.remove(exclusiveSyncPoint.syncId);
-                                              active.accumulateAndGet(exclusiveSyncPoint.route.toRanges(), Ranges::without);
-                                          }
-                                      });
-            });
-        }, durabilityLagMicros, MICROSECONDS);
-    }
 
     private void startGlobalSync()
     {
@@ -302,39 +341,6 @@ public class CoordinateDurabilityScheduling
         }
     }
 
-    private List<Ranges> rangesToShardSync(long nowMicros)
-    {
-        ShardDistributor distributor = node.commandStores().shardDistributor();
-        List<Ranges> result = new ArrayList<>();
-        for (Map.Entry<Shard, Integer> e : localIndexes.entrySet())
-        {
-            Shard shard = e.getKey();
-            int index = e.getValue();
-            int shardCycleTimeMicros = Math.max(this.shardCycleTimeMicros, Ints.saturatedCast(shard.rf() * 3L * frequencyMicros));
-            long microsOffset = (index * (long)shardCycleTimeMicros) / shard.rf();
-            long prevSyncTimeMicros = Math.max(prevShardSyncTimeMicros, nowMicros - ((shardCycleTimeMicros / shard.rf()) / 2L));
-            int from = (int) ((prevSyncTimeMicros + microsOffset) % shardCycleTimeMicros);
-            int to = (int) ((nowMicros + microsOffset) % shardCycleTimeMicros);
-            List<Range> ranges = new ArrayList<>();
-            if (from > to)
-            {
-                Range range = distributor.splitRange(shard.range, to, shardCycleTimeMicros, shardCycleTimeMicros);
-                if (range != null)
-                    ranges.add(range);
-                to = from;
-                from = 0;
-            }
-
-            Range range = distributor.splitRange(shard.range, from, to, shardCycleTimeMicros);
-            if (range != null)
-                ranges.add(range);
-
-            if (!ranges.isEmpty())
-                result.add(Ranges.of(ranges.toArray(new Range[0])));
-        }
-        return result;
-    }
-
     private void updateTopology()
     {
         Topology latestGlobal = node.topology().current();
@@ -350,9 +356,16 @@ public class CoordinateDurabilityScheduling
         Collections.sort(ids);
         globalIndex = ids.indexOf(node.id());
 
-        localIndexes.clear();
+        Map<Range, ShardScheduler> prev = new HashMap<>(this.shardSchedulers);
+        this.shardSchedulers.clear();
         for (Shard shard : latestLocal.shards())
-            localIndexes.put(shard, shard.nodes.find(node.id()));
+        {
+            ShardScheduler scheduler = prev.remove(shard.range);
+            if (scheduler == null) scheduler = new ShardScheduler();
+            shardSchedulers.put(shard.range, scheduler);
+            scheduler.update(shard, shard.nodes.find(node.id()));
+        }
+        prev.forEach((r, s) -> s.markDefunct());
     }
 
     /**
diff --git a/accord-core/src/main/java/accord/local/CommandStore.java b/accord-core/src/main/java/accord/local/CommandStore.java
index 5f220ee6..de58c861 100644
--- a/accord-core/src/main/java/accord/local/CommandStore.java
+++ b/accord-core/src/main/java/accord/local/CommandStore.java
@@ -27,6 +27,7 @@ import javax.annotation.Nullable;
 import accord.api.Agent;
 
 import accord.local.CommandStores.RangesForEpoch;
+import accord.primitives.Range;
 import accord.primitives.Routables;
 import accord.primitives.Unseekables;
 import accord.utils.async.AsyncChain;
@@ -44,6 +45,7 @@ import java.util.Set;
 import java.util.TreeMap;
 import java.util.concurrent.Callable;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 import java.util.function.Function;
@@ -477,18 +479,27 @@ public abstract class CommandStore implements AgentExecutor
         };
     }
 
-    // TODO (required): replace with a simple wait on suitable exclusive sync point(s)
+    // TODO (required, correctness): replace with a simple wait on suitable exclusive sync point(s)
     private void fetchMajorityDeps(AsyncResults.SettableResult<Void> coordination, Node node, long epoch, Ranges ranges)
+    {
+        AtomicInteger index = new AtomicInteger();
+        fetchMajorityDeps(coordination, node, epoch, ranges, index);
+    }
+
+    private void fetchMajorityDeps(AsyncResults.SettableResult<Void> coordination, Node node, long epoch, Ranges ranges, AtomicInteger index)
     {
         TxnId id = TxnId.fromValues(epoch - 1, 0, node.id());
         Timestamp before = Timestamp.minForEpoch(epoch);
-        FullRoute<?> route = node.computeRoute(id, ranges);
-        // TODO (required): we need to ensure anyone we receive a reply from proposes newer timestamps for anything we don't see
+        int rangeIndex = index.get() / 64;
+        int subRangeIndex = index.get() % 64;
+        Range nextRange = node.commandStores().shardDistributor().splitRange(ranges.get(rangeIndex), subRangeIndex, subRangeIndex + 1, 64);
+        FullRoute<?> route = node.computeRoute(id, Ranges.of(nextRange));
+        logger.info("Fetching deps to sync epoch {} for range {}", epoch, nextRange);
         CollectCalculatedDeps.withCalculatedDeps(node, id, route, route, before, (deps, fail) -> {
             if (fail != null)
             {
-                logger.warn("Failed to fetch deps for syncing epoch {} for ranges {}", epoch, ranges, fail);
-                node.scheduler().once(() -> fetchMajorityDeps(coordination, node, epoch, ranges), 1L, TimeUnit.MINUTES);
+                logger.warn("Failed to fetch deps for syncing epoch {} for range {}", epoch, nextRange, fail);
+                node.scheduler().once(() -> fetchMajorityDeps(coordination, node, epoch, ranges, index), 1L, TimeUnit.MINUTES);
                 node.agent().onUncaughtException(fail);
             }
             else
@@ -500,13 +511,16 @@ public abstract class CommandStore implements AgentExecutor
                 }).begin((success, fail2) -> {
                     if (fail2 != null)
                     {
-                        logger.warn("Failed to apply deps for syncing epoch {} for ranges {}", epoch, ranges, fail2);
-                        node.scheduler().once(() -> fetchMajorityDeps(coordination, node, epoch, ranges), 1L, TimeUnit.MINUTES);
+                        logger.warn("Failed to apply deps for syncing epoch {} for range {}", epoch, nextRange, fail2);
+                        node.scheduler().once(() -> fetchMajorityDeps(coordination, node, epoch, ranges, index), 1L, TimeUnit.MINUTES);
                         node.agent().onUncaughtException(fail2);
                     }
                     else
                     {
-                        coordination.setSuccess(null);
+                        int prev = index.getAndIncrement();
+                        Invariants.checkState(rangeIndex * 64 + subRangeIndex == prev);
+                        if (prev + 1 == ranges.size() * 64) coordination.setSuccess(null);
+                        else fetchMajorityDeps(coordination, node, epoch, ranges, index);
                     }
                 });
             }
diff --git a/accord-core/src/main/java/accord/local/CommandStores.java b/accord-core/src/main/java/accord/local/CommandStores.java
index c32b9ece..e10d80d3 100644
--- a/accord-core/src/main/java/accord/local/CommandStores.java
+++ b/accord-core/src/main/java/accord/local/CommandStores.java
@@ -707,12 +707,12 @@ public abstract class CommandStores
 
     public CommandStore select(Route<?> route)
     {
-        return  select(ranges -> ranges.intersects(route));
+        return select(ranges -> ranges.intersects(route));
     }
 
     public CommandStore select(Participants<?> participants)
     {
-        return  select(ranges -> ranges.intersects(participants));
+        return select(ranges -> ranges.intersects(participants));
     }
 
     private CommandStore select(Predicate<Ranges> fn)
diff --git a/accord-core/src/main/java/accord/messages/BeginRecovery.java b/accord-core/src/main/java/accord/messages/BeginRecovery.java
index b434a183..e91a4b05 100644
--- a/accord-core/src/main/java/accord/messages/BeginRecovery.java
+++ b/accord-core/src/main/java/accord/messages/BeginRecovery.java
@@ -121,7 +121,6 @@ public class BeginRecovery extends TxnRequest.WithUnsynced<BeginRecovery.Recover
         Deps localDeps = null;
         if (!command.known().deps.hasCommittedOrDecidedDeps())
         {
-            // TODO (required): consider whether we are safe ignoring the concept of minUnsyncedEpoch here
             localDeps = calculateDeps(safeStore, txnId, participants, constant(minEpoch), txnId);
         }
 
diff --git a/accord-core/src/main/java/accord/primitives/LatestDeps.java b/accord-core/src/main/java/accord/primitives/LatestDeps.java
index dd65779b..313eddb3 100644
--- a/accord-core/src/main/java/accord/primitives/LatestDeps.java
+++ b/accord-core/src/main/java/accord/primitives/LatestDeps.java
@@ -106,7 +106,10 @@ public class LatestDeps extends ReducingRangeMap<LatestDeps.LatestEntry>
 
         static LatestEntry reduce(LatestEntry a, LatestEntry b)
         {
-            return reduce(a, b, (v1, v2) -> new LatestEntry(v1.known, v1.ballot, v1.coordinatedDeps, v1.localDeps.with(v2.localDeps)));
+            return reduce(a, b, (v1, v2) -> new LatestEntry(v1.known, v1.ballot, v1.coordinatedDeps,
+                                                            v1.localDeps == null ? v2.localDeps
+                                                                                 : v2.localDeps == null ? v1.localDeps
+                                                                                                        : v1.localDeps.with(v2.localDeps)));
         }
 
         static LatestEntry slice(RoutingKey start, RoutingKey end, LatestEntry v)


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org