You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by be...@apache.org on 2024/10/13 11:14:19 UTC
(cassandra-accord) branch trunk updated: Do not advance the range we're processing for durability scheduling until success; clear Recover.recoverOks to free up memory once quorum reached
This is an automated email from the ASF dual-hosted git repository.
benedict pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git
The following commit(s) were added to refs/heads/trunk by this push:
new 63f89692 Do not advance the range we're processing for durability scheduling until success; clear Recover.recoverOks to free up memory once quorum reached
63f89692 is described below
commit 63f896923fd67e7ac228f06bdcfbfe4f407be1d2
Author: Benedict Elliott Smith <be...@apache.org>
AuthorDate: Sun Oct 13 12:13:28 2024 +0100
Do not advance the range we're processing for durability scheduling until success; clear Recover.recoverOks to free up memory once quorum reached
---
.../src/main/java/accord/coordinate/Recover.java | 36 +--
.../impl/CoordinateDurabilityScheduling.java | 273 +++++++++++----------
.../src/main/java/accord/local/CommandStore.java | 30 ++-
.../src/main/java/accord/local/CommandStores.java | 4 +-
.../main/java/accord/messages/BeginRecovery.java | 1 -
.../main/java/accord/primitives/LatestDeps.java | 5 +-
6 files changed, 192 insertions(+), 157 deletions(-)
diff --git a/accord-core/src/main/java/accord/coordinate/Recover.java b/accord-core/src/main/java/accord/coordinate/Recover.java
index 5f89c835..09ec89e5 100644
--- a/accord-core/src/main/java/accord/coordinate/Recover.java
+++ b/accord-core/src/main/java/accord/coordinate/Recover.java
@@ -101,7 +101,7 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
private final BiConsumer<Outcome, Throwable> callback;
private boolean isDone;
- private final SortedListMap<Id, RecoverOk> recoverOks;
+ private SortedListMap<Id, RecoverOk> recoverOks;
private final RecoveryTracker tracker;
private boolean isBallotPromised;
@@ -199,6 +199,8 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
Invariants.checkState(!isBallotPromised);
isBallotPromised = true;
+ SortedListMap<Id, RecoverOk> recoverOks = this.recoverOks;
+ if (!Invariants.debug()) this.recoverOks = null;
List<RecoverOk> recoverOkList = recoverOks.valuesAsNullableList();
RecoverOk acceptOrCommit = maxAccepted(recoverOkList);
RecoverOk acceptOrCommitNotTruncated = acceptOrCommit == null || acceptOrCommit.status != Status.Truncated
@@ -213,14 +215,14 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
case Truncated: throw illegalState("Truncate should be filtered");
case Invalidated:
{
- commitInvalidate();
+ commitInvalidate(invalidateUntil(recoverOks));
return;
}
case Applied:
case PreApplied:
{
- withCommittedDeps(executeAt, (stableDeps, withEpochFailure) -> {
+ withCommittedDeps(recoverOkList, executeAt, (stableDeps, withEpochFailure) -> {
if (withEpochFailure != null)
{
node.agent().onUncaughtException(CoordinationFailed.wrap(withEpochFailure));
@@ -242,7 +244,7 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
{
// TODO (required): if we have to calculate deps for any shard, should we first ensure they are stable?
// it should only be possible for a fast path decision, but we might have Stable in only one shard.
- withCommittedDeps(executeAt, (stableDeps, withEpochFailure) -> {
+ withCommittedDeps(recoverOkList, executeAt, (stableDeps, withEpochFailure) -> {
if (withEpochFailure != null)
{
node.agent().onUncaughtException(CoordinationFailed.wrap(withEpochFailure));
@@ -257,7 +259,7 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
case Committed:
{
// TODO (expected): should we only ask for txns with t0 < t0' < t?
- withCommittedDeps(executeAt, (committedDeps, withEpochFailure) -> {
+ withCommittedDeps(recoverOkList, executeAt, (committedDeps, withEpochFailure) -> {
if (withEpochFailure != null)
{
node.agent().onUncaughtException(CoordinationFailed.wrap(withEpochFailure));
@@ -280,7 +282,7 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
case AcceptedInvalidate:
{
- invalidate();
+ invalidate(recoverOks);
return;
}
@@ -321,7 +323,7 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
if (tracker.rejectsFastPath() || recoverOks.valuesAsNullableStream().anyMatch(ok -> ok != null && ok.rejectsFastPath))
{
- invalidate();
+ invalidate(recoverOks);
return;
}
@@ -348,9 +350,9 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
propose(txnId, proposeDeps);
}
- private void withCommittedDeps(Timestamp executeAt, BiConsumer<Deps, Throwable> withDeps)
+ private void withCommittedDeps(List<RecoverOk> nullableRecoverOkList, Timestamp executeAt, BiConsumer<Deps, Throwable> withDeps)
{
- LatestDeps.MergedCommitResult merged = LatestDeps.mergeCommit(txnId, executeAt, recoverOks.valuesAsNullableList(), ok -> ok == null ? null : ok.deps);
+ LatestDeps.MergedCommitResult merged = LatestDeps.mergeCommit(txnId, executeAt, nullableRecoverOkList, ok -> ok == null ? null : ok.deps);
node.withEpoch(executeAt.epoch(), (ignored, withEpochFailure) -> {
if (withEpochFailure != null)
{
@@ -379,22 +381,26 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
});
}
- private void invalidate()
+ private void invalidate(SortedListMap<Id, RecoverOk> recoverOks)
{
+ Timestamp invalidateUntil = invalidateUntil(recoverOks);
proposeInvalidate(node, ballot, txnId, route.homeKey(), (success, fail) -> {
if (fail != null) accept(null, fail);
- else commitInvalidate();
+ else commitInvalidate(invalidateUntil);
});
}
- private void commitInvalidate()
+ private Timestamp invalidateUntil(SortedListMap<Id, RecoverOk> recoverOks)
{
// If not accepted then the executeAt is not consistent cross the peers and likely different on every node. There is also an edge case
// when ranges are removed from the topology, during this case the executeAt won't know the ranges and the invalidate commit will fail.
- Timestamp invalidateUntil = recoverOks.valuesAsNullableStream()
- .map(ok -> ok == null ? null : ok.status.hasBeen(Status.Accepted) ? ok.executeAt : ok.txnId)
- .reduce(txnId, Timestamp::nonNullOrMax);
+ return recoverOks.valuesAsNullableStream()
+ .map(ok -> ok == null ? null : ok.status.hasBeen(Status.Accepted) ? ok.executeAt : ok.txnId)
+ .reduce(txnId, Timestamp::nonNullOrMax);
+ }
+ private void commitInvalidate(Timestamp invalidateUntil)
+ {
node.withEpoch(invalidateUntil.epoch(), (ignored, withEpochFailure) -> {
if (withEpochFailure != null)
{
diff --git a/accord-core/src/main/java/accord/impl/CoordinateDurabilityScheduling.java b/accord-core/src/main/java/accord/impl/CoordinateDurabilityScheduling.java
index bb6de074..3a224e94 100644
--- a/accord-core/src/main/java/accord/impl/CoordinateDurabilityScheduling.java
+++ b/accord-core/src/main/java/accord/impl/CoordinateDurabilityScheduling.java
@@ -23,11 +23,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Objects;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.stream.Collectors;
import com.google.common.primitives.Ints;
import org.slf4j.Logger;
@@ -87,32 +83,30 @@ public class CoordinateDurabilityScheduling
private final Node node;
private Scheduler.Scheduled scheduled;
- private final AtomicReference<Ranges> active = new AtomicReference<>(Ranges.EMPTY);
- private final ConcurrentHashMap<TxnId, Ranges> coordinating = new ConcurrentHashMap<>();
/*
* In each round at each node wait this amount of time between initiating new CoordinateShardDurable
*/
- private int frequencyMicros = Ints.saturatedCast(TimeUnit.MILLISECONDS.toMicros(500L));
+ private long frequencyMicros = TimeUnit.MILLISECONDS.toMicros(500L);
/*
* In each round at each node wait this amount of time between allocating a CoordinateShardDurable txnId
* and coordinating the shard durability
*/
- private int txnIdLagMicros = Ints.saturatedCast(TimeUnit.SECONDS.toMicros(1L));
+ private long txnIdLagMicros = TimeUnit.SECONDS.toMicros(1L);
/*
* In each round at each node wait this amount of time between allocating a CoordinateShardDurable txnId
* and coordinating the shard durability
*/
- private int durabilityLagMicros = Ints.saturatedCast(TimeUnit.MILLISECONDS.toMicros(500L));
+ private long durabilityLagMicros = TimeUnit.MILLISECONDS.toMicros(500L);
/*
* Target for how often the entire ring should be processed in microseconds. Every node will start at an offset in the current round that is based
* on this value / by (total # replicas * its index in the current round). The current round is determined by dividing time since the epoch by this
* duration.
*/
- private int shardCycleTimeMicros = Ints.saturatedCast(TimeUnit.SECONDS.toMicros(30));
+ private long shardCycleTimeMicros = TimeUnit.SECONDS.toMicros(30);
/*
* Every node will independently attempt to invoke CoordinateGloballyDurable
@@ -127,13 +121,138 @@ public class CoordinateDurabilityScheduling
private long globalCycleTimeMicros = TimeUnit.SECONDS.toMicros(30);
private Topology currentGlobalTopology;
- private final Map<Shard, Integer> localIndexes = new HashMap<>();
+ private final Map<Range, ShardScheduler> shardSchedulers = new HashMap<>();
private int globalIndex;
private long nextGlobalSyncTimeMicros;
- private long prevShardSyncTimeMicros;
volatile boolean stop;
+ private class ShardScheduler
+ {
+ Shard shard;
+
+ // based on ideal number of splits
+ int nodeOffset;
+
+ int index;
+ int numberOfSplits, desiredNumberOfSplits;
+ boolean defunct;
+ long shardCycleTimeMicros;
+ Scheduler.Scheduled scheduled;
+
+ private ShardScheduler()
+ {
+ }
+
+ synchronized void update(Shard shard, int offset)
+ {
+ this.shard = shard;
+ this.nodeOffset = offset;
+ this.shardCycleTimeMicros = Math.max(CoordinateDurabilityScheduling.this.shardCycleTimeMicros, shard.rf() * 3L * frequencyMicros);
+ this.desiredNumberOfSplits = (int) ((shardCycleTimeMicros + frequencyMicros - 1) / frequencyMicros);
+ if (numberOfSplits == 0 || numberOfSplits < desiredNumberOfSplits)
+ {
+ index = offset;
+ numberOfSplits = desiredNumberOfSplits;
+ }
+ }
+
+ synchronized void markDefunct()
+ {
+ defunct = true;
+ }
+
+ synchronized void schedule()
+ {
+ if (defunct)
+ return;
+
+ long nowMicros = node.elapsed(MICROSECONDS);
+ int cyclePosition = (nodeOffset + (((index * shard.rf()) + numberOfSplits - 1) / numberOfSplits)) % shard.rf();
+ long microsOffset = (cyclePosition * shardCycleTimeMicros) / shard.rf();
+ long scheduleAt = nowMicros - (nowMicros % shardCycleTimeMicros) + microsOffset;
+ if (nowMicros > scheduleAt)
+ scheduleAt += shardCycleTimeMicros;
+
+ ShardDistributor distributor = node.commandStores().shardDistributor();
+ Range range = distributor.splitRange(shard.range, index, index + 1, numberOfSplits);
+
+ scheduled = node.scheduler().once(() -> {
+ // TODO (required): allocate stale HLC from a reservation of HLCs for this purpose
+ TxnId syncId = node.nextTxnId(ExclusiveSyncPoint, Domain.Range);
+ startShardSync(syncId, Ranges.of(range));
+ }, scheduleAt - nowMicros, MICROSECONDS);
+ }
+
+ /**
+ * The first step for coordinating shard durable is to run an exclusive sync point
+ * the result of which can then be used to run
+ */
+ private void startShardSync(TxnId syncId, Ranges ranges)
+ {
+ scheduled = node.scheduler().once(() -> node.withEpoch(syncId.epoch(), (ignored, withEpochFailure) -> {
+ if (withEpochFailure != null)
+ {
+ startShardSync(syncId, ranges);
+ Throwable wrapped = CoordinationFailed.wrap(withEpochFailure);
+ logger.trace("Exception waiting for epoch before coordinating exclusive sync point for local shard durability, epoch " + syncId.epoch(), wrapped);
+ node.agent().onUncaughtException(wrapped);
+ return;
+ }
+ scheduled = null;
+ FullRoute<Range> route = (FullRoute<Range>) node.computeRoute(syncId, ranges);
+ exclusiveSyncPoint(node, syncId, route)
+ .addCallback((success, fail) -> {
+ if (fail != null)
+ {
+ synchronized (ShardScheduler.this)
+ {
+ index += numberOfSplits;
+ numberOfSplits *= 2;
+ // TODO (required): try to recover or invalidate prior sync point
+ schedule();
+ logger.warn("{}: Exception coordinating ExclusiveSyncPoint for {} durability. Increased numberOfSplits to " + numberOfSplits, syncId, ranges, fail);
+ }
+ }
+ else
+ {
+ coordinateShardDurableAfterExclusiveSyncPoint(node, success);
+ logger.trace("{}: Successfully coordinated ExclusiveSyncPoint for local shard durability of {}", syncId, ranges);
+ }
+ });
+ }), txnIdLagMicros, MICROSECONDS);
+ }
+
+ private void coordinateShardDurableAfterExclusiveSyncPoint(Node node, SyncPoint<Range> exclusiveSyncPoint)
+ {
+ scheduled = node.scheduler().once(() -> {
+ scheduled = null;
+ node.commandStores().any().execute(() -> {
+ CoordinateShardDurable.coordinate(node, exclusiveSyncPoint)
+ .addCallback((success, fail) -> {
+ if (fail != null && fail.getClass() != SyncPointErased.class)
+ {
+ logger.trace("Exception coordinating local shard durability, will retry immediately", fail);
+ coordinateShardDurableAfterExclusiveSyncPoint(node, exclusiveSyncPoint);
+ }
+ else
+ {
+ synchronized (ShardScheduler.this)
+ {
+ if (++index == 0 && numberOfSplits > desiredNumberOfSplits)
+ numberOfSplits = Math.max(desiredNumberOfSplits, (int)(numberOfSplits * 0.9));
+
+ schedule();
+ }
+ }
+ });
+ });
+ }, durabilityLagMicros, MICROSECONDS);
+ }
+
+ }
+
+
public CoordinateDurabilityScheduling(Node node)
{
this.node = node;
@@ -171,7 +290,6 @@ public class CoordinateDurabilityScheduling
{
Invariants.checkState(!stop); // cannot currently restart safely
long nowMicros = node.elapsed(MICROSECONDS);
- prevShardSyncTimeMicros = nowMicros;
setNextGlobalSyncTime(nowMicros);
scheduled = node.scheduler().recurring(this::run, frequencyMicros, MICROSECONDS);
}
@@ -184,106 +302,27 @@ public class CoordinateDurabilityScheduling
}
/**
- * Schedule the first CoordinateShardDurable execution for the current round. Sub-steps will be scheduled after
- * each sub-step completes, and once all are completed scheduleCoordinateShardDurable is called again.
+ * Update our topology information, and schedule any global syncs that may be pending.
*/
private void run()
{
if (stop)
return;
+ // TODO (expected): invoke this as soon as topology is updated in topology manager
updateTopology();
if (currentGlobalTopology == null || currentGlobalTopology.size() == 0)
return;
+ // TODO (expected): schedule this directly based on the global sync frequency - this is an artefact of previously scheduling shard syncs as well
long nowMicros = node.elapsed(MICROSECONDS);
if (nextGlobalSyncTimeMicros <= nowMicros)
{
startGlobalSync();
setNextGlobalSyncTime(nowMicros);
}
-
- List<Ranges> coordinate = rangesToShardSync(nowMicros);
- prevShardSyncTimeMicros = nowMicros;
- if (coordinate.isEmpty())
- {
- logger.trace("Nothing pending in schedule for time slot at {}", nowMicros);
- return;
- }
-
- logger.trace("Scheduling CoordinateShardDurable for {} at {}", coordinate, nowMicros);
- for (Ranges ranges : coordinate)
- {
- if (ranges.isEmpty())
- continue;
-
- Ranges inactiveRanges = ranges.without(active.get());
- if (!inactiveRanges.equals(ranges))
- {
- String waitingOn = coordinating.entrySet().stream().filter(e -> e.getValue().intersects(ranges)).map(Objects::toString).collect(Collectors.joining(", ", "[", "]"));
- logger.info("Not initiating new durability scheduling for {} as previous attempt(s) {} still in progress (scheduling {})", ranges.without(inactiveRanges), waitingOn, inactiveRanges);
- if (inactiveRanges.isEmpty())
- continue;
- }
- active.accumulateAndGet(inactiveRanges, Ranges::with);
- startShardSync(inactiveRanges);
- }
- }
-
- /**
- * The first step for coordinating shard durable is to run an exclusive sync point
- * the result of which can then be used to run
- */
- private void startShardSync(Ranges ranges)
- {
- TxnId at = node.nextTxnId(ExclusiveSyncPoint, Domain.Range);
- coordinating.put(at, ranges);
- node.scheduler().once(() -> node.withEpoch(at.epoch(), (ignored, withEpochFailure) -> {
- FullRoute<Range> route = (FullRoute<Range>) node.computeRoute(at, ranges);
- if (withEpochFailure != null)
- {
- Throwable wrapped = CoordinationFailed.wrap(withEpochFailure);
- logger.trace("Exception waiting for epoch before coordinating exclusive sync point for local shard durability, epoch " + at.epoch(), wrapped);
- node.agent().onUncaughtException(wrapped);
- return;
- }
- exclusiveSyncPoint(node, at, route)
- .addCallback((success, fail) -> {
- if (fail != null)
- {
- logger.trace("{}: Exception coordinating ExclusiveSyncPoint for local shard durability of {}", at, ranges, fail);
- coordinating.remove(at);
- active.accumulateAndGet(route.toRanges(), Ranges::without);
- }
- else
- {
- coordinateShardDurableAfterExclusiveSyncPoint(node, success);
- logger.trace("{}: Successfully coordinated ExclusiveSyncPoint for local shard durability of {}", at, ranges);
- }
- });
- }), txnIdLagMicros, MICROSECONDS);
}
- private void coordinateShardDurableAfterExclusiveSyncPoint(Node node, SyncPoint<Range> exclusiveSyncPoint)
- {
- node.scheduler().once(() -> {
- node.commandStores().any().execute(() -> {
- CoordinateShardDurable.coordinate(node, exclusiveSyncPoint)
- .addCallback((success, fail) -> {
- if (fail != null && fail.getClass() != SyncPointErased.class)
- {
- logger.trace("Exception coordinating local shard durability, will retry immediately", fail);
- coordinateShardDurableAfterExclusiveSyncPoint(node, exclusiveSyncPoint);
- }
- else
- {
- coordinating.remove(exclusiveSyncPoint.syncId);
- active.accumulateAndGet(exclusiveSyncPoint.route.toRanges(), Ranges::without);
- }
- });
- });
- }, durabilityLagMicros, MICROSECONDS);
- }
private void startGlobalSync()
{
@@ -302,39 +341,6 @@ public class CoordinateDurabilityScheduling
}
}
- private List<Ranges> rangesToShardSync(long nowMicros)
- {
- ShardDistributor distributor = node.commandStores().shardDistributor();
- List<Ranges> result = new ArrayList<>();
- for (Map.Entry<Shard, Integer> e : localIndexes.entrySet())
- {
- Shard shard = e.getKey();
- int index = e.getValue();
- int shardCycleTimeMicros = Math.max(this.shardCycleTimeMicros, Ints.saturatedCast(shard.rf() * 3L * frequencyMicros));
- long microsOffset = (index * (long)shardCycleTimeMicros) / shard.rf();
- long prevSyncTimeMicros = Math.max(prevShardSyncTimeMicros, nowMicros - ((shardCycleTimeMicros / shard.rf()) / 2L));
- int from = (int) ((prevSyncTimeMicros + microsOffset) % shardCycleTimeMicros);
- int to = (int) ((nowMicros + microsOffset) % shardCycleTimeMicros);
- List<Range> ranges = new ArrayList<>();
- if (from > to)
- {
- Range range = distributor.splitRange(shard.range, to, shardCycleTimeMicros, shardCycleTimeMicros);
- if (range != null)
- ranges.add(range);
- to = from;
- from = 0;
- }
-
- Range range = distributor.splitRange(shard.range, from, to, shardCycleTimeMicros);
- if (range != null)
- ranges.add(range);
-
- if (!ranges.isEmpty())
- result.add(Ranges.of(ranges.toArray(new Range[0])));
- }
- return result;
- }
-
private void updateTopology()
{
Topology latestGlobal = node.topology().current();
@@ -350,9 +356,16 @@ public class CoordinateDurabilityScheduling
Collections.sort(ids);
globalIndex = ids.indexOf(node.id());
- localIndexes.clear();
+ Map<Range, ShardScheduler> prev = new HashMap<>(this.shardSchedulers);
+ this.shardSchedulers.clear();
for (Shard shard : latestLocal.shards())
- localIndexes.put(shard, shard.nodes.find(node.id()));
+ {
+ ShardScheduler scheduler = prev.remove(shard.range);
+ if (scheduler == null) scheduler = new ShardScheduler();
+ shardSchedulers.put(shard.range, scheduler);
+ scheduler.update(shard, shard.nodes.find(node.id()));
+ }
+ prev.forEach((r, s) -> s.markDefunct());
}
/**
diff --git a/accord-core/src/main/java/accord/local/CommandStore.java b/accord-core/src/main/java/accord/local/CommandStore.java
index 5f220ee6..de58c861 100644
--- a/accord-core/src/main/java/accord/local/CommandStore.java
+++ b/accord-core/src/main/java/accord/local/CommandStore.java
@@ -27,6 +27,7 @@ import javax.annotation.Nullable;
import accord.api.Agent;
import accord.local.CommandStores.RangesForEpoch;
+import accord.primitives.Range;
import accord.primitives.Routables;
import accord.primitives.Unseekables;
import accord.utils.async.AsyncChain;
@@ -44,6 +45,7 @@ import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
import java.util.function.Function;
@@ -477,18 +479,27 @@ public abstract class CommandStore implements AgentExecutor
};
}
- // TODO (required): replace with a simple wait on suitable exclusive sync point(s)
+ // TODO (required, correctness): replace with a simple wait on suitable exclusive sync point(s)
private void fetchMajorityDeps(AsyncResults.SettableResult<Void> coordination, Node node, long epoch, Ranges ranges)
+ {
+ AtomicInteger index = new AtomicInteger();
+ fetchMajorityDeps(coordination, node, epoch, ranges, index);
+ }
+
+ private void fetchMajorityDeps(AsyncResults.SettableResult<Void> coordination, Node node, long epoch, Ranges ranges, AtomicInteger index)
{
TxnId id = TxnId.fromValues(epoch - 1, 0, node.id());
Timestamp before = Timestamp.minForEpoch(epoch);
- FullRoute<?> route = node.computeRoute(id, ranges);
- // TODO (required): we need to ensure anyone we receive a reply from proposes newer timestamps for anything we don't see
+ int rangeIndex = index.get() / 64;
+ int subRangeIndex = index.get() % 64;
+ Range nextRange = node.commandStores().shardDistributor().splitRange(ranges.get(rangeIndex), subRangeIndex, subRangeIndex + 1, 64);
+ FullRoute<?> route = node.computeRoute(id, Ranges.of(nextRange));
+ logger.info("Fetching deps to sync epoch {} for range {}", epoch, nextRange);
CollectCalculatedDeps.withCalculatedDeps(node, id, route, route, before, (deps, fail) -> {
if (fail != null)
{
- logger.warn("Failed to fetch deps for syncing epoch {} for ranges {}", epoch, ranges, fail);
- node.scheduler().once(() -> fetchMajorityDeps(coordination, node, epoch, ranges), 1L, TimeUnit.MINUTES);
+ logger.warn("Failed to fetch deps for syncing epoch {} for range {}", epoch, nextRange, fail);
+ node.scheduler().once(() -> fetchMajorityDeps(coordination, node, epoch, ranges, index), 1L, TimeUnit.MINUTES);
node.agent().onUncaughtException(fail);
}
else
@@ -500,13 +511,16 @@ public abstract class CommandStore implements AgentExecutor
}).begin((success, fail2) -> {
if (fail2 != null)
{
- logger.warn("Failed to apply deps for syncing epoch {} for ranges {}", epoch, ranges, fail2);
- node.scheduler().once(() -> fetchMajorityDeps(coordination, node, epoch, ranges), 1L, TimeUnit.MINUTES);
+ logger.warn("Failed to apply deps for syncing epoch {} for range {}", epoch, nextRange, fail2);
+ node.scheduler().once(() -> fetchMajorityDeps(coordination, node, epoch, ranges, index), 1L, TimeUnit.MINUTES);
node.agent().onUncaughtException(fail2);
}
else
{
- coordination.setSuccess(null);
+ int prev = index.getAndIncrement();
+ Invariants.checkState(rangeIndex * 64 + subRangeIndex == prev);
+ if (prev + 1 == ranges.size() * 64) coordination.setSuccess(null);
+ else fetchMajorityDeps(coordination, node, epoch, ranges, index);
}
});
}
diff --git a/accord-core/src/main/java/accord/local/CommandStores.java b/accord-core/src/main/java/accord/local/CommandStores.java
index c32b9ece..e10d80d3 100644
--- a/accord-core/src/main/java/accord/local/CommandStores.java
+++ b/accord-core/src/main/java/accord/local/CommandStores.java
@@ -707,12 +707,12 @@ public abstract class CommandStores
public CommandStore select(Route<?> route)
{
- return select(ranges -> ranges.intersects(route));
+ return select(ranges -> ranges.intersects(route));
}
public CommandStore select(Participants<?> participants)
{
- return select(ranges -> ranges.intersects(participants));
+ return select(ranges -> ranges.intersects(participants));
}
private CommandStore select(Predicate<Ranges> fn)
diff --git a/accord-core/src/main/java/accord/messages/BeginRecovery.java b/accord-core/src/main/java/accord/messages/BeginRecovery.java
index b434a183..e91a4b05 100644
--- a/accord-core/src/main/java/accord/messages/BeginRecovery.java
+++ b/accord-core/src/main/java/accord/messages/BeginRecovery.java
@@ -121,7 +121,6 @@ public class BeginRecovery extends TxnRequest.WithUnsynced<BeginRecovery.Recover
Deps localDeps = null;
if (!command.known().deps.hasCommittedOrDecidedDeps())
{
- // TODO (required): consider whether we are safe ignoring the concept of minUnsyncedEpoch here
localDeps = calculateDeps(safeStore, txnId, participants, constant(minEpoch), txnId);
}
diff --git a/accord-core/src/main/java/accord/primitives/LatestDeps.java b/accord-core/src/main/java/accord/primitives/LatestDeps.java
index dd65779b..313eddb3 100644
--- a/accord-core/src/main/java/accord/primitives/LatestDeps.java
+++ b/accord-core/src/main/java/accord/primitives/LatestDeps.java
@@ -106,7 +106,10 @@ public class LatestDeps extends ReducingRangeMap<LatestDeps.LatestEntry>
static LatestEntry reduce(LatestEntry a, LatestEntry b)
{
- return reduce(a, b, (v1, v2) -> new LatestEntry(v1.known, v1.ballot, v1.coordinatedDeps, v1.localDeps.with(v2.localDeps)));
+ return reduce(a, b, (v1, v2) -> new LatestEntry(v1.known, v1.ballot, v1.coordinatedDeps,
+ v1.localDeps == null ? v2.localDeps
+ : v2.localDeps == null ? v1.localDeps
+ : v1.localDeps.with(v2.localDeps)));
}
static LatestEntry slice(RoutingKey start, RoutingKey end, LatestEntry v)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org