You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by be...@apache.org on 2023/11/21 12:59:21 UTC
(cassandra-accord) branch trunk updated: Improve validation and address various discovered faults
This is an automated email from the ASF dual-hosted git repository.
benedict pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git
The following commit(s) were added to refs/heads/trunk by this push:
new 746dabe0 Improve validation and address various discovered faults
746dabe0 is described below
commit 746dabe0b43bf719badbd605e68a76037d01256d
Author: Benedict Elliott Smith <be...@apache.org>
AuthorDate: Tue Aug 8 09:09:40 2023 +0100
Improve validation and address various discovered faults
patch by Benedict; reviewed by Ariel Weisberg for CASSANDRA-19045
---
accord-core/build.gradle | 9 +
accord-core/src/main/java/accord/api/Agent.java | 1 +
.../src/main/java/accord/api/ProgressLog.java | 5 +
accord-core/src/main/java/accord/api/Result.java | 8 -
.../src/main/java/accord/coordinate/Barrier.java | 12 +-
.../main/java/accord/coordinate/CheckShards.java | 2 +-
.../java/accord/coordinate/CoordinateNoOp.java | 6 +-
.../src/main/java/accord/coordinate/FetchData.java | 37 +-
.../src/main/java/accord/coordinate/FindRoute.java | 3 +-
.../main/java/accord/coordinate/FindSomeRoute.java | 2 +-
.../src/main/java/accord/coordinate/Infer.java | 230 ++++++--
.../main/java/accord/coordinate/MaybeRecover.java | 30 +-
.../java/accord/coordinate/ReadCoordinator.java | 23 +-
.../src/main/java/accord/coordinate/Recover.java | 3 +
.../java/accord/coordinate/RecoverWithRoute.java | 53 +-
.../accord/coordinate/RecoverWithSomeRoute.java | 2 +-
.../accord/coordinate/tracking/ReadTracker.java | 5 +
.../java/accord/impl/AbstractFetchCoordinator.java | 1 +
.../java/accord/impl/AbstractSafeCommandStore.java | 8 +-
.../main/java/accord/impl/CommandTimeseries.java | 18 +-
.../main/java/accord/impl/ErasedSafeCommand.java | 4 +-
.../java/accord/impl/InMemoryCommandStore.java | 170 +++++-
.../main/java/accord/impl/SafeCommandsForKey.java | 11 +-
.../main/java/accord/impl/SimpleProgressLog.java | 124 ++--
.../src/main/java/accord/local/Bootstrap.java | 5 +-
.../src/main/java/accord/local/Command.java | 354 ++++++++----
.../src/main/java/accord/local/CommandStore.java | 208 +++++--
.../src/main/java/accord/local/CommandStores.java | 67 ++-
.../src/main/java/accord/local/Commands.java | 318 ++++++-----
.../src/main/java/accord/local/DurableBefore.java | 26 +-
accord-core/src/main/java/accord/local/Node.java | 3 +-
.../main/java/accord/local/RedundantBefore.java | 294 +++++++++-
.../main/java/accord/local/RedundantStatus.java | 114 +++-
.../src/main/java/accord/local/SafeCommand.java | 2 -
.../main/java/accord/local/SafeCommandStore.java | 88 +--
.../src/main/java/accord/local/SaveStatus.java | 77 ++-
.../main/java/accord/local/SerializerSupport.java | 12 +-
accord-core/src/main/java/accord/local/Status.java | 471 ++++++++++++---
.../main/java/accord/messages/AbstractExecute.java | 21 +-
.../src/main/java/accord/messages/Accept.java | 4 +-
.../src/main/java/accord/messages/Apply.java | 2 +
.../accord/messages/ApplyThenWaitUntilApplied.java | 6 +
.../java/accord/messages/BeginInvalidation.java | 5 -
.../main/java/accord/messages/BeginRecovery.java | 38 +-
.../src/main/java/accord/messages/CheckStatus.java | 629 +++++++++++++++++----
.../src/main/java/accord/messages/Commit.java | 2 +-
.../src/main/java/accord/messages/GetDeps.java | 3 +-
.../src/main/java/accord/messages/MessageType.java | 2 +-
.../src/main/java/accord/messages/PreAccept.java | 45 +-
.../src/main/java/accord/messages/Propagate.java | 297 +++++++---
.../src/main/java/accord/messages/ReadData.java | 9 +-
.../main/java/accord/primitives/AbstractKeys.java | 13 +-
.../java/accord/primitives/AbstractRanges.java | 36 +-
.../accord/primitives/AbstractUnseekableKeys.java | 64 ++-
.../src/main/java/accord/primitives/Deps.java | 5 +
.../main/java/accord/primitives/EpochSupplier.java | 5 +
.../src/main/java/accord/primitives/KeyDeps.java | 24 +
.../src/main/java/accord/primitives/KeyRoute.java | 16 +-
.../src/main/java/accord/primitives/Keys.java | 10 +
.../main/java/accord/primitives/PartialDeps.java | 7 +
.../main/java/accord/primitives/Participants.java | 43 +-
.../main/java/accord/primitives/ProgressToken.java | 4 +-
.../src/main/java/accord/primitives/Range.java | 60 ++
.../src/main/java/accord/primitives/RangeDeps.java | 219 +++++--
.../main/java/accord/primitives/RangeRoute.java | 11 +-
.../src/main/java/accord/primitives/Ranges.java | 6 +
.../src/main/java/accord/primitives/Routable.java | 5 +
.../src/main/java/accord/primitives/Routables.java | 1 +
.../src/main/java/accord/primitives/Route.java | 32 +-
.../src/main/java/accord/primitives/Txn.java | 71 ++-
.../src/main/java/accord/primitives/TxnId.java | 19 +
.../main/java/accord/primitives/Unseekables.java | 2 +
.../src/main/java/accord/utils/Invariants.java | 4 +-
.../src/main/java/accord/utils/RandomSource.java | 270 ++++-----
.../java/accord/utils/ReducingIntervalMap.java | 44 +-
.../main/java/accord/utils/ReducingRangeMap.java | 157 ++++-
.../java/accord/utils/SearchableRangeList.java | 41 +-
.../src/main/java/accord/utils/SimpleBitSet.java | 6 +
.../src/main/java/accord/utils/SortedArrays.java | 12 +-
.../src/main/java/accord/utils/random/Picker.java | 122 ++++
.../random/Suppliers.java} | 42 +-
.../src/test/java/accord/burn/BurnTest.java | 20 +-
.../src/test/java/accord/burn/TopologyUpdates.java | 19 +-
.../coordinate/tracking/TrackerReconciler.java | 2 +-
.../src/test/java/accord/impl/TestAgent.java | 9 +-
.../src/test/java/accord/impl/basic/Cluster.java | 264 ++++++++-
.../accord/impl/basic/DelayedCommandStores.java | 23 +-
.../src/test/java/accord/impl/basic/NodeSink.java | 78 +--
.../basic/SimulatedDelayedExecutorService.java | 25 +-
.../src/test/java/accord/impl/list/ListAgent.java | 11 +-
.../src/test/java/accord/impl/list/ListRead.java | 45 +-
.../test/java/accord/impl/list/ListRequest.java | 15 +-
.../src/test/java/accord/impl/list/ListWrite.java | 9 +-
.../java/accord/local/ImmutableCommandTest.java | 1 +
.../test/java/accord/primitives/RangeDepsTest.java | 23 +
.../java/accord/topology/TopologyRandomizer.java | 15 +
.../test/java/accord/utils/RandomSourceTest.java | 128 +++++
.../main/java/accord/maelstrom/MaelstromAgent.java | 5 +
98 files changed, 4426 insertions(+), 1486 deletions(-)
diff --git a/accord-core/build.gradle b/accord-core/build.gradle
index 877bac65..d87e5000 100644
--- a/accord-core/build.gradle
+++ b/accord-core/build.gradle
@@ -134,3 +134,12 @@ tasks {
}
}
+task allTestJar(type: Jar) {
+ dependsOn testJar
+ archiveBaseName = project.name + '-test-all'
+ duplicatesStrategy = DuplicatesStrategy.EXCLUDE
+ from { configurations.testCompileClasspath.collect { it.isDirectory() ? it : zipTree(it) } }
+ from { configurations.testRuntimeClasspath.collect { it.isDirectory() ? it : zipTree(it) } }
+ from { sourceSets.test.output }
+ with jar
+}
\ No newline at end of file
diff --git a/accord-core/src/main/java/accord/api/Agent.java b/accord-core/src/main/java/accord/api/Agent.java
index 4a4460d1..f229ab82 100644
--- a/accord-core/src/main/java/accord/api/Agent.java
+++ b/accord-core/src/main/java/accord/api/Agent.java
@@ -63,6 +63,7 @@ public interface Agent extends UncaughtExceptionListener
* outside of Accord.
*/
default void onLocalBarrier(@Nonnull Seekables<?, ?> keysOrRanges, @Nonnull Timestamp executeAt) {}
+ void onStale(Timestamp staleSince, Ranges ranges);
@Override
void onUncaughtException(Throwable t);
diff --git a/accord-core/src/main/java/accord/api/ProgressLog.java b/accord-core/src/main/java/accord/api/ProgressLog.java
index 47244c17..d3572312 100644
--- a/accord-core/src/main/java/accord/api/ProgressLog.java
+++ b/accord-core/src/main/java/accord/api/ProgressLog.java
@@ -114,6 +114,11 @@ public interface ProgressLog
*/
void accepted(Command command, ProgressShard shard);
+ /**
+ * The transaction's executeAt is known. This may be enough to satisfy some waiting transactions.
+ */
+ void precommitted(Command command);
+
/**
* Has committed
*
diff --git a/accord-core/src/main/java/accord/api/Result.java b/accord-core/src/main/java/accord/api/Result.java
index 0509c476..15fbe217 100644
--- a/accord-core/src/main/java/accord/api/Result.java
+++ b/accord-core/src/main/java/accord/api/Result.java
@@ -26,14 +26,6 @@ import accord.primitives.ProgressToken;
*/
public interface Result extends Outcome
{
- Result APPLIED = new Result() { };
-
- Result INVALIDATED = new Result()
- {
- @Override
- public ProgressToken asProgressToken() { return ProgressToken.INVALIDATED; }
- };
-
@Override
default ProgressToken asProgressToken() { return ProgressToken.APPLIED; }
}
diff --git a/accord-core/src/main/java/accord/coordinate/Barrier.java b/accord-core/src/main/java/accord/coordinate/Barrier.java
index 2939c479..84885476 100644
--- a/accord-core/src/main/java/accord/coordinate/Barrier.java
+++ b/accord-core/src/main/java/accord/coordinate/Barrier.java
@@ -18,6 +18,7 @@
package accord.coordinate;
+import java.util.Objects;
import java.util.concurrent.ExecutionException;
import javax.annotation.Nonnull;
@@ -34,7 +35,6 @@ import accord.local.PreLoadContext;
import accord.local.SafeCommand;
import accord.local.SafeCommandStore;
import accord.local.SafeCommandStore.TestDep;
-import accord.local.SafeCommandStore.TestKind;
import accord.local.SafeCommandStore.TestTimestamp;
import accord.local.Status;
import accord.primitives.Routable.Domain;
@@ -45,6 +45,7 @@ import accord.utils.MapReduceConsume;
import accord.utils.async.AsyncResults;
import static accord.local.PreLoadContext.contextFor;
+import static accord.primitives.Txn.Kind.Kinds.Any;
import static accord.utils.Invariants.checkArgument;
import static accord.utils.Invariants.checkState;
import static accord.utils.async.AsyncChains.getUninterruptibly;
@@ -252,27 +253,28 @@ public class Barrier<S extends Seekables<?, ?>> extends AsyncResults.AbstractRes
@Override
public BarrierTxn apply(SafeCommandStore safeStore)
{
- BarrierTxn found = safeStore.mapReduceWithTerminate(
+ BarrierTxn found = safeStore.mapReduce(
seekables,
safeStore.ranges().allAfter(minEpoch),
// Barriers are trying to establish that committed transactions are applied before the barrier (or in this case just minEpoch)
// so all existing transaction types should ensure that at this point. An earlier txnid may have an executeAt that is after
// this barrier or the transaction we listen on and that is fine
- TestKind.Any,
+ Any,
TestTimestamp.EXECUTES_AFTER,
TxnId.minForEpoch(minEpoch),
TestDep.ANY_DEPS,
null,
Status.Committed,
Status.Applied,
- (keyOrRange, txnId, executeAt, status, barrierTxn) -> {
+ (p1, keyOrRange, txnId, executeAt, status, barrierTxn) -> {
if (keyOrRange.domain() == Domain.Key)
return new BarrierTxn(txnId, executeAt, status, keyOrRange.asKey());
return null;
},
null,
+ null,
// Take the first one we find, and call it good enough to wait on
- barrierTxn -> barrierTxn != null);
+ Objects::nonNull);
// It's not applied so add a listener to find out when it is applied
if (found != null && !found.status.equals(Status.Applied))
{
diff --git a/accord-core/src/main/java/accord/coordinate/CheckShards.java b/accord-core/src/main/java/accord/coordinate/CheckShards.java
index d9bb0f05..656bab71 100644
--- a/accord-core/src/main/java/accord/coordinate/CheckShards.java
+++ b/accord-core/src/main/java/accord/coordinate/CheckShards.java
@@ -111,7 +111,7 @@ public abstract class CheckShards<U extends Unseekables<?>> extends ReadCoordina
@Override
protected void finishOnExhaustion()
{
- if (merged != null && merged.isTruncated()) finishOnFailure(new Truncated(txnId, null), false);
+ if (merged != null && merged.isTruncatedResponse()) finishOnFailure(new Truncated(txnId, null), false);
else super.finishOnExhaustion();
}
}
diff --git a/accord-core/src/main/java/accord/coordinate/CoordinateNoOp.java b/accord-core/src/main/java/accord/coordinate/CoordinateNoOp.java
index 84c3e465..bc25373e 100644
--- a/accord-core/src/main/java/accord/coordinate/CoordinateNoOp.java
+++ b/accord-core/src/main/java/accord/coordinate/CoordinateNoOp.java
@@ -76,14 +76,16 @@ public class CoordinateNoOp extends CoordinatePreAccept<Timestamp>
}
else
{
- new Propose<Timestamp>(node, topologies, Ballot.ZERO, txnId, txn, route, executeAt, Deps.NONE, this)
+ Deps preacceptDeps = Deps.merge(successes, ok -> ok.deps);
+ new Propose<Timestamp>(node, topologies, Ballot.ZERO, txnId, txn, route, executeAt, preacceptDeps, this)
{
@Override
void onAccepted()
{
Writes writes = txn.execute(txnId, txnId, null);
Result result = txn.result(txnId, executeAt, null);
- Apply.sendMaximal(node, txnId, route, txn, executeAt, deps, writes, result);
+ Deps acceptDeps = Deps.merge(this.acceptOks, ok -> ok.deps);
+ Apply.sendMaximal(node, txnId, route, txn, executeAt, acceptDeps, writes, result);
accept(executeAt, null);
}
}.start();
diff --git a/accord-core/src/main/java/accord/coordinate/FetchData.java b/accord-core/src/main/java/accord/coordinate/FetchData.java
index 0188a0a7..1ece4675 100644
--- a/accord-core/src/main/java/accord/coordinate/FetchData.java
+++ b/accord-core/src/main/java/accord/coordinate/FetchData.java
@@ -31,7 +31,6 @@ import accord.utils.Invariants;
import javax.annotation.Nullable;
import static accord.coordinate.Infer.InvalidateAndCallback.locallyInvalidateAndCallback;
-import static accord.messages.CheckStatus.WithQuorum.NoQuorum;
import static accord.primitives.Route.castToRoute;
import static accord.primitives.Route.isRoute;
@@ -44,26 +43,27 @@ public class FetchData extends CheckShards<Route<?>>
{
public static Object fetch(Known fetch, Node node, TxnId txnId, Unseekables<?> someUnseekables, BiConsumer<? super Known, Throwable> callback)
{
- return fetch(fetch, node, txnId, someUnseekables, null, callback);
+ return fetch(fetch, node, txnId, someUnseekables, null, null, callback);
}
- public static Object fetch(Known fetch, Node node, TxnId txnId, Unseekables<?> someUnseekables, @Nullable Timestamp executeAt, BiConsumer<? super Known, Throwable> callback)
+ public static Object fetch(Known fetch, Node node, TxnId txnId, Unseekables<?> someUnseekables, @Nullable EpochSupplier forLocalEpoch, @Nullable Timestamp executeAt, BiConsumer<? super Known, Throwable> callback)
{
- if (someUnseekables.kind().isRoute()) return fetch(fetch, node, txnId, castToRoute(someUnseekables), executeAt, callback);
- else return fetchViaSomeRoute(fetch, node, txnId, someUnseekables, executeAt, callback);
+ if (someUnseekables.kind().isRoute()) return fetch(fetch, node, txnId, castToRoute(someUnseekables), forLocalEpoch, executeAt, callback);
+ else return fetchViaSomeRoute(fetch, node, txnId, someUnseekables, forLocalEpoch, executeAt, callback);
}
- public static Object fetch(Known fetch, Node node, TxnId txnId, Route<?> route, @Nullable Timestamp executeAt, BiConsumer<? super Known, Throwable> callback)
+ public static Object fetch(Known fetch, Node node, TxnId txnId, Route<?> route, @Nullable EpochSupplier forLocalEpoch, @Nullable Timestamp executeAt, BiConsumer<? super Known, Throwable> callback)
{
long srcEpoch = fetch.fetchEpoch(txnId, executeAt);
if (!node.topology().hasEpoch(srcEpoch))
- return node.topology().awaitEpoch(srcEpoch).map(ignore -> fetch(fetch, node, txnId, route, executeAt, callback)).beginAsResult();
+ return node.topology().awaitEpoch(srcEpoch).map(ignore -> fetch(fetch, node, txnId, route, forLocalEpoch, executeAt, callback)).beginAsResult();
Invariants.checkArgument(node.topology().hasEpoch(srcEpoch), "Unknown epoch %d, latest known is %d", srcEpoch, node.epoch());
- Ranges ranges = node.topology().localRangesForEpochs(txnId.epoch(), srcEpoch);
+ long toLocalEpoch = Math.max(srcEpoch, forLocalEpoch == null ? 0 : forLocalEpoch.epoch());
+ Ranges ranges = node.topology().localRangesForEpochs(txnId.epoch(), toLocalEpoch);
if (!route.covers(ranges))
{
- return fetchWithIncompleteRoute(fetch, node, txnId, route, executeAt, callback);
+ return fetchWithIncompleteRoute(fetch, node, txnId, route, forLocalEpoch, executeAt, callback);
}
else
{
@@ -72,7 +72,7 @@ public class FetchData extends CheckShards<Route<?>>
}
@SuppressWarnings({"unchecked", "rawtypes"})
- private static Object fetchViaSomeRoute(Known fetch, Node node, TxnId txnId, Unseekables<?> someUnseekables, @Nullable Timestamp executeAt, BiConsumer<? super Known, Throwable> callback)
+ private static Object fetchViaSomeRoute(Known fetch, Node node, TxnId txnId, Unseekables<?> someUnseekables, @Nullable EpochSupplier forLocalEpoch, @Nullable Timestamp executeAt, BiConsumer<? super Known, Throwable> callback)
{
return FindSomeRoute.findSomeRoute(node, txnId, someUnseekables, (foundRoute, fail) -> {
if (fail != null) callback.accept(null, fail);
@@ -91,7 +91,7 @@ public class FetchData extends CheckShards<Route<?>>
Route<?> route = foundRoute.route;
if (isRoute(someUnseekables))
route = Route.merge(route, (Route)someUnseekables);
- fetch(fetch, node, txnId, route, executeAt, callback);
+ fetch(fetch, node, txnId, route, forLocalEpoch, executeAt, callback);
}
});
}
@@ -118,25 +118,27 @@ public class FetchData extends CheckShards<Route<?>>
case Erased:
case WasApply:
case Apply:
+ // TODO (required): we may now be stale
callback.accept(found, null);
}
}
- private static Object fetchWithIncompleteRoute(Known fetch, Node node, TxnId txnId, Route<?> someRoute, @Nullable Timestamp executeAt, BiConsumer<? super Known, Throwable> callback)
+ private static Object fetchWithIncompleteRoute(Known fetch, Node node, TxnId txnId, Route<?> someRoute, @Nullable EpochSupplier forLocalEpoch, @Nullable Timestamp executeAt, BiConsumer<? super Known, Throwable> callback)
{
long srcEpoch = fetch.fetchEpoch(txnId, executeAt);
Invariants.checkArgument(node.topology().hasEpoch(srcEpoch), "Unknown epoch %d, latest known is %d", srcEpoch, node.epoch());
return FindRoute.findRoute(node, txnId, someRoute.withHomeKey(), (foundRoute, fail) -> {
if (fail != null) callback.accept(null, fail);
- else if (foundRoute == null) fetchViaSomeRoute(fetch, node, txnId, someRoute, executeAt, callback);
- else fetch(fetch, node, txnId, foundRoute.route, foundRoute.executeAt, callback);
+ else if (foundRoute == null) fetchViaSomeRoute(fetch, node, txnId, someRoute, forLocalEpoch, executeAt, callback);
+ else fetch(fetch, node, txnId, foundRoute.route, forLocalEpoch, foundRoute.executeAt, callback);
});
}
- public static Object fetch(Known fetch, Node node, TxnId txnId, FullRoute<?> route, @Nullable Timestamp executeAt, BiConsumer<? super Known, Throwable> callback)
+ public static Object fetch(Known fetch, Node node, TxnId txnId, FullRoute<?> route, @Nullable EpochSupplier forLocalEpoch, @Nullable Timestamp executeAt, BiConsumer<? super Known, Throwable> callback)
{
return node.awaitEpoch(executeAt).map(ignore -> {
- Ranges ranges = node.topology().localRangesForEpochs(txnId.epoch(), fetch.fetchEpoch(txnId, executeAt));
+ long toEpoch = Math.max(fetch.fetchEpoch(txnId, executeAt), forLocalEpoch == null ? 0 : forLocalEpoch.epoch());
+ Ranges ranges = node.topology().localRangesForEpochs(txnId.epoch(), toEpoch);
return fetchInternal(ranges, fetch, node, txnId, route.sliceStrict(ranges), executeAt, callback);
}).beginAsResult();
}
@@ -200,7 +202,7 @@ public class FetchData extends CheckShards<Route<?>>
protected boolean isSufficient(Route<?> scope, CheckStatus.CheckStatusOk ok)
{
- return target.isSatisfiedBy(((CheckStatusOkFull)ok).sufficientFor(scope.participants(), NoQuorum));
+ return target.isSatisfiedBy(ok.knownFor(scope.participants()));
}
@Override
@@ -216,6 +218,7 @@ public class FetchData extends CheckShards<Route<?>>
if (success == ReadCoordinator.Success.Success)
Invariants.checkState(isSufficient(merged), "Status %s is not sufficient", merged);
+ // TODO (expected): should we automatically trigger a new fetch if we find executeAt but did not request enough information? would be more rob ust
Propagate.propagate(node, txnId, sourceEpoch, success.withQuorum, route(), target, (CheckStatusOkFull) merged, callback);
}
}
diff --git a/accord-core/src/main/java/accord/coordinate/FindRoute.java b/accord-core/src/main/java/accord/coordinate/FindRoute.java
index f85ba888..6997c836 100644
--- a/accord-core/src/main/java/accord/coordinate/FindRoute.java
+++ b/accord-core/src/main/java/accord/coordinate/FindRoute.java
@@ -21,7 +21,6 @@ package accord.coordinate;
import java.util.function.BiConsumer;
import accord.local.Node;
-import accord.local.Status;
import accord.messages.CheckStatus.CheckStatusOk;
import accord.messages.CheckStatus.IncludeInfo;
import accord.primitives.*;
@@ -47,7 +46,7 @@ public class FindRoute extends CheckShards<Route<?>>
public Result(CheckStatusOk ok)
{
this.route = Route.castToFullRoute(ok.route);
- this.executeAt = ok.saveStatus.status.compareTo(Status.PreCommitted) >= 0 ? ok.executeAt : null;
+ this.executeAt = ok.maxKnown().executeAt.isDecidedAndKnownToExecute() ? ok.executeAt : null;
}
}
diff --git a/accord-core/src/main/java/accord/coordinate/FindSomeRoute.java b/accord-core/src/main/java/accord/coordinate/FindSomeRoute.java
index 2babeac7..19060ed9 100644
--- a/accord-core/src/main/java/accord/coordinate/FindSomeRoute.java
+++ b/accord-core/src/main/java/accord/coordinate/FindSomeRoute.java
@@ -75,6 +75,6 @@ public class FindSomeRoute extends CheckShards<Unseekables<?>>
{
if (failure != null) callback.accept(null, failure);
else if (merged == null) callback.accept(new Result(null, Nothing, success.withQuorum), null);
- else callback.accept(new Result(merged.route, merged.inferredOrKnown(success.withQuorum), success.withQuorum), null);
+ else callback.accept(new Result(merged.route, merged.finish(this.route, success.withQuorum).knownFor(this.route), success.withQuorum), null);
}
}
diff --git a/accord-core/src/main/java/accord/coordinate/Infer.java b/accord-core/src/main/java/accord/coordinate/Infer.java
index dc42d834..ef04b8f3 100644
--- a/accord-core/src/main/java/accord/coordinate/Infer.java
+++ b/accord-core/src/main/java/accord/coordinate/Infer.java
@@ -26,11 +26,11 @@ import accord.local.Command;
import accord.local.Commands;
import accord.local.Node;
import accord.local.PreLoadContext;
+import accord.local.RedundantStatus;
import accord.local.SafeCommand;
import accord.local.SafeCommandStore;
-import accord.local.SaveStatus;
import accord.local.Status;
-import accord.messages.CheckStatus;
+import accord.local.Status.Known;
import accord.primitives.Participants;
import accord.primitives.Ranges;
import accord.primitives.Route;
@@ -40,17 +40,164 @@ import accord.primitives.Unseekables;
import accord.utils.Invariants;
import accord.utils.MapReduceConsume;
+import static accord.coordinate.Infer.InvalidIf.IfPreempted;
+import static accord.coordinate.Infer.InvalidIf.IfQuorum;
+import static accord.coordinate.Infer.InvalidIf.NotKnown;
+import static accord.coordinate.Infer.InvalidIfNot.IfUnknown;
+import static accord.coordinate.Infer.InvalidIfNot.IfUndecided;
import static accord.local.PreLoadContext.contextFor;
import static accord.local.Status.Durability.Majority;
-import static accord.local.Status.PreAccepted;
import static accord.local.Status.PreApplied;
import static accord.local.Status.PreCommitted;
-import static accord.messages.CheckStatus.WithQuorum.HasQuorum;
import static accord.primitives.Route.castToRoute;
import static accord.primitives.Route.isRoute;
public class Infer
{
+ public enum InvalidIfNot
+ {
+ /**
+ * There is no information to suggest the command is invalid
+ */
+ NotKnownToBeInvalid(NotKnown, NotKnown),
+
+ /**
+ * If the command has not been preaccepted on a majority of any shard and
+ * the command's original coordinator had been preempted prior to all responses we rely upon
+ * (so we are not racing with it)
+ */
+ IfUnknownAndPreempted(IfPreempted, NotKnown),
+
+ /**
+ * If the command has not had its execution timestamp agreed on any shard and
+ * the command's original coordinator had been preempted prior to all responses we rely upon
+ * (so we are not racing with it)
+ */
+ IfUndecidedAndPreempted(IfPreempted, IfPreempted),
+
+ /**
+ * If the command has not been preaccepted on a majority of any shard
+ */
+ IfUnknown(IfQuorum, NotKnown),
+
+ /**
+ * If the command has not had its execution timestamp agreed on any shard and
+ * the command's original coordinator had been preempted prior to all responses we rely upon
+ * (so we are not racing with it)
+ */
+ IfUnknownOrIfUndecidedAndPreempted(IfQuorum, IfPreempted),
+
+ /**
+ * If the command has not had its execution timestamp agreed on any shard
+ */
+ IfUndecided(IfQuorum, IfQuorum);
+
+ final InvalidIf unknown, undecided;
+ private static final InvalidIfNot[] LOOKUP;
+ private static final int invalidIfs = InvalidIf.values().length;
+
+ static
+ {
+ LOOKUP = new InvalidIfNot[invalidIfs * invalidIfs];
+ InvalidIfNot[] invalidIfNot = InvalidIfNot.values();
+ for (InvalidIfNot ifNot : invalidIfNot)
+ LOOKUP[ifNot.unknown.ordinal() * invalidIfs + ifNot.undecided.ordinal()] = ifNot;
+ }
+
+ InvalidIfNot(InvalidIf unknown, InvalidIf undecided)
+ {
+ this.unknown = unknown;
+ this.undecided = undecided;
+ }
+
+ public static boolean isMax(InvalidIfNot that)
+ {
+ return that == IfUndecided;
+ }
+
+ public InvalidIfNot atLeast(InvalidIfNot that)
+ {
+ return lookup(atLeast(this.unknown, that.unknown), atLeast(this.undecided, that.undecided));
+ }
+
+ public InvalidIfNot reduce(InvalidIfNot that)
+ {
+ return lookup(reduce(this.unknown, that.unknown), reduce(this.undecided, that.undecided));
+ }
+
+ private InvalidIfNot lookup(InvalidIf unknown, InvalidIf undecided)
+ {
+ return LOOKUP[unknown.ordinal() * invalidIfs + undecided.ordinal()];
+ }
+
+ private static InvalidIf atLeast(InvalidIf a, InvalidIf b)
+ {
+ if (a == b) return a;
+ return IfPreempted;
+ }
+
+ private static InvalidIf reduce(InvalidIf a, InvalidIf b)
+ {
+ return a.compareTo(b) <= 0 ? a : b;
+ }
+
+ public boolean inferInvalidWithQuorum(IsPreempted isPreempted, Known known)
+ {
+ return inferInvalidWithQuorum(undecided, isPreempted, !known.isDecided())
+ || inferInvalidWithQuorum(unknown, isPreempted, !known.hasDefinitionBeenKnown());
+ }
+
+ private static boolean inferInvalidWithQuorum(InvalidIf invalidIf, IsPreempted isPreempted, boolean hasCondition)
+ {
+ if (!hasCondition)
+ return false;
+
+ switch (invalidIf)
+ {
+ default: throw new AssertionError("Unhandled InvalidIf: " + invalidIf);
+ case NotKnown: break;
+ case IfQuorum: return true;
+ case IfPreempted:
+ if (isPreempted == IsPreempted.Preempted)
+ return true;
+ }
+
+ return false;
+ }
+ }
+
+ enum InvalidIf
+ {
+ NotKnown,
+
+ /**
+ * We did not have a quorum of responses with the associated lower bound, so we require that the command has been preempted at a quorum
+ */
+ IfPreempted,
+
+ /**
+ * If we obtain a quorum of responses with the associated lower bound, we can infer the command is invalidated if it has not been witnessed at the lower bound
+ */
+ IfQuorum
+ }
+
+ // only valid with a quorum of responses
+ public enum IsPreempted
+ {
+ NotPreempted, MaybePreempted, Preempted;
+
+ public IsPreempted merge(IsPreempted that)
+ {
+ if (this == that) return this;
+ return MaybePreempted;
+ }
+
+ public IsPreempted validForBoth(IsPreempted that)
+ {
+ return this.compareTo(that) <= 0 ? this : that;
+ }
+ }
+
private static abstract class CleanupAndCallback<T> implements MapReduceConsume<SafeCommandStore, Void>
{
final Node node;
@@ -121,36 +268,39 @@ public class Infer
}
}
- static class EraseNonParticipatingAndCallback<T> extends CleanupAndCallback<T>
+ /**
+ * Erase if it is safe to do so, i.e. if Infer.safeToCleanup permits it.
+ */
+ static class SafeEraseAndCallback<T> extends CleanupAndCallback<T>
{
- private EraseNonParticipatingAndCallback(Node node, TxnId txnId, Unseekables<?> someUnseekables, T param, BiConsumer<T, Throwable> callback)
+ private SafeEraseAndCallback(Node node, TxnId txnId, Unseekables<?> someUnseekables, T param, BiConsumer<T, Throwable> callback)
{
super(node, txnId, someUnseekables, param, callback);
}
- public static <T> void eraseNonParticipatingAndCallback(Node node, TxnId txnId, Unseekables<?> someUnseekables, T param, BiConsumer<T, Throwable> callback)
+ public static <T> void safeEraseAndCallback(Node node, TxnId txnId, Unseekables<?> someUnseekables, T param, BiConsumer<T, Throwable> callback)
{
if (!Route.isRoute(someUnseekables)) callback.accept(param, null);
- else new EraseNonParticipatingAndCallback<>(node, txnId, someUnseekables, param, callback).start();
+ else new SafeEraseAndCallback<>(node, txnId, someUnseekables, param, callback).start();
}
@Override
Void apply(SafeCommandStore safeStore, SafeCommand safeCommand)
{
- // we're applying an invalidation, so the record will not be cleaned up until the whole range is truncated
Command command = safeCommand.current();
+ // TODO (required): introduce a special form of Erased where we do not imply the phase is "Cleanup"
if (!command.hasBeen(PreApplied) && safeToCleanup(safeStore, command, Route.castToRoute(someUnseekables), null))
Commands.setErased(safeStore, safeCommand);
return null;
}
}
- public static Status invalidIfNotAtLeast(SafeCommandStore safeStore, TxnId txnId, Unseekables<?> query)
+ public static InvalidIfNot invalidIfNotAtLeast(SafeCommandStore safeStore, TxnId txnId, Unseekables<?> query)
{
if (safeStore.commandStore().globalDurability(txnId).compareTo(Majority) >= 0)
{
Unseekables<?> preacceptsWith = isRoute(query) ? castToRoute(query).withHomeKey() : query;
- return safeStore.commandStore().isRejectedIfNotPreAccepted(txnId, preacceptsWith) ? PreAccepted : PreCommitted;
+ return safeStore.commandStore().isRejectedIfNotPreAccepted(txnId, preacceptsWith) ? IfUnknown : IfUndecided;
}
// TODO (expected, consider): should we force this to be a Route or a Participants?
@@ -159,49 +309,51 @@ public class Infer
Participants<?> participants = castToRoute(query).participants();
// TODO (desired): limit to local participants to avoid O(n2) work across cluster
if (safeStore.commandStore().durableBefore().isSomeShardDurable(txnId, participants, Majority))
- return PreCommitted;
+ return IfUndecided;
}
if (safeStore.commandStore().durableBefore().isUniversal(txnId, safeStore.ranges().allAt(txnId.epoch())))
- return PreCommitted;
+ return IfUndecided;
- return Status.NotDefined;
+ return InvalidIfNot.NotKnownToBeInvalid;
}
- public static boolean inferInvalidated(CheckStatus.WithQuorum withQuorum, Status invalidIfNotAtLeast, SaveStatus saveStatus, SaveStatus maxSaveStatus)
+ public enum YesNoMaybe
{
- if (saveStatus == SaveStatus.Invalidated)
- return true;
-
- if (withQuorum != HasQuorum)
- return false;
-
- // should not be possible to reach a quorum without finding the definition unless cleanup is in progress
- Invariants.checkState(saveStatus != SaveStatus.Accepted || maxSaveStatus.phase == Status.Phase.Cleanup);
-
- if (saveStatus.status.compareTo(invalidIfNotAtLeast) < 0)
- return true;
-
- return invalidIfNotAtLeast.compareTo(PreAccepted) >= 0 && saveStatus == SaveStatus.AcceptedInvalidate;
+ No, Maybe, Yes
}
public static boolean safeToCleanup(SafeCommandStore safeStore, Command command, Route<?> fetchedWith, @Nullable Timestamp executeAt)
{
Invariants.checkArgument(fetchedWith != null || command.route() != null);
TxnId txnId = command.txnId();
- if (command.is(Status.NotDefined))
- return !command.saveStatus().isUninitialised();
+ if (command.route() == null || !fetchedWith.covers(safeStore.ranges().allAt(txnId.epoch())))
+ return false;
- if (command.route() != null || fetchedWith.covers(safeStore.ranges().allAt(txnId.epoch())))
- {
- Route<?> route = command.route();
- if (route == null) route = fetchedWith;
+ Route<?> route = command.route();
+ if (route == null) route = fetchedWith;
+
+ // TODO (required): is it safe to cleanup without an executeAt?
+ executeAt = command.executeAtIfKnown(Timestamp.nonNullOrMax(executeAt, txnId));
+ Ranges coordinateRanges = safeStore.ranges().coordinates(txnId);
+ Ranges acceptRanges = executeAt.epoch() == txnId.epoch() ? coordinateRanges : safeStore.ranges().allBetween(txnId, executeAt);
+ if (!route.participatesIn(coordinateRanges) && !route.participatesIn(acceptRanges))
+ return true;
- executeAt = command.executeAtIfKnown(Timestamp.nonNullOrMax(executeAt, txnId));
- Ranges coordinateRanges = safeStore.ranges().coordinates(txnId);
- Ranges acceptRanges = executeAt.epoch() == txnId.epoch() ? coordinateRanges : safeStore.ranges().allBetween(txnId, executeAt);
- return !route.participatesIn(coordinateRanges) && !route.participatesIn(acceptRanges);
+ RedundantStatus status = safeStore.commandStore().redundantBefore().status(txnId, executeAt, route.participants());
+ switch (status)
+ {
+ default: throw new AssertionError("Unhandled RedundantStatus: " + status);
+ case NOT_OWNED:
+ case LIVE:
+ case REDUNDANT_PRE_BOOTSTRAP_OR_STALE:
+ case PARTIALLY_PRE_BOOTSTRAP_OR_STALE:
+ return false;
+ case LOCALLY_REDUNDANT:
+ case SHARD_REDUNDANT:
+ Invariants.checkState(!command.hasBeen(PreCommitted));
+ case PRE_BOOTSTRAP_OR_STALE:
+ return true;
}
- return false;
}
}
diff --git a/accord-core/src/main/java/accord/coordinate/MaybeRecover.java b/accord-core/src/main/java/accord/coordinate/MaybeRecover.java
index ab4ea919..48c3754f 100644
--- a/accord-core/src/main/java/accord/coordinate/MaybeRecover.java
+++ b/accord-core/src/main/java/accord/coordinate/MaybeRecover.java
@@ -21,7 +21,6 @@ package accord.coordinate;
import java.util.function.BiConsumer;
import accord.local.Status.Known;
-import accord.messages.CheckStatus.WithQuorum;
import accord.primitives.*;
import accord.utils.Invariants;
@@ -29,9 +28,8 @@ import accord.local.Node;
import accord.messages.CheckStatus.CheckStatusOk;
import accord.messages.CheckStatus.IncludeInfo;
-import static accord.coordinate.Infer.EraseNonParticipatingAndCallback.eraseNonParticipatingAndCallback;
+import static accord.coordinate.Infer.SafeEraseAndCallback.safeEraseAndCallback;
import static accord.coordinate.Infer.InvalidateAndCallback.locallyInvalidateAndCallback;
-import static accord.utils.Functions.reduceNonNull;
/**
* A result of null indicates the transaction is globally persistent
@@ -79,45 +77,47 @@ public class MaybeRecover extends CheckShards<Route<?>>
else
{
Invariants.checkState(merged != null);
- Known known = merged.saveStatus.known;
- Route<?> someRoute = reduceNonNull(Route::union, (Route)this.route, merged.route);
+ CheckStatusOk full = merged.finish(this.route, success.withQuorum);
+ Known known = full.maxKnown();
+ Route<?> someRoute = full.route;
switch (known.outcome)
{
default: throw new AssertionError();
case Unknown:
- if (known.canProposeInvalidation() && !Route.isFullRoute(merged.route))
+ if (known.canProposeInvalidation() && !Route.isFullRoute(full.route))
{
// for correctness reasons, we have not necessarily preempted the initial pre-accept round and
// may have raced with it, so we must attempt to recover anything we see pre-accepted.
Invalidate.invalidate(node, txnId, someRoute, callback);
break;
}
+ // fall through otherwise to recovery
case Apply:
// we have included the home key, and one that witnessed the definition has responded, so it should also know the full route
- if (hasMadeProgress(merged))
+ if (hasMadeProgress(full))
{
- callback.accept(merged.toProgressToken(), null);
+ callback.accept(full.toProgressToken(), null);
}
else
{
- Invariants.checkState(Route.isFullRoute(merged.route), "Require a full route but given %s", merged.route);
- node.recover(txnId, Route.castToFullRoute(merged.route)).addCallback(callback);
+ Invariants.checkState(Route.isFullRoute(someRoute), "Require a full route but given %s", full.route);
+ node.recover(txnId, Route.castToFullRoute(someRoute)).addCallback(callback);
}
break;
case WasApply:
- callback.accept(merged.toProgressToken(), null);
+ callback.accept(full.toProgressToken(), null);
break;
case Invalidated:
- locallyInvalidateAndCallback(node, txnId, someRoute, merged.toProgressToken(), callback);
+ locallyInvalidateAndCallback(node, txnId, someRoute, full.toProgressToken(), callback);
+ break;
case Erased:
- WithQuorum withQuorum = success.withQuorum;
- if (!merged.inferInvalidated(withQuorum)) eraseNonParticipatingAndCallback(node, txnId, someRoute, merged.toProgressToken(), callback);
- else locallyInvalidateAndCallback(node, txnId, someRoute, merged.toProgressToken(), callback);
+ Invariants.checkState(!full.knownFor(route.participants()).isInvalidated());
+ safeEraseAndCallback(node, txnId, someRoute, full.toProgressToken(), callback);
}
}
}
diff --git a/accord-core/src/main/java/accord/coordinate/ReadCoordinator.java b/accord-core/src/main/java/accord/coordinate/ReadCoordinator.java
index 4cb55e83..d8f1bfb0 100644
--- a/accord-core/src/main/java/accord/coordinate/ReadCoordinator.java
+++ b/accord-core/src/main/java/accord/coordinate/ReadCoordinator.java
@@ -144,11 +144,7 @@ public abstract class ReadCoordinator<Reply extends accord.messages.Reply> exten
break;
case ApprovePartial:
- Ranges unavailable = unavailable(reply);
- RequestStatus result = recordPartialReadSuccess(from, unavailable);
- if (result == RequestStatus.Failed && failure == null)
- failure = new RangeUnavailable(unavailable, txnId);
- handle(result);
+ handle(recordPartialReadSuccess(from, unavailable(reply)));
break;
}
}
@@ -177,11 +173,9 @@ public abstract class ReadCoordinator<Reply extends accord.messages.Reply> exten
@Override
public void onCallbackFailure(Id from, Throwable failure)
{
+ node.agent().onUncaughtException(failure);
if (isDone)
- {
- node.agent().onUncaughtException(failure);
return;
- }
if (this.failure != null)
failure.addSuppressed(this.failure);
@@ -218,7 +212,18 @@ public abstract class ReadCoordinator<Reply extends accord.messages.Reply> exten
{
Invariants.checkState(!isDone);
if (failure == null)
- failure = new Exhausted(txnId, null);
+ {
+ Ranges unavailable = Ranges.EMPTY;
+ Ranges exhausted = Ranges.EMPTY;
+ for (ReadShardTracker tracker : trackers)
+ {
+ if (tracker.hasSucceeded())
+ continue;
+ if (tracker.unavailable() != null) unavailable = unavailable.with(tracker.unavailable());
+ else exhausted = exhausted.with(Ranges.of(tracker.shard.range));
+ }
+ failure = new Exhausted(txnId, null, (unavailable.isEmpty() ? "" : "unavailable: " + unavailable + (exhausted.isEmpty() ? "" : "; ")) + (exhausted.isEmpty() ? "" : "no response: " + exhausted));
+ }
finishOnFailure();
}
diff --git a/accord-core/src/main/java/accord/coordinate/Recover.java b/accord-core/src/main/java/accord/coordinate/Recover.java
index df56b791..add3e88a 100644
--- a/accord-core/src/main/java/accord/coordinate/Recover.java
+++ b/accord-core/src/main/java/accord/coordinate/Recover.java
@@ -139,6 +139,9 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
private Recover(Node node, Ballot ballot, TxnId txnId, Txn txn, FullRoute<?> route, BiConsumer<Outcome, Throwable> callback, Topologies topologies)
{
+ // TODO (required, correctness): we may have to contact all epochs to ensure we spot any future transaction that might not have taken us as dependency?
+ // or we need an exclusive sync point covering us and closing out the old epoch before recovering;
+ // or we need to manage dependencies for ranges we don't own in future epochs; this might be simplest
this.node = node;
this.ballot = ballot;
this.txnId = txnId;
diff --git a/accord-core/src/main/java/accord/coordinate/RecoverWithRoute.java b/accord-core/src/main/java/accord/coordinate/RecoverWithRoute.java
index dfcc3f0c..32a0797c 100644
--- a/accord-core/src/main/java/accord/coordinate/RecoverWithRoute.java
+++ b/accord-core/src/main/java/accord/coordinate/RecoverWithRoute.java
@@ -33,19 +33,19 @@ import accord.primitives.Ballot;
import accord.primitives.Deps;
import accord.primitives.FullRoute;
import accord.primitives.PartialRoute;
+import accord.primitives.Participants;
import accord.primitives.Ranges;
import accord.primitives.Route;
import accord.primitives.Txn;
import accord.primitives.TxnId;
-import accord.primitives.Unseekables;
import accord.topology.Topologies;
import accord.utils.Invariants;
import javax.annotation.Nullable;
+import static accord.local.Status.Durability.Majority;
+import static accord.local.Status.KnownDeps.DepsKnown;
import static accord.local.Status.KnownExecuteAt.ExecuteAtKnown;
import static accord.local.Status.Outcome.Apply;
-import static accord.messages.CheckStatus.WithQuorum.HasQuorum;
-import static accord.messages.CheckStatus.WithQuorum.NoQuorum;
import static accord.primitives.ProgressToken.APPLIED;
import static accord.primitives.ProgressToken.INVALIDATED;
import static accord.primitives.ProgressToken.TRUNCATED;
@@ -120,7 +120,7 @@ public class RecoverWithRoute extends CheckShards<FullRoute<?>>
protected boolean isSufficient(Route<?> route, CheckStatusOk ok)
{
CheckStatusOkFull full = (CheckStatusOkFull)ok;
- Known sufficientTo = full.sufficientFor(route.participants(), NoQuorum);
+ Known sufficientTo = full.knownFor(route.participants());
if (!sufficientTo.isDefinitionKnown())
return false;
@@ -140,8 +140,8 @@ public class RecoverWithRoute extends CheckShards<FullRoute<?>>
return;
}
- CheckStatusOkFull full = (CheckStatusOkFull) this.merged;
- Known known = full.sufficientFor(route.participants(), success == Success.Quorum ? HasQuorum : NoQuorum);
+ CheckStatusOkFull full = ((CheckStatusOkFull) this.merged).finish(route, success.withQuorum);
+ Known known = full.knownFor(route.participants());
switch (known.outcome)
{
@@ -152,38 +152,61 @@ public class RecoverWithRoute extends CheckShards<FullRoute<?>>
Txn txn = full.partialTxn.reconstitute(route);
Recover.recover(node, txnId, txn, route, callback);
}
- else
+ else if (!known.definition.isOrWasKnown())
{
+ // TODO (required): this logic should be put in Infer, alongside any similar inferences in Recover
if (witnessedByInvalidation != null && witnessedByInvalidation.compareTo(Status.PreAccepted) > 0)
throw new IllegalStateException("We previously invalidated, finding a status that should be recoverable");
Invalidate.invalidate(node, txnId, route, witnessedByInvalidation != null, callback);
}
+ else
+ {
+ callback.accept(full.toProgressToken(), null);
+ }
break;
case WasApply:
case Apply:
- if (!known.isDefinitionKnown() || known.executeAt != ExecuteAtKnown || known.outcome != Apply)
+ if (!known.isDefinitionKnown())
{
- if (!full.truncated.isEmpty() && known.executeAt == ExecuteAtKnown)
+ // TODO (expected): if we determine new durability, propagate it
+ CheckStatusOkFull propagate;
+ if (full.isTruncatedResponse())
{
// we might have only part of the full transaction, and a shard may have truncated;
// in this case we want to skip straight to apply, but only for the shards that haven't truncated
- Unseekables<?> sendTo = route.subtract(full.truncated);
+ Participants<?> sendTo = route.participants().subtract(full.truncatedResponse());
if (!sendTo.isEmpty())
{
- Invariants.checkState(full.committedDeps.covering.containsAll(sendTo));
- Invariants.checkState(full.partialTxn.covering().containsAll(sendTo));
- Persist.persistPartialMaximal(node, txnId, sendTo, route, full.partialTxn, full.executeAt, full.committedDeps, full.writes, full.result);
+ known = full.knownFor(sendTo);
+ // we might not know the Apply outcome because we might have raced with truncation on one shard, and the original apply on another,
+ // so that we know to WasApply, but not
+ if (known.executeAt == ExecuteAtKnown && known.deps == DepsKnown && known.outcome == Apply)
+ {
+ Invariants.checkState(full.committedDeps.covering.containsAll(sendTo));
+ Invariants.checkState(full.partialTxn.covering().containsAll(sendTo));
+ Persist.persistPartialMaximal(node, txnId, sendTo, route, full.partialTxn, full.executeAt, full.committedDeps, full.writes, full.result);
+ }
+ propagate = full;
}
+ else
+ {
+ // TODO (expected): tighten up / centralise this implication, perhaps in Infer
+ propagate = full.merge(Majority);
+ }
+ }
+ else
+ {
+ propagate = full;
}
- Propagate.propagate(node, txnId, sourceEpoch, success.withQuorum, route, null, full, (s, f) -> callback.accept(f == null ? full.toProgressToken() : null, f));
+ Propagate.propagate(node, txnId, sourceEpoch, success.withQuorum, route, null, propagate, (s, f) -> callback.accept(f == null ? propagate.toProgressToken() : null, f));
break;
}
// TODO (required): might not be able to fully recover transaction - may only have enough for local shard
Txn txn = full.partialTxn.reconstitute(route);
- if (known.deps.hasDecidedDeps())
+ if (known.executeAt.isDecidedAndKnownToExecute() && known.deps.hasDecidedDeps() && known.outcome == Apply)
{
Deps deps = full.committedDeps.reconstitute(route());
node.withEpoch(full.executeAt.epoch(), () -> {
diff --git a/accord-core/src/main/java/accord/coordinate/RecoverWithSomeRoute.java b/accord-core/src/main/java/accord/coordinate/RecoverWithSomeRoute.java
index d893ae80..a5d4883b 100644
--- a/accord-core/src/main/java/accord/coordinate/RecoverWithSomeRoute.java
+++ b/accord-core/src/main/java/accord/coordinate/RecoverWithSomeRoute.java
@@ -89,7 +89,7 @@ public class RecoverWithSomeRoute extends CheckShards<Route<?>> implements BiCon
callback.accept(null, new IllegalStateException());
return;
case Quorum:
- if (merged != null && merged.isTruncated())
+ if (merged != null && merged.isTruncatedResponse())
{
callback.accept(ProgressToken.TRUNCATED, null);
}
diff --git a/accord-core/src/main/java/accord/coordinate/tracking/ReadTracker.java b/accord-core/src/main/java/accord/coordinate/tracking/ReadTracker.java
index bc8cbf18..55639458 100644
--- a/accord-core/src/main/java/accord/coordinate/tracking/ReadTracker.java
+++ b/accord-core/src/main/java/accord/coordinate/tracking/ReadTracker.java
@@ -193,6 +193,11 @@ public class ReadTracker extends AbstractTracker<ReadTracker.ReadShardTracker>
{
return hasData;
}
+
+ public Ranges unavailable()
+ {
+ return unavailable;
+ }
}
// TODO (required): abstract the candidate selection process so the implementation may prioritise based on distance/health etc
diff --git a/accord-core/src/main/java/accord/impl/AbstractFetchCoordinator.java b/accord-core/src/main/java/accord/impl/AbstractFetchCoordinator.java
index 0f03927c..36bd6502 100644
--- a/accord-core/src/main/java/accord/impl/AbstractFetchCoordinator.java
+++ b/accord-core/src/main/java/accord/impl/AbstractFetchCoordinator.java
@@ -155,6 +155,7 @@ public abstract class AbstractFetchCoordinator extends FetchCoordinator
default: throw new AssertionError("Unhandled enum: " + reply);
case Invalid:
case Redundant:
+ // TODO (expected): stop fetch sync points from garbage collecting too quickly
throw new AssertionError(String.format("Unexpected reply: %s", reply));
}
}
diff --git a/accord-core/src/main/java/accord/impl/AbstractSafeCommandStore.java b/accord-core/src/main/java/accord/impl/AbstractSafeCommandStore.java
index 955e686b..aa053409 100644
--- a/accord-core/src/main/java/accord/impl/AbstractSafeCommandStore.java
+++ b/accord-core/src/main/java/accord/impl/AbstractSafeCommandStore.java
@@ -38,6 +38,7 @@ import accord.primitives.RoutableKey;
import accord.primitives.Seekable;
import accord.primitives.Seekables;
import accord.primitives.TxnId;
+import accord.utils.Invariants;
public abstract class AbstractSafeCommandStore<CommandType extends SafeCommand, CommandsForKeyType extends SafeCommandsForKey> extends SafeCommandStore
{
@@ -119,8 +120,8 @@ public abstract class AbstractSafeCommandStore<CommandType extends SafeCommand,
else
{
RedundantBefore.Entry entry = commandStore().redundantBefore().get(key.toUnseekable());
- if (entry != null && cfk.current().hasRedundant(entry.redundantBefore))
- cfk.set(cfk.current().withoutRedundant(entry.redundantBefore));
+ if (entry != null && cfk.current().hasRedundant(entry.shardRedundantBefore()))
+ cfk.set(cfk.current().withoutRedundant(entry.shardRedundantBefore()));
}
return cfk;
}
@@ -128,8 +129,7 @@ public abstract class AbstractSafeCommandStore<CommandType extends SafeCommand,
public CommandsForKeyType commandsForKey(RoutableKey key)
{
CommandsForKeyType cfk = getCommandsForKeyInternal(key);
- if (cfk == null)
- throw new IllegalStateException(String.format("%s was not specified in PreLoadContext", key));
+ Invariants.checkState(cfk != null, "%s was not specified in PreLoadContext", key);
if (cfk.isEmpty())
cfk.initialize(cfkLoader(key));
return cfk;
diff --git a/accord-core/src/main/java/accord/impl/CommandTimeseries.java b/accord-core/src/main/java/accord/impl/CommandTimeseries.java
index 66d26eed..abc319b8 100644
--- a/accord-core/src/main/java/accord/impl/CommandTimeseries.java
+++ b/accord-core/src/main/java/accord/impl/CommandTimeseries.java
@@ -26,7 +26,6 @@ import java.util.function.Predicate;
import java.util.stream.Stream;
import javax.annotation.Nullable;
-import com.google.common.base.Predicates;
import com.google.common.collect.ImmutableSortedMap;
import accord.api.Key;
@@ -36,6 +35,7 @@ import accord.local.SaveStatus;
import accord.local.Status;
import accord.primitives.Seekable;
import accord.primitives.Timestamp;
+import accord.primitives.Txn.Kind.Kinds;
import accord.primitives.TxnId;
import static accord.local.SafeCommandStore.TestDep.ANY_DEPS;
@@ -117,20 +117,10 @@ public class CommandTimeseries<D>
* <p>
* TODO (expected, efficiency): TestDep should be asynchronous; data should not be kept memory-resident as only used for recovery
*/
- public <T> T mapReduce(SafeCommandStore.TestKind testKind, TestTimestamp testTimestamp, Timestamp timestamp,
+ public <P1, T> T mapReduce(Kinds testKind, TestTimestamp testTimestamp, Timestamp timestamp,
SafeCommandStore.TestDep testDep, @Nullable TxnId depId,
@Nullable Status minStatus, @Nullable Status maxStatus,
- SafeCommandStore.CommandFunction<T, T> map, T initialValue, T terminalValue)
- {
- return mapReduceWithTerminate(testKind, testTimestamp, timestamp,
- testDep, depId,
- minStatus, maxStatus,
- map, initialValue, Predicates.equalTo(terminalValue));
- }
- public <T> T mapReduceWithTerminate(SafeCommandStore.TestKind testKind, TestTimestamp testTimestamp, Timestamp timestamp,
- SafeCommandStore.TestDep testDep, @Nullable TxnId depId,
- @Nullable Status minStatus, @Nullable Status maxStatus,
- SafeCommandStore.CommandFunction<T, T> map, T initialValue, Predicate<T> terminatePredicate)
+ SafeCommandStore.CommandFunction<P1, T, T> map, P1 p1, T initialValue, Predicate<? super T> terminatePredicate)
{
for (D data : (testTimestamp == TestTimestamp.BEFORE ? commands.headMap(timestamp, false) : commands.tailMap(timestamp, false)).values())
@@ -147,7 +137,7 @@ public class CommandTimeseries<D>
if (testDep != ANY_DEPS && (!status.known.deps.hasProposedOrDecidedDeps() || (deps.contains(depId) != (testDep == WITH))))
continue;
Timestamp executeAt = loader.executeAt(data);
- initialValue = map.apply(keyOrRange, txnId, executeAt, status.status, initialValue);
+ initialValue = map.apply(p1, keyOrRange, txnId, executeAt, status.status, initialValue);
if (terminatePredicate.test(initialValue))
break;
}
diff --git a/accord-core/src/main/java/accord/impl/ErasedSafeCommand.java b/accord-core/src/main/java/accord/impl/ErasedSafeCommand.java
index e9eea9b8..48cde3be 100644
--- a/accord-core/src/main/java/accord/impl/ErasedSafeCommand.java
+++ b/accord-core/src/main/java/accord/impl/ErasedSafeCommand.java
@@ -26,7 +26,7 @@ import accord.local.SaveStatus;
import accord.primitives.TxnId;
import static accord.local.Listeners.Immutable.EMPTY;
-import static accord.local.Status.Durability.DurableOrInvalidated;
+import static accord.local.Status.Durability.UniversalOrInvalidated;
public class ErasedSafeCommand extends SafeCommand
{
@@ -35,7 +35,7 @@ public class ErasedSafeCommand extends SafeCommand
public ErasedSafeCommand(TxnId txnId)
{
super(txnId);
- this.erased = new Command.Truncated(txnId, SaveStatus.Erased, DurableOrInvalidated, null, null, EMPTY, null, null);
+ this.erased = new Command.Truncated(txnId, SaveStatus.Erased, UniversalOrInvalidated, null, null, EMPTY, null, null);
}
@Override
diff --git a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
index fe47f27f..1a970c91 100644
--- a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
+++ b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
@@ -63,6 +64,7 @@ import accord.local.Status;
import accord.primitives.AbstractKeys;
import accord.primitives.Deps;
import accord.primitives.PartialDeps;
+import accord.primitives.Participants;
import accord.primitives.Range;
import accord.primitives.Ranges;
import accord.primitives.Routable;
@@ -71,6 +73,7 @@ import accord.primitives.Routables;
import accord.primitives.Seekable;
import accord.primitives.Seekables;
import accord.primitives.Timestamp;
+import accord.primitives.Txn.Kind.Kinds;
import accord.primitives.TxnId;
import accord.utils.Invariants;
import accord.utils.async.AsyncChain;
@@ -80,6 +83,8 @@ import static accord.local.Command.NotDefined.uninitialised;
import static accord.local.SafeCommandStore.TestDep.ANY_DEPS;
import static accord.local.SafeCommandStore.TestDep.WITH;
+import static accord.local.SaveStatus.Applying;
+import static accord.local.SaveStatus.ReadyToExecute;
import static accord.local.Status.Committed;
import static accord.local.Status.Truncated;
import static accord.primitives.Routables.Slice.Minimal;
@@ -87,8 +92,10 @@ import static accord.primitives.Routables.Slice.Minimal;
public abstract class InMemoryCommandStore extends CommandStore
{
private static final Logger logger = LoggerFactory.getLogger(InMemoryCommandStore.class);
+ private static final boolean CHECK_DEPENDENCY_INVARIANTS = false;
final NavigableMap<TxnId, GlobalCommand> commands = new TreeMap<>();
+ final NavigableMap<Timestamp, GlobalCommand> commandsByExecuteAt = new TreeMap<>();
private final NavigableMap<RoutableKey, GlobalCommandsForKey> commandsForKey = new TreeMap<>();
// TODO (find library, efficiency): this is obviously super inefficient, need some range map
@@ -103,6 +110,11 @@ public abstract class InMemoryCommandStore extends CommandStore
super(id, time, agent, store, progressLogFactory, epochUpdateHolder);
}
+ protected boolean canExposeUnloaded()
+ {
+ return true;
+ }
+
@Override
public Agent agent()
{
@@ -126,7 +138,80 @@ public abstract class InMemoryCommandStore extends CommandStore
public GlobalCommand command(TxnId txnId)
{
- return commands.computeIfAbsent(txnId, GlobalCommand::new);
+ return commands.computeIfAbsent(txnId, this::newGlobalCommand);
+ }
+
+ public void onInitialise(GlobalCommand newGlobalCommand)
+ {
+ if (CHECK_DEPENDENCY_INVARIANTS)
+ {
+
+ newGlobalCommand.addListener(new Command.TransientListener()
+ {
+ @Override
+ public void onChange(SafeCommandStore safeStore, SafeCommand safeCommand)
+ {
+ Command cur = safeCommand.current();
+ if (cur.saveStatus() == ReadyToExecute || cur.saveStatus() == Applying) // TODO (desired): only run the check once
+ {
+ long epoch = cur.executeAt().epoch();
+ Ranges ranges = safeStore.ranges().allAt(epoch);
+ Participants<?> participants = cur.route().participants(ranges, Minimal);
+ // TODO (required): look forwards also, but we only need to look at ?=ReadyToExecute transactions as they have already run their backwards checks
+ Iterator<GlobalCommand> iter = commandsByExecuteAt.descendingMap().tailMap(cur.executeAt(), false).values().iterator();
+ while (iter.hasNext())
+ {
+ GlobalCommand prevGlobal = iter.next();
+ Command prev = prevGlobal.value();
+ Timestamp prevExecuteAt = prev.executeAtIfKnown();
+ if (prevExecuteAt == null)
+ {
+ iter.remove();
+ continue;
+ }
+
+ if (prevExecuteAt.epoch() < epoch)
+ {
+ epoch = prevExecuteAt.epoch();
+ ranges = ranges.slice(safeStore.ranges().allAt(epoch), Minimal);
+ participants = participants.slice(ranges, Minimal);
+ }
+
+ if (participants.isEmpty())
+ break;
+
+ Participants participantsOfPrev = prev.route().participants(ranges, Minimal);
+ Participants intersectingParticipants = participants.intersect(participantsOfPrev);
+ if (intersectingParticipants.isEmpty())
+ continue;
+
+ if (!cur.txnId().rw().witnesses().test(prev.txnId().rw()) && !cur.partialDeps().contains(prev.txnId()))
+ continue;
+
+ Participants<?> depParticipants = cur.partialDeps().participants(prev.txnId());
+ if (!depParticipants.containsAll(intersectingParticipants))
+ Invariants.illegalState(cur.txnId() + " does not maintain dependency invariants with immediately preceding transaction " + prev.txnId() + "; intersecting participants: " + intersectingParticipants + "; dependency participants: " + depParticipants);
+
+ if (prev.txnId().isWrite())
+ participants = participants.subtract(intersectingParticipants);
+ };
+ }
+ }
+
+ @Override
+ public PreLoadContext listenerPreLoadContext(TxnId caller)
+ {
+ return PreLoadContext.contextFor(caller);
+ }
+ });
+ }
+ }
+
+ private GlobalCommand newGlobalCommand(TxnId txnId)
+ {
+ GlobalCommand globalCommand = new GlobalCommand(txnId);
+ onInitialise(globalCommand);
+ return globalCommand;
}
public InMemorySafeCommand lazyReference(TxnId txnId)
@@ -170,8 +255,9 @@ public abstract class InMemoryCommandStore extends CommandStore
});
return mutable;
case Range:
+ // TODO (required): confirm we slice to owned ranges on C*
rangeCommands.computeIfAbsent(command.txnId(), ignore -> new RangeCommand(commands.get(command.txnId())))
- .update((Ranges)keysOrRanges);
+ .update(((Ranges)keysOrRanges).slice(slice, Minimal));
}
return attrs;
}
@@ -190,13 +276,14 @@ public abstract class InMemoryCommandStore extends CommandStore
});
return mutable;
case Range:
+ // TODO (required): confirm we slice to owned ranges on C*
rangeCommands.computeIfAbsent(command.txnId(), ignore -> new RangeCommand(commands.get(command.txnId())))
- .update(Ranges.of((Range)keyOrRange));
+ .update(Ranges.of((Range)keyOrRange).slice(slice, Minimal));
}
return attrs;
}
- private <O> O mapReduceForKey(InMemorySafeStore safeStore, Routables<?> keysOrRanges, Ranges slice, BiFunction<CommandsForKey, O, O> map, O accumulate, Predicate<O> terminate)
+ private <O> O mapReduceForKey(InMemorySafeStore safeStore, Routables<?> keysOrRanges, Ranges slice, BiFunction<CommandsForKey, O, O> map, O accumulate, Predicate<? super O> terminate)
{
switch (keysOrRanges.domain()) {
default:
@@ -279,11 +366,20 @@ public abstract class InMemoryCommandStore extends CommandStore
// TODO (now): apply on retrieval
historicalRangeCommands.entrySet().removeIf(next -> next.getKey().compareTo(syncId) < 0 && next.getValue().intersects(ranges));
- rangeCommands.entrySet().removeIf(next -> {
- if (!(next.getKey().compareTo(syncId) < 0 && next.getValue().ranges.intersects(ranges)))
+ rangeCommands.entrySet().removeIf(tx -> {
+ if (tx.getKey().compareTo(syncId) >= 0)
return false;
- maxRedundant = Timestamp.nonNullOrMax(maxRedundant, next.getValue().command.value().executeAt());
- return true;
+ Ranges newRanges = tx.getValue().ranges.subtract(ranges);
+ if (!newRanges.isEmpty())
+ {
+ tx.getValue().ranges = newRanges;
+ return false;
+ }
+ else
+ {
+ maxRedundant = Timestamp.nonNullOrMax(maxRedundant, tx.getValue().command.value().executeAt());
+ return true;
+ }
});
ranges.forEach(r -> {
commandsForKey.subMap(r.start(), r.startInclusive(), r.end(), r.endInclusive()).values().forEach(forKey -> {
@@ -431,7 +527,7 @@ public abstract class InMemoryCommandStore extends CommandStore
private Command loadForCFK(CFKEntry entry)
{
GlobalCommand globalCommand = ifPresent(entry);
- if (globalCommand != null)
+ if (globalCommand != null && globalCommand.value() != null)
return globalCommand.value();
if (entry.uninitialised)
return uninitialised(entry);
@@ -610,6 +706,11 @@ public abstract class InMemoryCommandStore extends CommandStore
@Override
protected InMemorySafeCommand getIfLoaded(TxnId txnId)
{
+ InMemorySafeCommand safeCommand = commands.get(txnId);
+ if (safeCommand != null)
+ return safeCommand;
+ if (!commandStore.canExposeUnloaded())
+ return null;
GlobalCommand global = commandStore.ifPresent(txnId);
return global != null ? global.createSafeReference() : null;
}
@@ -617,6 +718,11 @@ public abstract class InMemoryCommandStore extends CommandStore
@Override
protected InMemorySafeCommandsForKey getIfLoaded(RoutableKey key)
{
+ InMemorySafeCommandsForKey safeCommandsForKey = commandsForKey.get((Key) key);
+ if (safeCommandsForKey != null)
+ return safeCommandsForKey;
+ if (!commandStore.canExposeUnloaded())
+ return null;
GlobalCommandsForKey global = commandStore.ifPresent((Key) key);
return global != null ? global.createSafeReference() : null;
}
@@ -661,7 +767,8 @@ public abstract class InMemoryCommandStore extends CommandStore
if (command.ranges.intersects(sliced))
timestamp = Timestamp.nonNullOrMax(timestamp, command.command.value().executeAt());
}
- return Timestamp.nonNullOrMax(timestamp, commandStore.maxRedundant);
+ // TODO (required): check we have this on C*-side; accept TxnId instead of Ranges when updating API, to avoid slicing
+ return Timestamp.nonNullOrMax(timestamp, commandStore.redundantBefore().minConflict(keysOrRanges.slice(slice)));
}
@Override
@@ -691,7 +798,7 @@ public abstract class InMemoryCommandStore extends CommandStore
});
TreeMap<TxnId, RangeCommand> rangeCommands = commandStore.rangeCommands();
TreeMap<TxnId, Ranges> historicalRangeCommands = commandStore.historicalRangeCommands();
- deps.rangeDeps.forEachUniqueTxnId(allRanges, txnId -> {
+ deps.rangeDeps.forEachUniqueTxnId(allRanges, null, (ignore, txnId) -> {
if (rangeCommands.containsKey(txnId))
return;
@@ -712,14 +819,9 @@ public abstract class InMemoryCommandStore extends CommandStore
return commandStore.time;
}
+ // TODO (expected): instead of accepting a slice, accept the min/max epoch and let implementation handle it
@Override
- public <T> T mapReduce(Seekables<?, ?> keysOrRanges, Ranges slice, TestKind testKind, TestTimestamp testTimestamp, Timestamp timestamp, TestDep testDep, @Nullable TxnId depId, @Nullable Status minStatus, @Nullable Status maxStatus, CommandFunction<T, T> map, T accumulate, T terminalValue)
- {
- return mapReduceWithTerminate(keysOrRanges, slice, testKind, testTimestamp, timestamp, testDep, depId, minStatus, maxStatus, map, accumulate, Predicate.isEqual(terminalValue));
- }
-
- @Override
- public <T> T mapReduceWithTerminate(Seekables<?, ?> keysOrRanges, Ranges slice, TestKind testKind, TestTimestamp testTimestamp, Timestamp timestamp, TestDep testDep, @Nullable TxnId depId, @Nullable Status minStatus, @Nullable Status maxStatus, CommandFunction<T, T> map, T accumulate, Predicate<T> terminate)
+ public <P1, T> T mapReduce(Seekables<?, ?> keysOrRanges, Ranges slice, Kinds testKind, TestTimestamp testTimestamp, Timestamp timestamp, TestDep testDep, @Nullable TxnId depId, @Nullable Status minStatus, @Nullable Status maxStatus, CommandFunction<P1, T, T> map, P1 p1, T accumulate, Predicate<? super T> terminate)
{
accumulate = commandStore.mapReduceForKey(this, keysOrRanges, slice, (forKey, prev) -> {
CommandTimeseries<?> timeseries;
@@ -746,7 +848,7 @@ public abstract class InMemoryCommandStore extends CommandStore
case MAY_EXECUTE_BEFORE:
remapTestTimestamp = CommandTimeseries.TestTimestamp.BEFORE;
}
- return timeseries.mapReduceWithTerminate(testKind, remapTestTimestamp, timestamp, testDep, depId, minStatus, maxStatus, map, prev, terminate);
+ return timeseries.mapReduce(testKind, remapTestTimestamp, timestamp, testDep, depId, minStatus, maxStatus, map, p1, prev, terminate);
}, accumulate, terminate);
if (terminate.test(accumulate))
@@ -758,7 +860,7 @@ public abstract class InMemoryCommandStore extends CommandStore
commandStore.rangeCommands.forEach(((txnId, rangeCommand) -> {
Command command = rangeCommand.command.value();
// TODO (now): probably this isn't safe - want to ensure we take dependency on any relevant syncId
- if (command.is(Truncated))
+ if (command.saveStatus().compareTo(SaveStatus.Erased) >= 0)
return;
Invariants.nonNull(command);
@@ -794,7 +896,18 @@ public abstract class InMemoryCommandStore extends CommandStore
if (!command.known().deps.hasProposedOrDecidedDeps())
return;
- if ((testDep == WITH) == !command.partialDeps().contains(depId))
+ // TODO (required): ensure C* matches this behaviour
+ // We are looking for transactions A that have (or have not) B as a dependency.
+ // If B covers ranges [1..3] and A covers [2..3], but the command store only covers ranges [1..2],
+ // we could have A adopt B as a dependency on [3..3] only, and have that A intersects B on this
+ // command store, but also that there is no dependency relation between them on the overlapping
+ // key range [2..2].
+
+ // This can lead to problems on recovery, where we believe a transaction is a dependency
+ // and so it is safe to execute, when in fact it is only a dependency on a different shard
+ // (and that other shard, perhaps, does not know that it is a dependency - and so it is not durably known)
+ // TODO (required): consider this some more
+ if ((testDep == WITH) == !command.partialDeps().intersects(depId, rangeCommand.ranges))
return;
}
@@ -847,7 +960,9 @@ public abstract class InMemoryCommandStore extends CommandStore
for (Map.Entry<TxnId, TimestampAndStatus> command : e.getValue())
{
T initial = accumulate;
- accumulate = map.apply(e.getKey(), command.getKey(), command.getValue().timestamp, command.getValue().status, initial);
+ accumulate = map.apply(p1, e.getKey(), command.getKey(), command.getValue().timestamp, command.getValue().status, initial);
+ if (terminate.test(accumulate))
+ return accumulate;
}
}
@@ -875,6 +990,17 @@ public abstract class InMemoryCommandStore extends CommandStore
@Override
protected void invalidateSafeState()
{
+ commands.values().forEach(c -> {
+ if (c != null && c.current() != null)
+ {
+ Timestamp executeAt = c.current().executeAtIfKnown();
+ if (executeAt == null)
+ return;
+
+ if (c.current().hasBeen(Truncated)) commandStore.commandsByExecuteAt.remove(executeAt);
+ else commandStore.commandsByExecuteAt.put(executeAt, commandStore.command(c.txnId()));
+ }
+ });
commands.values().forEach(SafeState::invalidate);
commandsForKey.values().forEach(SafeState::invalidate);
}
diff --git a/accord-core/src/main/java/accord/impl/SafeCommandsForKey.java b/accord-core/src/main/java/accord/impl/SafeCommandsForKey.java
index 06b5a542..1e5e41b0 100644
--- a/accord-core/src/main/java/accord/impl/SafeCommandsForKey.java
+++ b/accord-core/src/main/java/accord/impl/SafeCommandsForKey.java
@@ -22,6 +22,7 @@ import accord.api.Key;
import accord.api.VisibleForImplementation;
import accord.impl.CommandTimeseries.CommandLoader;
import accord.local.Command;
+import accord.local.SafeCommandStore;
import accord.primitives.Timestamp;
import accord.primitives.TxnId;
import accord.utils.Invariants;
@@ -165,14 +166,18 @@ public abstract class SafeCommandsForKey implements SafeState<CommandsForKey>
}
@VisibleForImplementation
- public <D> CommandsForKey updateLastExecutionTimestamps(Timestamp executeAt, boolean isForWriteTxn)
+ public <D> CommandsForKey updateLastExecutionTimestamps(SafeCommandStore safeStore, Timestamp executeAt, boolean isForWriteTxn)
{
CommandsForKey current = current();
Timestamp lastWrite = current.lastWriteTimestamp();
if (executeAt.compareTo(lastWrite) < 0)
+ {
+ if (!safeStore.commandStore().safeToReadAt(executeAt).contains(key.toUnseekable()))
+ return current;
throw new IllegalArgumentException(String.format("%s is less than the most recent write timestamp %s", executeAt, lastWrite));
+ }
Timestamp lastExecuted = current.lastExecutedTimestamp();
int cmp = executeAt.compareTo(lastExecuted);
@@ -180,7 +185,11 @@ public abstract class SafeCommandsForKey implements SafeState<CommandsForKey>
if (cmp == 0 || (!isForWriteTxn && cmp < 0))
return current;
if (cmp < 0)
+ {
+ if (!safeStore.commandStore().safeToReadAt(executeAt).contains(key.toUnseekable()))
+ return current;
throw new IllegalArgumentException(String.format("%s is less than the most recent executed timestamp %s", executeAt, lastExecuted));
+ }
long micros = executeAt.hlc();
long lastMicros = current.lastExecutedHlc();
diff --git a/accord-core/src/main/java/accord/impl/SimpleProgressLog.java b/accord-core/src/main/java/accord/impl/SimpleProgressLog.java
index ae395921..d7477088 100644
--- a/accord-core/src/main/java/accord/impl/SimpleProgressLog.java
+++ b/accord-core/src/main/java/accord/impl/SimpleProgressLog.java
@@ -32,7 +32,6 @@ import org.slf4j.LoggerFactory;
import accord.api.ProgressLog;
import accord.coordinate.FetchData;
-import accord.coordinate.Invalidate;
import accord.coordinate.Outcome;
import accord.local.Command;
import accord.local.CommandStore;
@@ -45,8 +44,10 @@ import accord.local.SaveStatus;
import accord.local.SaveStatus.LocalExecution;
import accord.local.Status;
import accord.local.Status.Known;
+import accord.primitives.EpochSupplier;
import accord.primitives.Participants;
import accord.primitives.ProgressToken;
+import accord.primitives.Ranges;
import accord.primitives.Route;
import accord.primitives.Timestamp;
import accord.primitives.TxnId;
@@ -70,9 +71,8 @@ import static accord.local.PreLoadContext.contextFor;
import static accord.local.PreLoadContext.empty;
import static accord.local.SaveStatus.LocalExecution.NotReady;
import static accord.local.SaveStatus.LocalExecution.WaitingToApply;
-import static accord.local.Status.Durability.Majority;
+import static accord.local.Status.Durability.MajorityOrInvalidated;
import static accord.local.Status.PreApplied;
-import static accord.local.Status.PreCommitted;
// TODO (desired, consider): consider propagating invalidations in the same way as we do applied
// TODO (expected): report long-lived recurring transactions / operations
@@ -240,6 +240,7 @@ public class SimpleProgressLog implements ProgressLog.Factory
throw new IllegalStateException("Unexpected status: " + status);
case ReadyToExecute:
+ // TODO (expected): we should have already exited this loop if this conditions is true
if (command.durability().isDurableOrInvalidated())
{
// should not reach here with an invalidated state
@@ -251,47 +252,35 @@ public class SimpleProgressLog implements ProgressLog.Factory
case Uncommitted:
{
Invariants.checkState(token.status != Status.Invalidated, "ProgressToken is invalidated, but we have not cleared the ProgressLog");
- // TODO (expected): this should be encapsulated in Recover/FetchData
- if (!command.durability().isDurable() && token.durability.isDurableOrInvalidated() && (token.durability.isDurable() || command.hasBeen(PreCommitted)))
- command = Commands.setDurability(safeStore, safeCommand, Majority);
-
- if (command.durability().isDurable() || token.durability.isDurableOrInvalidated())
+ // TODO (expected): we should have already exited this loop if either of these conditions are true
+ if (command.durability().isDurableOrInvalidated() || token.durability.isDurableOrInvalidated())
{
- // not guaranteed to know the execution epoch, as might have received durability info obliquely
- Timestamp executeAt = command.executeAtIfKnown();
- long epoch = executeAt == null ? txnId.epoch() : executeAt.epoch();
- Route<?> route = command.route();
-
- node.withEpoch(epoch, () -> debugInvestigating = FetchData.fetch(PreApplied.minKnown, node, txnId, route, executeAt, (success, fail) -> {
- commandStore.execute(empty(), ignore -> {
- // should have found enough information to apply the result, but in case we did not reset progress
- if (progress() == Investigating)
- setProgress(Expected);
- }).begin(commandStore.agent());
- }));
+ if (!command.durability().isDurableOrInvalidated())
+ Commands.setDurability(safeStore, safeCommand, MajorityOrInvalidated);
+
+ durableGlobal();
+ return;
}
- else
- {
- Route<?> route = Invariants.nonNull(command.route()).withHomeKey();
- node.withEpoch(txnId.epoch(), () -> {
- AsyncResult<? extends Outcome> recover = node.maybeRecover(txnId, route, token);
- recover.addCallback((success, fail) -> {
- // TODO (expected): callback should be on safeStore, and should provide safeStore as a parameter
- commandStore.execute(contextFor(txnId), safeStore0 -> {
- if (status.isAtMostReadyToExecute() && progress() == Investigating)
- {
- setProgress(Expected);
- if (fail != null)
- return;
-
- updateMax(success.asProgressToken());
- }
- });
- });
- debugInvestigating = recover;
+ Route<?> route = Invariants.nonNull(command.route()).withHomeKey();
+ node.withEpoch(txnId.epoch(), () -> {
+ AsyncResult<? extends Outcome> recover = node.maybeRecover(txnId, route, token);
+ recover.addCallback((success, fail) -> {
+ // TODO (expected): callback should be on safeStore, and should provide safeStore as a parameter
+ commandStore.execute(contextFor(txnId), safeStore0 -> {
+ if (status.isAtMostReadyToExecute() && progress() == Investigating)
+ {
+ setProgress(Expected);
+ if (fail != null)
+ return;
+
+ updateMax(success.asProgressToken());
+ }
+ });
});
- }
+
+ debugInvestigating = recover;
+ });
}
}
}
@@ -352,10 +341,10 @@ public class SimpleProgressLog implements ProgressLog.Factory
setProgress(Investigating);
// first make sure we have enough information to obtain the command locally
Timestamp executeAt = command.executeAtIfKnown();
- Participants<?> maxParticipants = maxParticipants(command);
// we want to fetch a route if we have it, so that we can go to our neighbouring shards for info
// (rather than the home shard, which may have GC'd its state if the result is durable)
Unseekables<?> fetchKeys = maxContact(command);
+ EpochSupplier forLocalEpoch = safeStore.ranges().latestEpochWithNewParticipants(txnId.epoch(), fetchKeys);
BiConsumer<Known, Throwable> callback = (success, fail) -> {
// TODO (expected): this should be invoked on this commandStore; also do not need to load txn unless in DEBUG mode
@@ -364,17 +353,12 @@ public class SimpleProgressLog implements ProgressLog.Factory
return;
setProgress(Expected);
- if (fail == null && blockedUntil.isSatisfiedBy(success))
- {
- Command test = safeStore0.ifInitialised(txnId).current();
- Invariants.checkState(test.has(success), "Command %s was expected to have known %s, but had %s", test, success, test.known());
- record(success);
- }
+ Invariants.checkState(fail != null || !blockedUntil.isSatisfiedBy(success.propagates()));
}).begin(commandStore.agent());
};
node.withEpoch(blockedUntil.fetchEpoch(txnId, executeAt), () -> {
- debugInvestigating = FetchData.fetch(blockedUntil.requires, node, txnId, fetchKeys, executeAt, callback);
+ debugInvestigating = FetchData.fetch(blockedUntil.requires, node, txnId, fetchKeys, forLocalEpoch, executeAt, callback);
});
}
@@ -393,24 +377,6 @@ public class SimpleProgressLog implements ProgressLog.Factory
return Participants.merge(route == null ? null : route.participants(), (Participants) participants);
}
- private void invalidate(Node node, TxnId txnId, Participants<?> participants)
- {
- setProgress(Investigating);
- // TODO (now): we should avoid invalidating with home key, as this simplifies erasing of invalidated transactions
- // TODO (required): exponential back-off or time-slicing
- debugInvestigating = Invalidate.invalidate(node, txnId, participants, (success, fail) -> {
- commandStore.execute(contextFor(txnId), safeStore -> {
- if (progress() != Investigating)
- return;
-
- setProgress(Expected);
- SafeCommand safeCommand = safeStore.ifInitialised(txnId);
- Invariants.checkState(safeCommand != null, "No initialised command for %s but have active progress log", txnId);
- record(safeCommand.current().known());
- }).begin(commandStore.agent());
- });
- }
-
@Override
public String toString()
{
@@ -473,7 +439,7 @@ public class SimpleProgressLog implements ProgressLog.Factory
blockingState.recordBlocking(waitingFor, route, participants);
}
- CoordinateState local()
+ CoordinateState coordinate()
{
if (coordinateState == null)
coordinateState = new CoordinateState();
@@ -482,12 +448,12 @@ public class SimpleProgressLog implements ProgressLog.Factory
void ensureAtLeast(Command command, CoordinateStatus newStatus, Progress newProgress)
{
- local().ensureAtLeast(command, newStatus, newProgress);
+ coordinate().ensureAtLeast(command, newStatus, newProgress);
}
void ensureAtLeast(CoordinateStatus newStatus, Progress newProgress)
{
- local().ensureAtLeast(newStatus, newProgress);
+ coordinate().ensureAtLeast(newStatus, newProgress);
}
void touchNonHomeUnsafe()
@@ -595,6 +561,14 @@ public class SimpleProgressLog implements ProgressLog.Factory
ensureSafeOrAtLeast(command, shard, Uncommitted, Expected);
}
+ @Override
+ public void precommitted(Command command)
+ {
+ State state = stateMap.get(command.txnId());
+ if (state != null && state.blockingState != null)
+ state.blockingState.record(SaveStatus.PreCommitted.known);
+ }
+
@Override
public void committed(Command command, ProgressShard shard)
{
@@ -644,10 +618,14 @@ public class SimpleProgressLog implements ProgressLog.Factory
// TODO (expected): we should delete the in-memory state once we reach Done, and guard against backward progress with Command state
// however, we need to be careful:
// - we might participate in the execution epoch so need to be sure we have received a route covering both
- if (!command.status().hasBeen(PreApplied) && command.route() != null && command.route().hasParticipants())
+ if (command.route() == null)
+ return;
+
+ Ranges coordinateRanges = commandStore.unsafeRangesForEpoch().allAt(command.txnId().epoch());
+ if (!command.status().hasBeen(PreApplied) && command.route().participatesIn(coordinateRanges))
state.recordBlocking(command.txnId(), WaitingToApply, command.route(), null);
- if (state.coordinateState != null)
- state.coordinateState.durableGlobal();
+ if (coordinateRanges.contains(command.route().homeKey()))
+ state.coordinate().durableGlobal();
}
@Override
@@ -658,6 +636,8 @@ public class SimpleProgressLog implements ProgressLog.Factory
// ensure we have a record to work with later; otherwise may think has been truncated
blockedBy.initialise();
+ if (blockedBy.current().has(blockedUntil.requires))
+ return;
// TODO (consider): consider triggering a preemption of existing coordinator (if any) in some circumstances;
// today, an LWT can pre-empt more efficiently (i.e. instantly) a failed operation whereas Accord will
diff --git a/accord-core/src/main/java/accord/local/Bootstrap.java b/accord-core/src/main/java/accord/local/Bootstrap.java
index 41ce28e1..0652fd3c 100644
--- a/accord-core/src/main/java/accord/local/Bootstrap.java
+++ b/accord-core/src/main/java/accord/local/Bootstrap.java
@@ -362,7 +362,7 @@ class Bootstrap
}
}
- store.markSafeToRead(state.safeToReadAt, newDone);
+ store.markSafeToRead(globalSyncId, state.safeToReadAt, newDone);
fetchedAndSafeToRead = fetchedAndSafeToRead.with(newDone);
maybeComplete();
}
@@ -399,6 +399,9 @@ class Bootstrap
if (!fetchedAndSafeToRead.containsAll(fetchCompleted ? fetched : valid))
return;
+ // normalise fetched and fetchedAndSafeToRead against remaining valid ranges before completion
+ fetched = fetched.slice(valid, Minimal);
+ fetchedAndSafeToRead = fetchedAndSafeToRead.slice(valid, Minimal);
retry = valid.subtract(fetchedAndSafeToRead);
completed = true;
}
diff --git a/accord-core/src/main/java/accord/local/Command.java b/accord-core/src/main/java/accord/local/Command.java
index 49ebb5c4..3c680a8e 100644
--- a/accord-core/src/main/java/accord/local/Command.java
+++ b/accord-core/src/main/java/accord/local/Command.java
@@ -22,34 +22,36 @@ import java.util.Objects;
import com.google.common.collect.ImmutableSortedSet;
-import accord.api.Data;
import accord.api.Result;
import accord.api.RoutingKey;
import accord.api.VisibleForImplementation;
+import accord.local.Status.Durability;
+import accord.local.Status.Known;
import accord.primitives.Ballot;
import accord.primitives.Deps;
+import accord.primitives.FullRoute;
import accord.primitives.Keys;
import accord.primitives.PartialDeps;
import accord.primitives.PartialTxn;
-import accord.primitives.Ranges;
import accord.primitives.Route;
import accord.primitives.Timestamp;
import accord.primitives.Txn;
import accord.primitives.TxnId;
-import accord.primitives.Unseekables;
import accord.primitives.Writes;
import accord.utils.ImmutableBitSet;
import accord.utils.IndexedQuadConsumer;
import accord.utils.Invariants;
import accord.utils.SimpleBitSet;
-import accord.utils.async.AsyncChain;
import javax.annotation.Nullable;
+import static accord.local.Command.AbstractCommand.validate;
import static accord.local.Listeners.Immutable.EMPTY;
import static accord.local.SaveStatus.Uninitialised;
-import static accord.local.Status.Durability.DurableOrInvalidated;
import static accord.local.Status.Durability.Local;
import static accord.local.Status.Durability.NotDurable;
+import static accord.local.Status.Durability.ShardUniversal;
+import static accord.local.Status.Durability.UniversalOrInvalidated;
+import static accord.local.Status.Invalidated;
import static accord.local.Status.KnownExecuteAt.ExecuteAtKnown;
import static accord.utils.SortedArrays.forEachIntersection;
import static accord.utils.Utils.ensureImmutable;
@@ -130,9 +132,9 @@ public abstract class Command implements CommonAttributes
return command instanceof PreLoadContext ? (PreLoadContext) command : PreLoadContext.contextFor(command.txnId(), command.partialTxn().keys());
}
- private static Status.Durability durability(Status.Durability durability, SaveStatus status)
+ private static Durability durability(Durability durability, SaveStatus status)
{
- if (status.compareTo(SaveStatus.PreApplied) >= 0 && durability == NotDurable)
+ if (status.compareTo(SaveStatus.PreApplied) >= 0 && !status.hasBeen(Invalidated) && durability == NotDurable)
return Local; // not necessary anywhere, but helps for logical consistency
return durability;
}
@@ -218,16 +220,16 @@ public abstract class Command implements CommonAttributes
*/
public abstract boolean isEqualOrFuller(Command other);
- private abstract static class AbstractCommand extends Command
+ abstract static class AbstractCommand extends Command
{
private final TxnId txnId;
private final SaveStatus status;
- private final Status.Durability durability;
+ private final Durability durability;
private final Route<?> route;
private final Ballot promised;
private final Listeners.Immutable listeners;
- private AbstractCommand(TxnId txnId, SaveStatus status, Status.Durability durability, Route<?> route, Ballot promised, Listeners.Immutable listeners)
+ private AbstractCommand(TxnId txnId, SaveStatus status, Durability durability, Route<?> route, Ballot promised, Listeners.Immutable listeners)
{
this.txnId = txnId;
this.status = validateCommandClass(status, getClass());
@@ -299,7 +301,7 @@ public abstract class Command implements CommonAttributes
}
@Override
- public Status.Durability durability()
+ public Durability durability()
{
return Command.durability(durability, saveStatus());
}
@@ -315,6 +317,87 @@ public abstract class Command implements CommonAttributes
{
return status;
}
+
+ public static <T extends AbstractCommand> T validate(T validate)
+ {
+ Known known = validate.known();
+ switch (known.route)
+ {
+ default: throw new AssertionError("Unhandled KnownRoute: " + known.route);
+ case Maybe: break;
+ case Full: Invariants.checkState(Route.isFullRoute(validate.route())); break;
+ case Covering: Invariants.checkState(Route.isRoute(validate.route())); break;
+ }
+ {
+ PartialTxn partialTxn = validate.partialTxn();
+ switch (known.definition)
+ {
+ default: throw new AssertionError("Unhandled Definition: " + known.definition);
+ case DefinitionErased:
+ case DefinitionUnknown:
+ case NoOp:
+ Invariants.checkState(partialTxn == null);
+ break;
+ case DefinitionKnown:
+ Invariants.checkState(partialTxn != null);
+ break;
+ }
+ }
+ {
+ Timestamp executeAt = validate.executeAt();
+ switch (known.executeAt)
+ {
+ default: throw new AssertionError("Unhandled KnownExecuteAt: " + known.executeAt);
+ case ExecuteAtErased:
+ case ExecuteAtUnknown: break;
+ case ExecuteAtProposed:
+ case ExecuteAtKnown:
+ Invariants.checkState(executeAt != null && !executeAt.equals(Timestamp.NONE));
+ break;
+ case NoExecuteAt:
+ Invariants.checkState(executeAt.equals(Timestamp.NONE));
+ break;
+ }
+ }
+ {
+ PartialDeps deps = validate.partialDeps();
+ switch (known.deps)
+ {
+ default: throw new AssertionError("Unhandled KnownDeps: " + known.deps);
+ case DepsUnknown:
+ case DepsErased:
+ case NoDeps:
+ Invariants.checkState(deps == null);
+ break;
+ case DepsProposed:
+ case DepsKnown:
+ Invariants.checkState(deps != null);
+ break;
+ }
+ }
+ {
+ Writes writes = validate.writes();
+ Result result = validate.result();
+ switch (known.outcome)
+ {
+ default: throw new AssertionError("Unhandled Outcome: " + known.outcome);
+ case Apply:
+ Invariants.checkState(writes != null);
+ Invariants.checkState(result != null);
+ break;
+ case Invalidated:
+ Invariants.checkState(validate.durability().isMaybeInvalidated());
+ case Unknown:
+ Invariants.checkState(validate.durability() != Local);
+ case Erased:
+ case WasApply:
+ Invariants.checkState(writes == null);
+ Invariants.checkState(result == null);
+ break;
+ }
+ }
+ return validate;
+ }
}
@Override
@@ -361,7 +444,7 @@ public abstract class Command implements CommonAttributes
public abstract TxnId txnId();
public abstract Ballot promised();
@Override
- public abstract Status.Durability durability();
+ public abstract Durability durability();
@Override
public abstract Listeners.Immutable<DurableAndIdempotentListener> durableListeners();
public abstract SaveStatus saveStatus();
@@ -440,12 +523,18 @@ public abstract class Command implements CommonAttributes
return executeAt == null || executeAt.equals(Timestamp.NONE) ? txnId() : executeAt;
}
+ public final Timestamp executeAtIfKnownOrTxnId()
+ {
+ Timestamp executeAt = executeAtIfKnown();
+ return executeAt == null || executeAt.equals(Timestamp.NONE) ? txnId() : executeAt;
+ }
+
public final Status status()
{
return saveStatus().status;
}
- public final Status.Known known()
+ public final Known known()
{
return saveStatus().known;
}
@@ -455,7 +544,7 @@ public abstract class Command implements CommonAttributes
return status().compareTo(status) >= 0;
}
- public boolean has(Status.Known known)
+ public boolean has(Known known)
{
return known.isSatisfiedBy(saveStatus().known);
}
@@ -465,16 +554,6 @@ public abstract class Command implements CommonAttributes
return saveStatus().execution.compareTo(execution) >= 0;
}
- public boolean has(Status.Definition definition)
- {
- return known().definition.compareTo(definition) >= 0;
- }
-
- public boolean has(Status.Outcome outcome)
- {
- return known().outcome.compareTo(outcome) >= 0;
- }
-
public boolean is(Status status)
{
return status() == status;
@@ -549,7 +628,7 @@ public abstract class Command implements CommonAttributes
public static final class NotDefined extends AbstractCommand
{
- NotDefined(TxnId txnId, SaveStatus status, Status.Durability durability, Route<?> route, Ballot promised, Listeners.Immutable listeners)
+ NotDefined(TxnId txnId, SaveStatus status, Durability durability, Route<?> route, Ballot promised, Listeners.Immutable listeners)
{
super(txnId, status, durability, route, promised, listeners);
}
@@ -562,17 +641,17 @@ public abstract class Command implements CommonAttributes
@Override
public Command updateAttributes(CommonAttributes attrs, Ballot promised)
{
- return new NotDefined(attrs, initialise(saveStatus()), promised);
+ return validate(new NotDefined(attrs, initialise(saveStatus()), promised));
}
public static NotDefined notDefined(CommonAttributes common, Ballot promised)
{
- return new NotDefined(common, SaveStatus.NotDefined, promised);
+ return validate(new NotDefined(common, SaveStatus.NotDefined, promised));
}
public static NotDefined uninitialised(TxnId txnId)
{
- return new NotDefined(txnId, Uninitialised, NotDurable, null, Ballot.ZERO, null);
+ return validate(new NotDefined(txnId, Uninitialised, NotDurable, null, Ballot.ZERO, null));
}
@Override
@@ -626,7 +705,7 @@ public abstract class Command implements CommonAttributes
this.result = result;
}
- public Truncated(TxnId txnId, SaveStatus saveStatus, Status.Durability durability, Route<?> route, @Nullable Timestamp executeAt, Listeners.Immutable listeners, @Nullable Writes writes, @Nullable Result result)
+ public Truncated(TxnId txnId, SaveStatus saveStatus, Durability durability, Route<?> route, @Nullable Timestamp executeAt, Listeners.Immutable listeners, @Nullable Writes writes, @Nullable Result result)
{
super(txnId, saveStatus, durability, route, Ballot.MAX, listeners);
this.executeAt = executeAt;
@@ -660,23 +739,32 @@ public abstract class Command implements CommonAttributes
public static Truncated erased(Command command)
{
- return erased(command.txnId(), command.durability(), command.route());
+ Durability durability = Durability.mergeAtLeast(command.durability(), UniversalOrInvalidated);
+ return erased(command.txnId(), durability, command.route());
}
public static Truncated erased(TxnId txnId, Status.Durability durability, Route<?> route)
{
- return new Truncated(txnId, SaveStatus.Erased, durability, route, null, EMPTY, null, null);
+ return validate(new Truncated(txnId, SaveStatus.Erased, durability, route, null, EMPTY, null, null));
}
public static Truncated truncatedApply(Command command)
{
- Invariants.checkArgument(command.known().executeAt.hasDecidedExecuteAt());
- return new Truncated(command.txnId(), SaveStatus.TruncatedApply, command.durability(), command.route(), command.executeAtIfKnown(), EMPTY, null, null);
+ return truncatedApply(command, null);
+ }
+
+ public static Truncated truncatedApply(Command command, @Nullable FullRoute<?> route)
+ {
+ Invariants.checkArgument(command.known().executeAt.isDecidedAndKnownToExecute());
+ if (route == null) route = Route.castToNonNullFullRoute(command.route());
+ Durability durability = Durability.mergeAtLeast(command.durability(), ShardUniversal);
+ return validate(new Truncated(command.txnId(), SaveStatus.TruncatedApply, durability, route, command.executeAt(), EMPTY, null, null));
}
public static Truncated truncatedApplyWithOutcome(Executed command)
{
- return new Truncated(command.txnId(), SaveStatus.TruncatedApplyWithOutcome, command.durability(), command.route(), command.executeAt(), EMPTY, command.writes, command.result);
+ Durability durability = Durability.mergeAtLeast(command.durability(), ShardUniversal);
+ return validate(new Truncated(command.txnId(), SaveStatus.TruncatedApplyWithOutcome, durability, command.route(), command.executeAt(), EMPTY, command.writes, command.result));
}
public static Truncated truncatedApply(CommonAttributes common, SaveStatus saveStatus, Timestamp executeAt, Writes writes, Result result)
@@ -684,7 +772,8 @@ public abstract class Command implements CommonAttributes
Invariants.checkArgument(executeAt != null);
Invariants.checkArgument(saveStatus == SaveStatus.TruncatedApply || saveStatus == SaveStatus.TruncatedApplyWithDeps || saveStatus == SaveStatus.TruncatedApplyWithOutcome);
// TODO review Is this correctly handling all three versions of truncatedApplyStatus? Should writes be null some of the time?
- return new Truncated(common.txnId(), saveStatus, common.durability(), common.route(), executeAt, EMPTY, writes, result);
+ Durability durability = Durability.mergeAtLeast(common.durability(), ShardUniversal);
+ return validate(new Truncated(common.txnId(), saveStatus, durability, common.route(), executeAt, EMPTY, writes, result));
}
public static Truncated invalidated(Command command)
@@ -696,7 +785,9 @@ public abstract class Command implements CommonAttributes
public static Truncated invalidated(TxnId txnId, Listeners.Immutable durableListeners)
{
- return new Truncated(txnId, SaveStatus.Invalidated, DurableOrInvalidated, null, Timestamp.NONE, durableListeners, null, Result.INVALIDATED);
+ // TODO (expected): migrate to using null for executeAt when invalidated
+ // TODO (expected): is UniversalOrInvalidated correct here? Should we have a lower implication pure Invalidated?
+ return validate(new Truncated(txnId, SaveStatus.Invalidated, UniversalOrInvalidated, null, Timestamp.NONE, durableListeners, null, null));
}
@Override
@@ -739,7 +830,7 @@ public abstract class Command implements CommonAttributes
public Command updateAttributes(CommonAttributes attrs, Ballot promised)
{
// TODO (now): invoke listeners precisely once when we adopt this state, then we can simply return `this`
- return new Truncated(txnId(), saveStatus(), attrs.durability(), attrs.route(), executeAt, attrs.durableListeners(), writes, result);
+ return validate(new Truncated(txnId(), saveStatus(), attrs.durability(), attrs.route(), executeAt, attrs.durableListeners(), writes, result));
}
}
@@ -749,18 +840,23 @@ public abstract class Command implements CommonAttributes
private final PartialTxn partialTxn;
private final @Nullable PartialDeps partialDeps;
- private PreAccepted(CommonAttributes common, SaveStatus status, Timestamp executeAt, Ballot promised)
+ private PreAccepted(CommonAttributes common, SaveStatus status, Ballot promised, Timestamp executeAt)
+ {
+ this(common, status, promised, executeAt, common.partialTxn(), common.partialDeps());
+ }
+
+ private PreAccepted(CommonAttributes common, SaveStatus status, Ballot promised, Timestamp executeAt, PartialTxn partialTxn, PartialDeps partialDeps)
{
super(common, status, promised);
this.executeAt = executeAt;
- this.partialTxn = common.partialTxn();
- this.partialDeps = common.partialDeps();
+ this.partialTxn = partialTxn;
+ this.partialDeps = partialDeps;
}
@Override
public Command updateAttributes(CommonAttributes attrs, Ballot promised)
{
- return new PreAccepted(attrs, saveStatus(), executeAt(), promised);
+ return new PreAccepted(attrs, saveStatus(), promised, executeAt());
}
@Override
@@ -790,7 +886,7 @@ public abstract class Command implements CommonAttributes
public static PreAccepted preAccepted(CommonAttributes common, Timestamp executeAt, Ballot promised)
{
- return new PreAccepted(common, SaveStatus.PreAccepted, executeAt, promised);
+ return validate(new PreAccepted(common, SaveStatus.PreAccepted, promised, executeAt));
}
public static PreAccepted preAccepted(PreAccepted command, CommonAttributes common, Ballot promised)
@@ -830,16 +926,22 @@ public abstract class Command implements CommonAttributes
{
private final Ballot accepted;
- Accepted(CommonAttributes common, SaveStatus status, Timestamp executeAt, Ballot promised, Ballot accepted)
+ Accepted(CommonAttributes common, SaveStatus status, Ballot promised, Timestamp executeAt, Ballot accepted)
{
- super(common, status, executeAt, promised);
+ super(common, status, promised, executeAt);
+ this.accepted = accepted;
+ }
+
+ Accepted(CommonAttributes common, SaveStatus status, Ballot promised, Timestamp executeAt, PartialTxn partialTxn, PartialDeps partialDeps, Ballot accepted)
+ {
+ super(common, status, promised, executeAt, partialTxn, partialDeps);
this.accepted = accepted;
}
@Override
public Command updateAttributes(CommonAttributes attrs, Ballot promised)
{
- return new Accepted(attrs, saveStatus(), executeAt(), promised, accepted());
+ return validate(new Accepted(attrs, saveStatus(), promised, executeAt(), accepted()));
}
@Override
@@ -864,14 +966,14 @@ public abstract class Command implements CommonAttributes
static Accepted accepted(CommonAttributes common, SaveStatus status, Timestamp executeAt, Ballot promised, Ballot accepted)
{
- return new Accepted(common, status, executeAt, promised, accepted);
+ return validate(new Accepted(common, status, promised, executeAt, accepted));
}
static Accepted accepted(Accepted command, CommonAttributes common, SaveStatus status, Ballot promised)
{
checkPromised(command, promised);
checkSameClass(command, Accepted.class, "Cannot update");
- return new Accepted(common, status, command.executeAt(), promised, command.accepted());
+ return validate(new Accepted(common, status, promised, command.executeAt(), command.accepted()));
}
static Accepted accepted(Accepted command, CommonAttributes common, Ballot promised)
@@ -892,16 +994,16 @@ public abstract class Command implements CommonAttributes
private Committed(CommonAttributes common, SaveStatus status, Timestamp executeAt, Ballot promised, Ballot accepted, WaitingOn waitingOn)
{
- super(common, status, executeAt, promised, accepted);
+ super(common, status, promised, executeAt, accepted);
this.waitingOn = waitingOn;
Invariants.checkState(common.route().kind().isFullRoute(), "Expected a full route but given %s", common.route().kind());
- Invariants.checkState(waitingOn.deps.equals(common.partialDeps()), "Deps do not match; expected %s == %s", waitingOn.deps, common.partialDeps());
+ Invariants.checkState(waitingOn == WaitingOn.EMPTY || waitingOn.deps.equals(common.partialDeps()), "Deps do not match; expected %s == %s", waitingOn.deps, common.partialDeps());
}
@Override
public Command updateAttributes(CommonAttributes attrs, Ballot promised)
{
- return new Committed(attrs, saveStatus(), executeAt(), promised, accepted(), waitingOn());
+ return validate(new Committed(attrs, saveStatus(), executeAt(), promised, accepted(), waitingOn()));
}
@Override
@@ -929,7 +1031,7 @@ public abstract class Command implements CommonAttributes
{
checkPromised(command, promised);
checkSameClass(command, Committed.class, "Cannot update");
- return new Committed(common, status, command.executeAt(), promised, command.accepted(), waitingOn);
+ return validate(new Committed(common, status, command.executeAt(), promised, command.accepted(), waitingOn));
}
static Committed committed(Committed command, CommonAttributes common, Ballot promised)
@@ -949,12 +1051,7 @@ public abstract class Command implements CommonAttributes
static Committed committed(CommonAttributes common, SaveStatus status, Timestamp executeAt, Ballot promised, Ballot accepted, WaitingOn waitingOn)
{
- return new Committed(common, status, executeAt, promised, accepted, waitingOn);
- }
-
- public AsyncChain<Data> read(SafeCommandStore safeStore)
- {
- return partialTxn().read(safeStore, executeAt());
+ return validate(new Committed(common, status, executeAt, promised, accepted, waitingOn));
}
public WaitingOn waitingOn()
@@ -994,7 +1091,7 @@ public abstract class Command implements CommonAttributes
@Override
public Command updateAttributes(CommonAttributes attrs, Ballot promised)
{
- return new Executed(attrs, saveStatus(), executeAt(), promised, accepted(), waitingOn(), writes, result);
+ return validate(new Executed(attrs, saveStatus(), executeAt(), promised, accepted(), waitingOn(), writes, result));
}
@Override
@@ -1022,7 +1119,7 @@ public abstract class Command implements CommonAttributes
public static Executed executed(Executed command, CommonAttributes common, SaveStatus status, Ballot promised, WaitingOn waitingOn)
{
checkSameClass(command, Executed.class, "Cannot update");
- return new Executed(common, status, command.executeAt(), promised, command.accepted(), waitingOn, command.writes(), command.result());
+ return validate(new Executed(common, status, command.executeAt(), promised, command.accepted(), waitingOn, command.writes(), command.result()));
}
public static Executed executed(Executed command, CommonAttributes common, SaveStatus status)
@@ -1042,7 +1139,7 @@ public abstract class Command implements CommonAttributes
public static Executed executed(CommonAttributes common, SaveStatus status, Timestamp executeAt, Ballot promised, Ballot accepted, WaitingOn waitingOn, Writes writes, Result result)
{
- return new Executed(common, status, executeAt, promised, accepted, waitingOn, writes, result);
+ return validate(new Executed(common, status, executeAt, promised, accepted, waitingOn, writes, result));
}
public Writes writes()
@@ -1225,18 +1322,10 @@ public abstract class Command implements CommonAttributes
this(committed.waitingOn);
}
- public Update(Ranges ranges, Unseekables<?> participants, Deps deps)
+ public Update(Deps deps)
{
this.deps = deps;
this.waitingOnCommit = new SimpleBitSet(deps.txnIdCount(), false);
- deps.keyDeps.forEach(ranges, 0, deps.keyDeps.txnIdCount(), this, null, (u, v, i) -> {
- u.waitingOnCommit.set(i);
- });
- // we select range deps on actual participants rather than covered ranges,
- // since we may otherwise adopt false dependencies for range txns
- deps.rangeDeps.forEach(participants, this, (u, i) -> {
- u.waitingOnCommit.set(u.deps.keyDeps.txnIdCount() + i);
- });
this.waitingOnApply = new SimpleBitSet(deps.txnIdCount(), false);
this.appliedOrInvalidated = new SimpleBitSet(deps.txnIdCount(), false);
}
@@ -1282,89 +1371,127 @@ public abstract class Command implements CommonAttributes
return true;
}
- public boolean removeWaitingOnApply(TxnId txnId)
+ void initialiseWaitingOnCommit(int index)
{
- int index = deps.indexOf(txnId);
- if (!waitingOnApply.get(index))
- return false;
+ waitingOnCommit.set(index);
+ }
- waitingOnApply = ensureMutable(waitingOnApply);
- waitingOnApply.unset(index);
- return true;
+ public boolean isEmpty()
+ {
+ return waitingOnApply.isEmpty() && waitingOnCommit.isEmpty();
}
- public boolean removeWaitingOn(TxnId txnId)
+ public TxnId minWaitingOnTxnId()
{
- return removeWaitingOnCommit(txnId) || removeWaitingOnApply(txnId);
+ return WaitingOn.minWaitingOnTxnId(deps, waitingOnCommit, waitingOnApply);
}
- public boolean removeInvalidatedAppliedOrTruncated(TxnId txnId)
+ public boolean isWaitingOn(int txnIdx)
{
- int index = this.deps.indexOf(txnId);
- return setAppliedOrInvalidated(index);
+ return waitingOnApply.get(txnIdx) || waitingOnCommit.get(txnIdx);
}
- public boolean isEmpty()
+ public boolean isWaitingOnRangeIdx(int txnIdx)
{
- return waitingOnApply.isEmpty() && waitingOnCommit.isEmpty();
+ txnIdx += deps.keyDeps.txnIdCount();
+ return waitingOnApply.get(txnIdx) || waitingOnCommit.get(txnIdx);
}
- public boolean removeApplied(TxnId txnId, WaitingOn propagate)
+ boolean removeWaitingOn(int i, boolean isAppliedOrInvalidated)
{
- int index = this.deps.indexOf(txnId);
- if (!setAppliedOrInvalidated(index))
+ if (!removeWaitingOn(i))
return false;
- if (!propagate.appliedOrInvalidated.isEmpty())
+ if (isAppliedOrInvalidated)
{
- forEachIntersection(propagate.deps.keyDeps.txnIds(), deps.keyDeps.txnIds(),
- (from, to, ignore, i1, i2) -> {
- if (from.get(i1))
- to.setAppliedOrInvalidated(i2);
- }, propagate.appliedOrInvalidated, this, null);
-
- forEachIntersection(propagate.deps.rangeDeps.txnIds(), deps.rangeDeps.txnIds(),
- (from, to, ignore, i1, i2) -> {
- if (from.isAppliedOrInvalidatedRangeIdx(i1))
- to.setAppliedOrInvalidatedRangeIdx(i2);
- }, propagate, this, null);
+ appliedOrInvalidated = ensureMutable(appliedOrInvalidated);
+ appliedOrInvalidated.set(i);
}
-
return true;
}
- public TxnId minWaitingOnTxnId()
+ public boolean removeWaitingOn(TxnId txnId)
{
- return WaitingOn.minWaitingOnTxnId(deps, waitingOnCommit, waitingOnApply);
+ int index = this.deps.indexOf(txnId);
+ return removeWaitingOn(index);
}
- boolean setAppliedOrInvalidatedRangeIdx(int i)
+ boolean removeWaitingOnRangeIdx(int i)
{
- return setAppliedOrInvalidated(i + deps.keyDeps.txnIdCount());
+ return removeWaitingOn(i + deps.keyDeps.txnIdCount());
}
- boolean setAppliedOrInvalidated(int i)
+ boolean removeWaitingOn(int i)
{
- if (appliedOrInvalidated.get(i))
- return false;
-
if (waitingOnCommit.get(i))
{
waitingOnCommit = ensureMutable(waitingOnCommit);
waitingOnCommit.unset(i);
+ return true;
}
else if (waitingOnApply.get(i))
{
waitingOnApply = ensureMutable(waitingOnApply);
waitingOnApply.unset(i);
+ return true;
}
else
{
return false;
}
+ }
+
+ /**
+ * Warning: DO NOT invoke this when you really mean removeWaitingOn.
+ * This propagates the applied/invalidated status to dependent transactions, which may
+ * adopt a different set of dependency relations on this transaction. If the transaction
+ * is e.g. partially stale, pre-bootstrap etc this could lead to an erroneous propagation
+ * unless the transaction is truly (and fully) applied or invalidated locally.
+ */
+ public boolean setAppliedOrInvalidated(TxnId txnId)
+ {
+ int index = this.deps.indexOf(txnId);
+ return setAppliedOrInvalidated(index);
+ }
+
+ boolean setAppliedOrInvalidatedRangeIdx(int i)
+ {
+ return setAppliedOrInvalidated(i + deps.keyDeps.txnIdCount());
+ }
+
+ public boolean setAppliedOrInvalidated(int i)
+ {
+ if (appliedOrInvalidated.get(i))
+ return false;
+
+ if (!removeWaitingOn(i))
+ return false;
appliedOrInvalidated = ensureMutable(appliedOrInvalidated);
appliedOrInvalidated.set(i);
+ return true;
+ }
+
+ public boolean setAppliedAndPropagate(TxnId txnId, WaitingOn propagate)
+ {
+ int index = this.deps.indexOf(txnId);
+ if (!setAppliedOrInvalidated(index))
+ return false;
+
+ if (!propagate.appliedOrInvalidated.isEmpty())
+ {
+ forEachIntersection(propagate.deps.keyDeps.txnIds(), deps.keyDeps.txnIds(),
+ (from, to, ignore, i1, i2) -> {
+ if (from.get(i1))
+ to.setAppliedOrInvalidated(i2);
+ }, propagate.appliedOrInvalidated, this, null);
+
+ forEachIntersection(propagate.deps.rangeDeps.txnIds(), deps.rangeDeps.txnIds(),
+ (from, to, ignore, i1, i2) -> {
+ if (from.isAppliedOrInvalidatedRangeIdx(i1))
+ to.setAppliedOrInvalidatedRangeIdx(i2);
+ }, propagate, this, null);
+ }
return true;
}
@@ -1422,8 +1549,9 @@ public abstract class Command implements CommonAttributes
}
else if (command.status() == Status.AcceptedInvalidate && command.executeAt() == null)
{
+ // TODO (now): reconsider this special-casing
Command.Accepted accepted = command.asAccepted();
- return Command.Accepted.accepted(attrs, accepted.saveStatus(), executeAt, ballot, accepted.accepted());
+ return Command.Accepted.accepted(attrs, SaveStatus.enrich(accepted.saveStatus(), SaveStatus.PreAccepted.known), executeAt, ballot, accepted.accepted());
}
else
{
@@ -1435,29 +1563,29 @@ public abstract class Command implements CommonAttributes
static Command.Accepted markDefined(Command command, CommonAttributes attributes, Ballot promised)
{
if (Command.isSameClass(command, Command.Accepted.class))
- return Command.Accepted.accepted(command.asAccepted(), attributes, SaveStatus.enrich(command.saveStatus(), Status.Known.DefinitionOnly), promised);
+ return Command.Accepted.accepted(command.asAccepted(), attributes, SaveStatus.enrich(command.saveStatus(), Known.DefinitionOnly), promised);
return (Command.Accepted) command.updateAttributes(attributes, promised);
}
static Command.Accepted accept(Command command, CommonAttributes attrs, Timestamp executeAt, Ballot ballot)
{
- return new Command.Accepted(attrs, SaveStatus.get(Status.Accepted, command.known()), executeAt, ballot, ballot);
+ return validate(new Command.Accepted(attrs, SaveStatus.get(Status.Accepted, command.known()), ballot, executeAt, ballot));
}
static Command.Accepted acceptInvalidated(Command command, Ballot ballot)
{
Timestamp executeAt = command.isDefined() ? command.asWitnessed().executeAt() : null;
- return new Command.Accepted(command, SaveStatus.get(Status.AcceptedInvalidate, command.known()), executeAt, ballot, ballot);
+ return validate(new Command.Accepted(command, SaveStatus.get(Status.AcceptedInvalidate, command.known()), ballot, executeAt, command.partialTxn(), null, ballot));
}
static Command.Committed commit(Command command, CommonAttributes attrs, Timestamp executeAt, Command.WaitingOn waitingOn)
{
- return Command.Committed.committed(attrs, SaveStatus.get(Status.Committed, command.known()), executeAt, command.promised(), command.accepted(), waitingOn);
+ return validate(Command.Committed.committed(attrs, SaveStatus.get(Status.Committed, command.known()), executeAt, command.promised(), command.accepted(), waitingOn));
}
static Command precommit(CommonAttributes attrs, Command command, Timestamp executeAt)
{
- return new Command.Accepted(attrs, SaveStatus.get(Status.PreCommitted, command.known()), executeAt, command.promised(), command.accepted());
+ return validate(new Command.Accepted(attrs, SaveStatus.get(Status.PreCommitted, command.known()), command.promised(), executeAt, command.accepted()));
}
static Command.Committed readyToExecute(Command.Committed command)
diff --git a/accord-core/src/main/java/accord/local/CommandStore.java b/accord-core/src/main/java/accord/local/CommandStore.java
index ae453818..3ba61142 100644
--- a/accord-core/src/main/java/accord/local/CommandStore.java
+++ b/accord-core/src/main/java/accord/local/CommandStore.java
@@ -18,8 +18,29 @@
package accord.local;
+import accord.api.ProgressLog;
+import accord.api.DataStore;
+import accord.coordinate.CollectDeps;
+import accord.local.Command.WaitingOn;
+
+import javax.annotation.Nullable;
+import accord.api.Agent;
+
+import accord.local.CommandStores.RangesForEpoch;
+import accord.primitives.Range;
+import accord.primitives.Txn.Kind.Kinds;
+import accord.utils.RelationMultiMap.SortedRelationList;
+import accord.utils.async.AsyncChain;
+
+import accord.api.ConfigurationService.EpochReady;
+import accord.utils.DeterministicIdentitySet;
+import accord.utils.Invariants;
+import accord.utils.ReducingRangeMap;
+import accord.utils.async.AsyncResult;
+
import java.util.AbstractMap.SimpleImmutableEntry;
import java.util.Collections;
+import java.util.HashMap;
import java.util.Map;
import java.util.NavigableMap;
import java.util.Objects;
@@ -30,16 +51,9 @@ import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.Supplier;
-import javax.annotation.Nullable;
import com.google.common.collect.ImmutableSortedMap;
-import accord.api.Agent;
-import accord.api.ConfigurationService.EpochReady;
-import accord.api.DataStore;
-import accord.api.ProgressLog;
-import accord.coordinate.CollectDeps;
-import accord.local.Command.WaitingOn;
import accord.primitives.FullRoute;
import accord.primitives.KeyDeps;
import accord.primitives.Participants;
@@ -49,14 +63,7 @@ import accord.primitives.Seekables;
import accord.primitives.Timestamp;
import accord.primitives.TxnId;
import accord.primitives.Unseekables;
-import accord.utils.DeterministicIdentitySet;
-import accord.utils.Invariants;
-import accord.utils.ReducingRangeMap;
-import accord.utils.RelationMultiMap.SortedRelationList;
-import accord.utils.async.AsyncChain;
-import accord.utils.async.AsyncResult;
import accord.utils.async.AsyncResults;
-import accord.local.CommandStores.RangesForEpoch;
import static accord.api.ConfigurationService.EpochReady.DONE;
import static accord.local.PreLoadContext.contextFor;
@@ -98,13 +105,13 @@ public abstract class CommandStore implements AgentExecutor
// TODO (desired): can better encapsulate by accepting only the newRangesForEpoch and deriving the add/remove ranges
public void add(long epoch, RangesForEpoch newRangesForEpoch, Ranges addRanges)
{
- RedundantBefore addRedundantBefore = RedundantBefore.create(addRanges, epoch, Long.MAX_VALUE, TxnId.NONE, TxnId.minForEpoch(epoch));
+ RedundantBefore addRedundantBefore = RedundantBefore.create(addRanges, epoch, Long.MAX_VALUE, TxnId.NONE, TxnId.NONE, TxnId.minForEpoch(epoch));
update(newRangesForEpoch, addRedundantBefore);
}
public void remove(long epoch, RangesForEpoch newRangesForEpoch, Ranges removeRanges)
{
- RedundantBefore addRedundantBefore = RedundantBefore.create(removeRanges, Long.MIN_VALUE, epoch, TxnId.NONE, TxnId.NONE);
+ RedundantBefore addRedundantBefore = RedundantBefore.create(removeRanges, Long.MIN_VALUE, epoch, TxnId.NONE, TxnId.NONE, TxnId.NONE);
update(newRangesForEpoch, addRedundantBefore);
}
@@ -145,6 +152,20 @@ public abstract class CommandStore implements AgentExecutor
protected RangesForEpoch rangesForEpoch;
// TODO (desired): merge with redundantBefore?
+ /**
+ * safeToRead is related to RedundantBefore, but a distinct concept.
+ * While bootstrappedAt defines the txnId bounds we expect to maintain data for locally,
+ * safeToRead defines executeAt bounds we can safely participate in transaction execution for.
+ * safeToRead is defined by the no-op transaction we execute after a bootstrap is initiated,
+ * and creates a global bound before which we know we have complete data from our bootstrap.
+ *
+ * There's a smearing period during bootstrap where some keys may be ahead of others, for instance,
+ * since we do not create a precise instant in the transaction log for bootstrap so as to avoid impeding execution.
+ *
+ * We also update safeToRead when we go stale, to remove ranges we may have bootstrapped but that are now known to
+ * be incomplete. In this case we permit transactions to execute in any order for the unsafe key ranges.
+ * But they may still be ordered for other key ranges they participate in.
+ */
private NavigableMap<Timestamp, Ranges> safeToRead = ImmutableSortedMap.of(Timestamp.NONE, Ranges.EMPTY);
private final Set<Bootstrap> bootstraps = Collections.synchronizedSet(new DeterministicIdentitySet<>());
// TODO (required): we must synchronise this on joining, prior to marking ourselves ready to coordinate
@@ -201,12 +222,12 @@ public abstract class CommandStore implements AgentExecutor
private static Timestamp maxApplied(SafeCommandStore safeStore, Seekables<?, ?> keysOrRanges, Ranges slice)
{
- return safeStore.mapReduce(keysOrRanges, slice, SafeCommandStore.TestKind.Ws,
+ return safeStore.mapReduce(keysOrRanges, slice, Kinds.Ws,
SafeCommandStore.TestTimestamp.STARTED_AFTER, Timestamp.NONE,
SafeCommandStore.TestDep.ANY_DEPS, null,
- Status.Applied, Status.Applied,
- (key, txnId, executeAt, status, max) -> Timestamp.max(max, executeAt),
- Timestamp.NONE, Timestamp.MAX);
+ Status.PreApplied, Status.Truncated,
+ (p1, key, txnId, executeAt, status, max) -> Timestamp.max(max, executeAt),
+ null, Timestamp.NONE, Timestamp.MAX::equals);
}
public AsyncChain<Timestamp> maxAppliedFor(Seekables<?, ?> keysOrRanges, Ranges slice)
@@ -262,16 +283,6 @@ public abstract class CommandStore implements AgentExecutor
this.safeToRead = newSafeToRead;
}
- public NavigableMap<TxnId, Ranges> bootstrapBeganAt()
- {
- return bootstrapBeganAt;
- }
-
- public NavigableMap<Timestamp, Ranges> safeToRead()
- {
- return safeToRead;
- }
-
public final void markExclusiveSyncPoint(SafeCommandStore safeStore, TxnId txnId, Ranges ranges)
{
// TODO (desired): narrow ranges to those that are owned
@@ -281,6 +292,14 @@ public abstract class CommandStore implements AgentExecutor
setRejectBefore(newRejectBefore);
}
+ public final void markExclusiveSyncPointApplied(SafeCommandStore safeStore, TxnId txnId, Ranges ranges)
+ {
+ // TODO (desired): narrow ranges to those that are owned
+ Invariants.checkArgument(txnId.rw() == ExclusiveSyncPoint);
+ RedundantBefore newRedundantBefore = RedundantBefore.merge(redundantBefore, RedundantBefore.create(ranges, txnId, TxnId.NONE, TxnId.NONE));
+ setRedundantBefore(newRedundantBefore);
+ }
+
final Timestamp preaccept(TxnId txnId, Seekables<?, ?> keys, SafeCommandStore safeStore, boolean permitFastPath)
{
NodeTimeService time = safeStore.time();
@@ -463,7 +482,7 @@ public abstract class CommandStore implements AgentExecutor
final void markBootstrapping(SafeCommandStore safeStore, TxnId globalSyncId, Ranges ranges)
{
setBootstrapBeganAt(bootstrap(globalSyncId, ranges, bootstrapBeganAt));
- RedundantBefore addRedundantBefore = RedundantBefore.create(ranges, Long.MIN_VALUE, Long.MAX_VALUE, TxnId.NONE, globalSyncId);
+ RedundantBefore addRedundantBefore = RedundantBefore.create(ranges, Long.MIN_VALUE, Long.MAX_VALUE, TxnId.NONE, TxnId.NONE, globalSyncId);
setRedundantBefore(RedundantBefore.merge(redundantBefore, addRedundantBefore));
DurableBefore addDurableBefore = DurableBefore.create(ranges, TxnId.NONE, TxnId.NONE);
setDurableBefore(DurableBefore.merge(durableBefore, addDurableBefore));
@@ -472,14 +491,36 @@ public abstract class CommandStore implements AgentExecutor
// TODO (expected): we can immediately truncate dependencies locally once an exclusiveSyncPoint applies, we don't need to wait for the whole shard
public void markShardDurable(SafeCommandStore safeStore, TxnId globalSyncId, Ranges ranges)
{
- // TODO (now): IMPORTANT we should insert additional entries for each epoch our ownership differs
ranges = ranges.slice(safeStore.ranges().allUntil(globalSyncId.epoch()), Minimal);
- RedundantBefore addRedundantBefore = RedundantBefore.create(ranges, Long.MIN_VALUE, Long.MAX_VALUE, globalSyncId, TxnId.NONE);
+ RedundantBefore addRedundantBefore = RedundantBefore.create(ranges, Long.MIN_VALUE, Long.MAX_VALUE, TxnId.NONE, globalSyncId, TxnId.NONE);
setRedundantBefore(RedundantBefore.merge(redundantBefore, addRedundantBefore));
DurableBefore addDurableBefore = DurableBefore.create(ranges, globalSyncId, globalSyncId);
setDurableBefore(DurableBefore.merge(durableBefore, addDurableBefore));
}
+ // TODO (expected): we can immediately truncate dependencies locally once an exclusiveSyncPoint applies, we don't need to wait for the whole shard
+ public void markShardStale(SafeCommandStore safeStore, Timestamp staleSince, Ranges ranges, boolean isSincePrecise)
+ {
+ Timestamp staleUntilAtLeast = staleSince;
+ if (isSincePrecise)
+ {
+ ranges = ranges.slice(safeStore.ranges().allAt(staleSince.epoch()), Minimal);
+ }
+ else
+ {
+ ranges = ranges.slice(safeStore.ranges().allSince(staleSince.epoch()), Minimal);
+ // make sure no in-progress bootstrap attempts will override the stale since for commands whose staleness bounds are unknown
+ staleUntilAtLeast = Timestamp.max(bootstrapBeganAt.lastKey(), staleUntilAtLeast);
+ }
+ agent.onStale(staleSince, ranges);
+
+ RedundantBefore addRedundantBefore = RedundantBefore.create(ranges, TxnId.NONE, TxnId.NONE, TxnId.NONE, staleUntilAtLeast);
+ setRedundantBefore(RedundantBefore.merge(redundantBefore, addRedundantBefore));
+ // find which ranges need to bootstrap, subtracting those already in progress that cover the id
+
+ markUnsafeToRead(ranges);
+ }
+
// MUST be invoked before CommandStore reference leaks to anyone
Supplier<EpochReady> initialise(long epoch, Ranges ranges)
{
@@ -490,7 +531,7 @@ public abstract class CommandStore implements AgentExecutor
return () -> new EpochReady(epoch, DONE, DONE, DONE, DONE);
}
- final Ranges safeToReadAt(Timestamp at)
+ public final Ranges safeToReadAt(Timestamp at)
{
return safeToRead.floorEntry(at).getValue();
}
@@ -506,6 +547,9 @@ public abstract class CommandStore implements AgentExecutor
return redundantBefore;
}
+ protected NavigableMap<TxnId, Ranges> bootstrapBeganAt() { return bootstrapBeganAt; }
+ protected NavigableMap<Timestamp, Ranges> safeToRead() { return safeToRead; }
+
public final boolean isRejectedIfNotPreAccepted(TxnId txnId, Unseekables<?> participants)
{
if (rejectBefore == null)
@@ -521,7 +565,8 @@ public abstract class CommandStore implements AgentExecutor
// TODO (required): make sure we have no races on HLC around SyncPoint else this resolution may not work (we need to know the micros equivalent timestamp of the snapshot)
KeyDeps keyDeps = builder.deps.keyDeps;
redundantBefore().foldl(keyDeps.keys(), (e, b, d, p2, ki, kj) -> {
- int txnIdx = d.txnIds().find(e.locallyRedundantBefore());
+ boolean isAppliedOrInvalidated = e.locallyAppliedOrInvalidatedBefore.compareTo(e.bootstrappedAt) >= 0;
+ int txnIdx = d.txnIds().find(isAppliedOrInvalidated ? e.locallyAppliedOrInvalidatedBefore : e.bootstrappedAt);
if (txnIdx < 0) txnIdx = -1 - txnIdx;
while (ki < kj)
{
@@ -530,26 +575,86 @@ public abstract class CommandStore implements AgentExecutor
if (ti < 0)
ti = -1 - ti;
for (int i = 0 ; i < ti ; ++i)
- b.setAppliedOrInvalidated(txnIdsForKey.getValueIndex(i));
+ b.removeWaitingOn(txnIdsForKey.getValueIndex(i), isAppliedOrInvalidated);
}
return b;
}, builder, keyDeps, null, ignore -> false);
- RangeDeps rangeDeps = builder.deps.rangeDeps;
- // TODO (now): slice to only those ranges we own, maybe don't even construct rangeDeps.covering()
- redundantBefore().foldl(participants, (e, b, d, rs, pi, pj) -> {
- // TODO (now): foldlInt so we can track the lower rangeidx bound and not revisit unnecessarily
- boolean useBootstrap = e.isLocalRedundancyViaBootstrap();
+
+ /**
+ * If we have to handle bootstrapping ranges for range transactions, these may only partially cover the
+ * transaction, in which case we should not remove the transaction as a dependency. But if it is fully
+ * covered by bootstrapping ranges then we *must* remove it as a dependency.
+ */
+ class RangeState
+ {
+ final WaitingOn.Update builder;
int txnIdx;
+ Range range;
+ Map<Integer, Ranges> partiallyBootstrapping;
+
+ RangeState(WaitingOn.Update builder)
{
- int tmp = d.txnIds().find(useBootstrap ? e.bootstrappedAt : e.redundantBefore);
- txnIdx = tmp < 0 ? -1 - tmp : tmp;
+ this.builder = builder;
}
- rangeDeps.forEach(participants, e.range, pi, pj, d, (d0, txnIdx0) -> {
- if (txnIdx0 < txnIdx)
- b.setAppliedOrInvalidatedRangeIdx(txnIdx0);
+
+ /**
+ * Are the participating ranges for the txn fully covered by bootstrapping ranges for this command store
+ */
+ boolean isFullyBootstrapping(int rangeTxnIdx)
+ {
+ if (partiallyBootstrapping == null)
+ partiallyBootstrapping = new HashMap<>();
+ Ranges prev = partiallyBootstrapping.get(rangeTxnIdx);
+ Ranges remaining = prev;
+ if (remaining == null) remaining = builder.deps.rangeDeps.ranges(rangeTxnIdx);
+ else Invariants.checkState(!remaining.isEmpty());
+ remaining = remaining.subtract(Ranges.of(range));
+ if (prev == null) Invariants.checkState(!remaining.isEmpty());
+ partiallyBootstrapping.put(txnIdx, remaining);
+ return remaining.isEmpty();
+ }
+ }
+ RangeDeps rangeDeps = builder.deps.rangeDeps;
+ // TODO (required, consider): slice to only those ranges we own, maybe don't even construct rangeDeps.covering()
+ redundantBefore().foldl(participants, (e, s, d, ps, pi, pj) -> {
+ // TODO (desired, efficiency): foldlInt so we can track the lower rangeidx bound and not revisit unnecessarily
+ WaitingOn.Update b = s.builder;
+ {
+ // find the txnIdx below which we are known to be fully redundant locally due to having been applied or invalidated
+ int tmp = d.txnIds().find(e.locallyAppliedOrInvalidatedBefore);
+ s.txnIdx = tmp < 0 ? -1 - tmp : tmp;
+ }
+ // remove intersecting transactions with known redundant txnId
+ d.forEach(ps, e.range, pi, pj, b, s, (b0, s0, txnIdx) -> {
+ if (txnIdx < s0.txnIdx)
+ b0.removeWaitingOnRangeIdx(txnIdx);
});
- return b;
- }, builder, rangeDeps, participants, ignore -> false);
+ if (e.bootstrappedAt.compareTo(e.locallyAppliedOrInvalidatedBefore) > 0)
+ {
+ // if we have any ranges where bootstrap is ahead of the latest known fully redundant txnId,
+ // we have to do a more complicated dance since this may imply only partial redundancy
+ // (we may still depend on the transaction for some other range)
+ {
+ int tmp = d.txnIds().find(e.bootstrappedAt);
+ s.txnIdx = tmp < 0 ? -1 - tmp : tmp;
+ }
+ s.range = e.range;
+ d.forEach(ps, e.range, pi, pj, b, s, (b0, s0, txnIdx) -> {
+ if (txnIdx < s0.txnIdx && b0.isWaitingOnRangeIdx(txnIdx))
+ {
+ if (b0.deps.rangeDeps.foldEachRange(txnIdx, s0.range, true, (r1, r2, p) -> p && r1.contains(r2)))
+ {
+ b0.removeWaitingOnRangeIdx(txnIdx);
+ }
+ else if (s0.isFullyBootstrapping(txnIdx))
+ {
+ b0.removeWaitingOnRangeIdx(txnIdx);
+ }
+ }
+ });
+ }
+ return s;
+ }, new RangeState(builder), rangeDeps, participants, ignore -> false);
}
public final boolean hasLocallyRedundantDependencies(TxnId minimumDependencyId, Timestamp executeAt, Participants<?> participantsOfWaitingTxn)
@@ -557,7 +662,7 @@ public abstract class CommandStore implements AgentExecutor
// TODO (required): consider race conditions when bootstrapping into an active command store, that may have seen a higher txnId than this?
// might benefit from maintaining a per-CommandStore largest TxnId register to ensure we allocate a higher TxnId for our ExclSync,
// or from using whatever summary records we have for the range, once we maintain them
- return redundantBefore.status(minimumDependencyId, executeAt, participantsOfWaitingTxn).compareTo(RedundantStatus.PARTIALLY_PRE_BOOTSTRAP) >= 0;
+ return redundantBefore.status(minimumDependencyId, executeAt, participantsOfWaitingTxn).compareTo(RedundantStatus.PARTIALLY_PRE_BOOTSTRAP_OR_STALE) >= 0;
}
final synchronized void markUnsafeToRead(Ranges ranges)
@@ -566,9 +671,10 @@ public abstract class CommandStore implements AgentExecutor
setSafeToRead(purgeHistory(safeToRead, ranges));
}
- final synchronized void markSafeToRead(Timestamp at, Ranges ranges)
+ final synchronized void markSafeToRead(Timestamp forBootstrapAt, Timestamp at, Ranges ranges)
{
- setSafeToRead(purgeAndInsert(safeToRead, at, ranges));
+ Ranges validatedSafeToRead = redundantBefore.validateSafeToRead(forBootstrapAt, ranges);
+ setSafeToRead(purgeAndInsert(safeToRead, at, validatedSafeToRead));
}
private static <T extends Timestamp> ImmutableSortedMap<T, Ranges> bootstrap(T at, Ranges ranges, NavigableMap<T, Ranges> bootstrappedAt)
diff --git a/accord-core/src/main/java/accord/local/CommandStores.java b/accord-core/src/main/java/accord/local/CommandStores.java
index a1308b0e..9c70c8e6 100644
--- a/accord-core/src/main/java/accord/local/CommandStores.java
+++ b/accord-core/src/main/java/accord/local/CommandStores.java
@@ -38,6 +38,7 @@ import accord.api.Key;
import accord.api.ProgressLog;
import accord.api.RoutingKey;
import accord.local.CommandStore.EpochUpdateHolder;
+import accord.primitives.EpochSupplier;
import accord.primitives.Range;
import accord.primitives.Ranges;
import accord.primitives.Routables;
@@ -45,6 +46,7 @@ import accord.primitives.Route;
import accord.primitives.RoutingKeys;
import accord.primitives.Timestamp;
import accord.primitives.TxnId;
+import accord.primitives.Unseekables;
import accord.topology.Topology;
import accord.utils.Invariants;
import accord.utils.MapReduce;
@@ -53,6 +55,8 @@ import accord.utils.RandomSource;
import accord.utils.async.AsyncChain;
import accord.utils.async.AsyncChains;
import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
import org.agrona.collections.Hashing;
import org.agrona.collections.Int2ObjectHashMap;
@@ -61,6 +65,7 @@ import org.slf4j.LoggerFactory;
import static accord.api.ConfigurationService.EpochReady.done;
import static accord.local.PreLoadContext.empty;
+import static accord.primitives.EpochSupplier.constant;
import static accord.primitives.Routables.Slice.Minimal;
import static accord.utils.Invariants.checkArgument;
import static java.util.stream.Collectors.toList;
@@ -129,7 +134,9 @@ public abstract class CommandStores
}
}
- // TODO (now): split into deterministic and non-deterministic methods; ensure we only use deterministic methods during commands execution
+ // TODO (required): ensure all deterministic results across runs; updating RangesForEpoch should happen once per
+ // execution batch and be logged
+ // TODO (expected): merge with RedundantBefore, and standardise executeRanges() to treat removing stale ranges the same as adding new epoch ranges
public static class RangesForEpoch
{
final long[] epochs;
@@ -167,11 +174,6 @@ public abstract class CommandStores
return allAt(txnId);
}
- public @Nonnull Ranges safeToReadAt(Timestamp at)
- {
- return allAt(at).slice(store.safeToReadAt(at), Minimal);
- }
-
public @Nonnull Ranges unsafeToReadAt(Timestamp at)
{
return allAt(at).subtract(store.safeToReadAt(at));
@@ -184,18 +186,43 @@ public abstract class CommandStores
public @Nonnull Ranges allAt(long epoch)
{
- int i = Arrays.binarySearch(epochs, epoch);
- if (i < 0) i = -2 -i;
+ int i = floorIndex(epoch);
if (i < 0) return Ranges.EMPTY;
return ranges[i];
}
- public @Nonnull Ranges allBetween(Timestamp fromInclusive, Timestamp toInclusive)
+ /**
+ * Extend a previously computed set of Ranges that included {@code fromInclusive}
+ * to include {@code toInclusive}
+ */
+ public @Nonnull Ranges extend(Ranges extend, Timestamp fromInclusive, Timestamp toInclusive)
+ {
+ return extend(extend, fromInclusive.epoch(), toInclusive.epoch());
+ }
+
+ /**
+ * Extend a previously computed set of Ranges that included {@code fromInclusive}
+ * to include ranges up to {@code toInclusive}
+ */
+ public @Nonnull Ranges extend(Ranges extend, long fromInclusive, long toInclusive)
+ {
+ if (fromInclusive == toInclusive)
+ return extend;
+
+ int startIndex = 1 + floorIndex(fromInclusive);
+ int endIndex = 1 + floorIndex(toInclusive);
+ for (int i = startIndex ; i < endIndex; ++i)
+ extend = extend.with(ranges[i]); // want to always return extend for equal ranges, so same ranges are identity-equals
+
+ return extend;
+ }
+
+ public @Nonnull Ranges allBetween(EpochSupplier fromInclusive, EpochSupplier toInclusive)
{
return allBetween(fromInclusive.epoch(), toInclusive.epoch());
}
- public @Nonnull Ranges allBetween(long fromInclusive, Timestamp toInclusive)
+ public @Nonnull Ranges allBetween(long fromInclusive, EpochSupplier toInclusive)
{
return allBetween(fromInclusive, toInclusive.epoch());
}
@@ -231,6 +258,11 @@ public abstract class CommandStores
return allInternal(0, 1 + floorIndex(toInclusive));
}
+ public @Nonnull Ranges allSince(long fromInclusive)
+ {
+ return allInternal(Math.max(0, floorIndex(fromInclusive)), epochs.length);
+ }
+
private int floorIndex(long epoch)
{
int i = Arrays.binarySearch(epochs, epoch);
@@ -271,6 +303,21 @@ public abstract class CommandStores
return IntStream.range(0, ranges.length).mapToObj(i -> epochs[i] + ": " + ranges[i])
.collect(Collectors.joining(", "));
}
+
+ public @Nullable EpochSupplier latestEpochWithNewParticipants(long sinceEpoch, Unseekables<?> keysOrRanges)
+ {
+ int i = floorIndex(sinceEpoch);
+ long latest = -1;
+ Ranges existing = i < 0 ? Ranges.EMPTY : ranges[i];
+ while (++i < ranges.length)
+ {
+ if (ranges[i].subtract(existing).intersects(keysOrRanges))
+ latest = epochs[i];
+ existing = existing.with(ranges[i]);
+ }
+
+ return latest == -1 ? null : constant(latest);
+ }
}
static class Snapshot
diff --git a/accord-core/src/main/java/accord/local/Commands.java b/accord-core/src/main/java/accord/local/Commands.java
index 3193e70d..88e925b7 100644
--- a/accord-core/src/main/java/accord/local/Commands.java
+++ b/accord-core/src/main/java/accord/local/Commands.java
@@ -67,14 +67,17 @@ import static accord.local.Commands.Cleanup.ERASE;
import static accord.local.Commands.Cleanup.NO;
import static accord.local.Commands.Cleanup.TRUNCATE;
import static accord.local.Commands.EnsureAction.Add;
-import static accord.local.Commands.EnsureAction.Check;
+import static accord.local.Commands.EnsureAction.TryAdd;
import static accord.local.Commands.EnsureAction.Ignore;
import static accord.local.Commands.EnsureAction.Set;
import static accord.local.Commands.EnsureAction.TrySet;
+import static accord.local.RedundantBefore.PreBootstrapOrStale.FULLY;
import static accord.local.RedundantStatus.NOT_OWNED;
-import static accord.local.RedundantStatus.PRE_BOOTSTRAP;
+import static accord.local.RedundantStatus.PRE_BOOTSTRAP_OR_STALE;
import static accord.local.SaveStatus.Applying;
import static accord.local.SaveStatus.Erased;
+import static accord.local.SaveStatus.LocalExecution.ReadyToExclude;
+import static accord.local.SaveStatus.LocalExecution.WaitingToExecute;
import static accord.local.SaveStatus.TruncatedApply;
import static accord.local.SaveStatus.TruncatedApplyWithOutcome;
import static accord.local.SaveStatus.Uninitialised;
@@ -92,7 +95,10 @@ import static accord.local.Status.PreApplied;
import static accord.local.Status.PreCommitted;
import static accord.local.Status.ReadyToExecute;
import static accord.local.Status.Truncated;
+import static accord.primitives.Routables.Slice.Minimal;
import static accord.primitives.Route.isFullRoute;
+import static accord.primitives.Txn.Kind.ExclusiveSyncPoint;
+import static accord.utils.Invariants.illegalState;
public class Commands
{
@@ -145,7 +151,7 @@ public class Commands
if (command.hasBeen(Truncated))
{
logger.trace("{}: skipping preaccept - command is truncated", txnId);
- return command.is(Invalidated) ? AcceptOutcome.Redundant : AcceptOutcome.Truncated;
+ return command.is(Invalidated) ? AcceptOutcome.RejectedBallot : AcceptOutcome.Truncated;
}
int compareBallots = command.promised().compareTo(ballot);
@@ -165,13 +171,13 @@ public class Commands
return ballot.equals(Ballot.ZERO) ? AcceptOutcome.Redundant : AcceptOutcome.Success;
}
- Ranges coordinateRanges = coordinateRanges(safeStore, txnId, acceptEpoch);
- Invariants.checkState(!coordinateRanges.isEmpty());
- ProgressShard shard = progressShard(route, progressKey, coordinateRanges);
- Invariants.checkState(validate(command.status(), command, Ranges.EMPTY, coordinateRanges, shard, route, Set, partialTxn, Set, null, Ignore));
+ Ranges preacceptRanges = preacceptRanges(safeStore, txnId, acceptEpoch);
+ Invariants.checkState(!preacceptRanges.isEmpty());
+ ProgressShard shard = progressShard(route, progressKey, preacceptRanges);
+ Invariants.checkState(validate(command.status(), command, Ranges.EMPTY, preacceptRanges, shard, route, Set, partialTxn, Set, null, Ignore));
// FIXME: this should go into a consumer method
- CommonAttributes attrs = set(safeStore, command, command, Ranges.EMPTY, coordinateRanges, shard, route, partialTxn, Set, null, Ignore);
+ CommonAttributes attrs = set(safeStore, command, command, Ranges.EMPTY, preacceptRanges, shard, route, partialTxn, Set, null, Ignore);
if (command.executeAt() == null)
{
// unlike in the Accord paper, we partition shards within a node, so that to ensure a total order we must either:
@@ -224,7 +230,7 @@ public class Commands
if (command.is(Invalidated))
{
logger.trace("{}: skipping accept - command is invalidated", txnId);
- return AcceptOutcome.Redundant;
+ return AcceptOutcome.RejectedBallot;
}
if (command.is(Truncated))
@@ -285,7 +291,7 @@ public class Commands
}
logger.trace("{}: skipping accept invalidated - already committed ({})", command.txnId(), command.status());
- return AcceptOutcome.Redundant;
+ return AcceptOutcome.RejectedBallot;
}
if (command.promised().compareTo(ballot) > 0)
@@ -328,16 +334,14 @@ public class Commands
Ranges coordinateRanges = coordinateRanges(safeStore, txnId);
Ranges acceptRanges = acceptRanges(safeStore, txnId, executeAt, coordinateRanges);
- // TODO (expected, consider): consider ranges between coordinateRanges and executeRanges? Perhaps don't need them
- Ranges executeRanges = executeRanges(safeStore, executeAt);
ProgressShard shard = progressShard(route, progressKey, coordinateRanges);
- if (!validate(command.status(), command, acceptRanges, executeRanges, shard, route, Check, partialTxn, Add, partialDeps, Set))
+ if (!validate(command.status(), command, coordinateRanges, acceptRanges, shard, route, Add, partialTxn, Add, partialDeps, Set))
return CommitOutcome.Insufficient;
// FIXME: split up set
- CommonAttributes attrs = set(safeStore, command, command, acceptRanges, executeRanges, shard, route, partialTxn, Add, partialDeps, Set);
+ CommonAttributes attrs = set(safeStore, command, command, coordinateRanges, acceptRanges, shard, route, partialTxn, Add, partialDeps, Set);
logger.trace("{}: committed with executeAt: {}, deps: {}", txnId, executeAt, partialDeps);
WaitingOn waitingOn = initialiseWaitingOn(safeStore, txnId, executeAt, attrs.partialDeps(), attrs.route());
@@ -373,13 +377,10 @@ public class Commands
CommonAttributes attrs = command;
if (command.route() == null || !command.route().kind().isFullRoute())
- {
- if (!route.kind().isFullRoute())
- return CommitOutcome.Insufficient;
attrs = updateRoute(command, route);
- }
safeCommand.precommit(attrs, executeAt);
+ safeStore.progressLog().precommitted(command);
safeStore.notifyListeners(safeCommand);
logger.trace("{}: precommitted with executeAt: {}", txnId, executeAt);
return CommitOutcome.Success;
@@ -452,7 +453,7 @@ public class Commands
safeStore.notifyListeners(safeCommand, command, command.durableListeners(), safeCommand.transientListeners());
}
- public enum ApplyOutcome {Success, Redundant, Insufficient}
+ public enum ApplyOutcome { Success, Redundant, Insufficient }
public static ApplyOutcome apply(SafeCommandStore safeStore, SafeCommand safeCommand, TxnId txnId, Route<?> route, @Nullable RoutingKey progressKey, Timestamp executeAt, @Nullable PartialDeps partialDeps, @Nullable PartialTxn partialTxn, Writes writes, Result result)
{
@@ -471,14 +472,13 @@ public class Commands
Ranges coordinateRanges = coordinateRanges(safeStore, txnId);
Ranges acceptRanges = acceptRanges(safeStore, txnId, executeAt, coordinateRanges);
- Ranges executeRanges = executeRanges(safeStore, executeAt);
ProgressShard shard = progressShard(route, progressKey, coordinateRanges);
- if (!validate(command.status(), command, acceptRanges, executeRanges, shard, route, Check, partialTxn, Add, partialDeps, command.hasBeen(Committed) ? Check : TrySet))
+ if (!validate(command.status(), command, coordinateRanges, acceptRanges, shard, route, TryAdd, partialTxn, Add, partialDeps, command.hasBeen(Committed) ? TryAdd : TrySet, safeStore))
return ApplyOutcome.Insufficient; // TODO (expected, consider): this should probably be an assertion failure if !TrySet
- CommonAttributes attrs = set(safeStore, command, command, acceptRanges, executeRanges, shard, route, partialTxn, Add, partialDeps, command.hasBeen(Committed) ? Check : TrySet);
+ CommonAttributes attrs = set(safeStore, command, command, coordinateRanges, acceptRanges, shard, route, partialTxn, Add, partialDeps, command.hasBeen(Committed) ? TryAdd : TrySet);
WaitingOn waitingOn = !command.hasBeen(Committed) ? initialiseWaitingOn(safeStore, txnId, executeAt, attrs.partialDeps(), attrs.route()) : command.asCommitted().waitingOn();
safeCommand.preapplied(attrs, executeAt, waitingOn, writes, result);
@@ -534,7 +534,7 @@ public class Commands
{
logger.trace("{} applied, setting status to Applied and notifying listeners", txnId);
SafeCommand safeCommand = safeStore.get(txnId);
- safeCommand.applied();
+ Command.Executed command = safeCommand.applied();
safeStore.notifyListeners(safeCommand);
}
@@ -549,14 +549,14 @@ public class Commands
/**
* The ranges for which we participate in the consensus decision of when a transaction executes
*/
- private static Ranges coordinateRanges(SafeCommandStore safeStore, TxnId txnId, long untilEpoch)
+ private static Ranges preacceptRanges(SafeCommandStore safeStore, TxnId txnId, long untilEpoch)
{
return safeStore.ranges().allBetween(txnId.epoch(), untilEpoch);
}
private static Ranges acceptRanges(SafeCommandStore safeStore, TxnId txnId, Timestamp executeAt, Ranges coordinateRanges)
{
- return txnId.epoch() == executeAt.epoch() ? coordinateRanges : safeStore.ranges().allBetween(txnId, executeAt);
+ return safeStore.ranges().extend(coordinateRanges, txnId, executeAt);
}
private static Ranges executeRanges(SafeCommandStore safeStore, Timestamp executeAt)
@@ -669,6 +669,12 @@ public class Commands
// but: if we later support transitive dependency elision this could be dangerous
logger.trace("{}: applying no-op", command.txnId());
safeCommand.applied();
+ if (command.txnId().rw() == ExclusiveSyncPoint)
+ {
+ Ranges ranges = safeStore.ranges().allAt(command.txnId().epoch());
+ ranges = command.route().slice(ranges, Minimal).participants().toRanges();
+ safeStore.commandStore().markExclusiveSyncPointApplied(safeStore, command.txnId(), ranges);
+ }
safeStore.notifyListeners(safeCommand);
return true;
}
@@ -677,22 +683,30 @@ public class Commands
}
}
- protected static WaitingOn initialiseWaitingOn(SafeCommandStore safeStore, TxnId waitingId, Timestamp executeWaitingAt, PartialDeps partialDeps, Route<?> route)
+ protected static WaitingOn initialiseWaitingOn(SafeCommandStore safeStore, TxnId waitingId, Timestamp executeWaitingAt, PartialDeps deps, Route<?> route)
{
Ranges ranges = safeStore.ranges().allAt(executeWaitingAt);
- Unseekables<?> executionParticipants = route.participants().slice(ranges);
- WaitingOn.Update update = new WaitingOn.Update(ranges, executionParticipants, partialDeps);
+ Unseekables<?> executionParticipants = route.participants().slice(ranges, Minimal);
+ WaitingOn.Update update = new WaitingOn.Update(deps);
+ deps.keyDeps.forEach(ranges, 0, deps.keyDeps.txnIdCount(), update, null, (u, v, i) -> {
+ u.initialiseWaitingOnCommit(i);
+ });
+ // we select range deps on actual participants rather than covered ranges,
+ // since we may otherwise adopt false dependencies for range txns
+ deps.rangeDeps.forEach(executionParticipants, update, (u, i) -> {
+ u.initialiseWaitingOnCommit(u.deps.keyDeps.txnIdCount() + i);
+ });
return updateWaitingOn(safeStore, waitingId, executeWaitingAt, update, route.participants()).build();
}
- protected static WaitingOn.Update updateWaitingOn(SafeCommandStore safeStore, TxnId txnId, Timestamp executeAt, WaitingOn.Update update, Participants<?> participants)
+ protected static WaitingOn.Update updateWaitingOn(SafeCommandStore safeStore, TxnId waitingId, Timestamp executeAt, WaitingOn.Update update, Participants<?> participants)
{
CommandStore commandStore = safeStore.commandStore();
TxnId minWaitingOnTxnId = update.minWaitingOnTxnId();
if (minWaitingOnTxnId != null && commandStore.hasLocallyRedundantDependencies(update.minWaitingOnTxnId(), executeAt, participants))
safeStore.commandStore().removeRedundantDependencies(participants, update);
- update.forEachWaitingOnCommit(safeStore, update, txnId, executeAt, (safeStore0, upd, id, exec, i) -> {
+ update.forEachWaitingOnCommit(safeStore, update, waitingId, executeAt, (safeStore0, upd, id, exec, i) -> {
// TODO (expected): load read-only to reduce overhead; upgrade only if we need to remove listener
SafeCommand dep = safeStore0.ifLoadedAndInitialised(upd.deps.txnId(i));
if (dep == null || !dep.current().hasBeen(PreCommitted))
@@ -700,7 +714,7 @@ public class Commands
updateWaitingOn(safeStore0, id, exec, upd, dep);
});
- update.forEachWaitingOnApply(safeStore, update, txnId, executeAt, (store, upd, id, exec, i) -> {
+ update.forEachWaitingOnApply(safeStore, update, waitingId, executeAt, (store, upd, id, exec, i) -> {
SafeCommand dep = store.ifLoadedAndInitialised(upd.deps.txnId(i));
if (dep == null || !dep.current().hasBeen(PreCommitted))
return;
@@ -723,13 +737,15 @@ public class Commands
{
logger.trace("{}: {} is truncated. Stop listening and removing from waiting on commit set.", waitingId, dependencyId);
dependencySafeCommand.removeListener(new ProxyListener(waitingId));
- return waitingOn.removeInvalidatedAppliedOrTruncated(dependencyId);
+ // TODO (required): provide clear mechanism for translating SaveStatus -> isAppliedOrInvalidated
+ // so can propagate to setAppliedOrInvalidated or removeWaitingOn as appropriate
+ return waitingOn.removeWaitingOn(dependencyId);
}
else if (dependency.is(Invalidated))
{
logger.trace("{}: {} is invalidated. Stop listening and removing from waiting on commit set.", waitingId, dependencyId);
dependencySafeCommand.removeListener(new ProxyListener(waitingId));
- return waitingOn.removeInvalidatedAppliedOrTruncated(dependencyId);
+ return waitingOn.setAppliedOrInvalidated(dependencyId);
}
else if (dependency.executeAt().compareTo(executeWaitingAt) > 0 && !waitingId.rw().awaitsFutureDeps())
{
@@ -742,7 +758,7 @@ public class Commands
{
logger.trace("{}: {} has been applied. Stop listening and removing from waiting on apply set.", waitingId, dependencyId);
dependencySafeCommand.removeListener(new ProxyListener(waitingId));
- return waitingOn.removeApplied(dependencyId, dependency.asCommitted().waitingOn());
+ return waitingOn.setAppliedAndPropagate(dependencyId, dependency.asCommitted().waitingOn());
}
else if (waitingOn.removeWaitingOnCommit(dependencyId))
{
@@ -755,7 +771,7 @@ public class Commands
{
return false;
}
- else if (safeStore.isFullyPreBootstrap(dependency, waitingOn.deps.participants(dependency.txnId())))
+ else if (safeStore.isFullyPreBootstrapOrStale(dependency, waitingOn.deps.participants(dependency.txnId())))
{
// TODO (expected): erase or otherwise prevent from executing
return false;
@@ -808,15 +824,22 @@ public class Commands
// TODO (now): document and justify all calls
public static void setTruncatedApply(SafeCommandStore safeStore, SafeCommand safeCommand)
{
- cleanup(safeStore, safeCommand, TRUNCATE, true);
+ setTruncatedApply(safeStore, safeCommand, null);
+ }
+
+ public static void setTruncatedApply(SafeCommandStore safeStore, SafeCommand safeCommand, Route<?> maybeFullRoute)
+ {
+ cleanup(safeStore, safeCommand, maybeFullRoute, TRUNCATE, true);
}
public static void setErased(SafeCommandStore safeStore, SafeCommand safeCommand)
{
- cleanup(safeStore, safeCommand, ERASE, true);
+ Listeners.Immutable durableListeners = safeCommand.current().durableListeners();
+ Command command = cleanup(safeStore, safeCommand, null, ERASE, true);
+ safeStore.notifyListeners(safeCommand, command, durableListeners, safeCommand.transientListeners());
}
- public static Command cleanup(SafeCommandStore safeStore, SafeCommand safeCommand, Cleanup cleanup, boolean notifyListeners)
+ public static Command cleanup(SafeCommandStore safeStore, SafeCommand safeCommand, @Nullable Unseekables<?> maybeFullRoute, Cleanup cleanup, boolean notifyListeners)
{
Command command = safeCommand.current();
@@ -826,7 +849,7 @@ public class Commands
// TODO (desired): consider if there are better invariants we can impose for undecided transactions, to verify they aren't later committed (should be detected already, but more is better)
// note that our invariant here is imperfectly applied to keep the code cleaner: we don't verify that the caller was safe to invoke if we don't already have a route in the command and we're only PreCommitted
Invariants.checkState(command.hasBeen(Applied) || !command.hasBeen(PreCommitted)
- || (command.route() == null || Infer.safeToCleanup(safeStore, command, command.route(), command.executeAt()) || safeStore.isFullyPreBootstrap(command, command.route().participants()))
+ || (command.route() == null || Infer.safeToCleanup(safeStore, command, command.route(), command.executeAt()) || safeStore.isFullyPreBootstrapOrStale(command, command.route().participants()))
, "Command %s could not be truncated", command);
Command result;
@@ -844,8 +867,8 @@ public class Commands
case TRUNCATE:
Invariants.checkState(command.saveStatus().compareTo(TruncatedApply) < 0);
- if (command.hasBeen(PreCommitted)) result = truncatedApply(command);
- else result = erased(command); // TODO (expected): check if we are stale; if not, is erase anyway correct outcome?
+ if (command.hasBeen(PreCommitted)) result = truncatedApply(command, Route.tryCastToFullRoute(maybeFullRoute));
+ else result = command; // do nothing; we don't have enough information
break;
case ERASE:
@@ -867,7 +890,7 @@ public class Commands
if (command.saveStatus().compareTo(cleanup.appliesIfNot) >= 0)
return false;
- cleanup(safeStore, safeCommand, cleanup, true);
+ cleanup(safeStore, safeCommand, maybeFullRoute, cleanup, true);
return true;
}
@@ -879,7 +902,7 @@ public class Commands
Route<?> route = command.route();
if (!Route.isFullRoute(route))
{
- if (command.known().hasFullRoute()) throw new IllegalStateException(command.saveStatus() + " should include full route, but only found " + route);
+ if (command.known().hasFullRoute()) illegalState(command.saveStatus() + " should include full route, but only found " + route);
else if (Route.isFullRoute(maybeFullRoute)) route = Route.castToFullRoute(maybeFullRoute);
else return Cleanup.NO;
}
@@ -902,7 +925,7 @@ public class Commands
if (durableBefore.min(txnId) == Universal)
{
if (status.hasBeen(PreCommitted) && !status.hasBeen(Applied)) // TODO (expected): may be stale
- throw new IllegalStateException("Loading universally-durable command that has been PreCommitted but not Applied");
+ illegalState("Loading universally-durable command that has been PreCommitted but not Applied");
return Cleanup.ERASE;
}
@@ -919,14 +942,16 @@ public class Commands
// TODO (expected): we can impose additional validations here IF we receive an epoch upper bound
// TODO (expected): we should be more robust to the presence/absence of executeAt
if (route.isParticipatingHomeKey() || redundantBefore.get(txnId, toEpoch, route.homeKey()) == NOT_OWNED)
- throw new IllegalStateException("Command " + txnId + " that is being loaded is not owned by this shard on route " + route);
+ illegalState("Command " + txnId + " that is being loaded is not owned by this shard on route " + route);
case LIVE:
- case PARTIALLY_PRE_BOOTSTRAP:
- case PRE_BOOTSTRAP:
- return NO;
+ case PARTIALLY_PRE_BOOTSTRAP_OR_STALE:
+ case PRE_BOOTSTRAP_OR_STALE:
+ case REDUNDANT_PRE_BOOTSTRAP_OR_STALE:
case LOCALLY_REDUNDANT:
- if (enforceInvariants && status.hasBeen(PreCommitted) && !status.hasBeen(Applied)) // TODO (required): may be stale
- throw new IllegalStateException("Loading redundant command that has been PreCommitted but not Applied");
+ return NO;
+ case SHARD_REDUNDANT:
+ if (enforceInvariants && status.hasBeen(PreCommitted) && !status.hasBeen(Applied) && redundantBefore.preBootstrapOrStale(txnId, toEpoch, route.participants()) != FULLY)
+ illegalState("Loading redundant command that has been PreCommitted but not Applied");
}
Durability min = durableBefore.min(txnId, route);
@@ -934,17 +959,18 @@ public class Commands
{
default:
case Local:
- case DurableOrInvalidated:
throw new AssertionError("Unexpected durability: " + min);
+
case NotDurable:
if (durability == Majority)
return Cleanup.TRUNCATE;
-
return Cleanup.TRUNCATE_WITH_OUTCOME;
+ case MajorityOrInvalidated:
case Majority:
return Cleanup.TRUNCATE;
+ case UniversalOrInvalidated:
case Universal:
return Cleanup.ERASE;
}
@@ -961,7 +987,7 @@ public class Commands
return command;
CommonAttributes attrs = route == null ? command : updateRoute(command, route);
- if (executeAt != null && command.status().hasBeen(Committed) && !command.asCommitted().executeAt().equals(executeAt))
+ if (executeAt != null && command.status().hasBeen(Committed) && !command.executeAt().equals(executeAt))
safeStore.agent().onInconsistentTimestamp(command, command.asCommitted().executeAt(), executeAt);
attrs = attrs.mutable().durability(durability);
command = safeCommand.updateAttributes(attrs);
@@ -1017,13 +1043,15 @@ public class Commands
{
default: throw new AssertionError("Unexpected redundant status: " + redundantStatus);
case NOT_OWNED: throw new AssertionError("Invalid state: waiting for execution of command that is not owned at the execution time");
+ case SHARD_REDUNDANT:
case LOCALLY_REDUNDANT:
- case PRE_BOOTSTRAP:
- removeRedundantDependencies(safeStore, prevSafe, txnIds[depth], redundantStatus == PRE_BOOTSTRAP);
+ case REDUNDANT_PRE_BOOTSTRAP_OR_STALE:
+ case PRE_BOOTSTRAP_OR_STALE:
+ removeRedundantDependencies(safeStore, prevSafe, txnIds[depth]);
prevSafe = get(safeStore, --depth - 1);
break;
case LIVE:
- case PARTIALLY_PRE_BOOTSTRAP:
+ case PARTIALLY_PRE_BOOTSTRAP_OR_STALE:
initialise(safeStore, depth);
}
}
@@ -1085,18 +1113,23 @@ public class Commands
}
else if ((directlyBlockedOnCommit = waitingOn.nextWaitingOnCommit()) != null)
{
- push(directlyBlockedOnCommit, LocalExecution.ReadyToExclude);
+ push(directlyBlockedOnCommit, WaitingToExecute);
prevSafe = curSafe;
}
else
{
- if (cur.hasBeen(Committed) && !cur.is(ReadyToExecute) && cur.saveStatus() != Applying && !cur.asCommitted().isWaitingOnDependency())
+ if (cur.hasBeen(Committed))
{
- if (!maybeExecute(safeStore, curSafe, false, false))
- throw new AssertionError("Is able to Apply, but has not done so");
- // loop and re-test the command's status; we may still want to notify blocking, esp. if not homeShard
- continue;
+ if (!cur.is(ReadyToExecute) && cur.saveStatus() != Applying && !cur.asCommitted().isWaitingOnDependency())
+ {
+ if (!maybeExecute(safeStore, curSafe, false, false))
+ throw new AssertionError("Is able to Apply, but has not done so");
+ // loop and re-test the command's status; we may still want to notify blocking, esp. if not homeShard
+ continue;
+ }
}
+ else if (!cur.hasBeen(PreCommitted))
+ until = ReadyToExclude;
Timestamp executeAt = prev == null ? cur.executeAt() : prev.executeAt();
Participants<?> participants = prev != null // TODO (desired): slightly costly to invert a large partialDeps collection
@@ -1109,22 +1142,23 @@ public class Commands
default: throw new AssertionError("Unknown redundant status: " + redundantStatus);
case NOT_OWNED: throw new AssertionError("Invalid state: waiting for execution of command that is not owned at the execution time");
case LIVE:
- case PARTIALLY_PRE_BOOTSTRAP:
+ case PARTIALLY_PRE_BOOTSTRAP_OR_STALE:
logger.trace("{} blocked on {} until {}", txnIds[0], cur.txnId(), until);
safeStore.progressLog().waiting(curSafe, until, null, participants);
break loop;
- case PRE_BOOTSTRAP:
+ case PRE_BOOTSTRAP_OR_STALE:
+ case REDUNDANT_PRE_BOOTSTRAP_OR_STALE:
case LOCALLY_REDUNDANT:
- Invariants.checkState(cur.hasBeen(Applied) || !cur.hasBeen(PreCommitted) || redundantStatus == PRE_BOOTSTRAP);
+ case SHARD_REDUNDANT:
+ Invariants.checkState(cur.hasBeen(Applied) || !cur.hasBeen(PreCommitted) || redundantStatus == PRE_BOOTSTRAP_OR_STALE);
if (prev == null)
return;
curSafe.removeListener(prev.asListener());
// we've been applied, invalidated, or are no longer relevant
- if (cur.hasBeen(Applied)) removeInvalidatedAppliedOrTruncatedDependency(prevSafe, cur.txnId());
- else removeRedundantDependencies(safeStore, prevSafe, cur.txnId(), redundantStatus == PRE_BOOTSTRAP);
+ removeRedundantDependencies(safeStore, prevSafe, cur.txnId());
--depth;
prevSafe = get(safeStore, depth - 1);
@@ -1184,15 +1218,7 @@ public class Commands
}
}
- static Command removeInvalidatedAppliedOrTruncatedDependency(SafeCommand safeCommand, TxnId redundant)
- {
- Command.Committed current = safeCommand.current().asCommitted();
- WaitingOn.Update update = new WaitingOn.Update(current.waitingOn);
- update.removeInvalidatedAppliedOrTruncated(redundant);
- return safeCommand.updateWaitingOn(update);
- }
-
- static Command removeRedundantDependencies(SafeCommandStore safeStore, SafeCommand safeCommand, TxnId redundant, boolean isPreBootstrap)
+ static Command removeRedundantDependencies(SafeCommandStore safeStore, SafeCommand safeCommand, TxnId redundant)
{
CommandStore commandStore = safeStore.commandStore();
Command.Committed current = safeCommand.current().asCommitted();
@@ -1201,8 +1227,8 @@ public class Commands
if (minWaitingOnTxnId != null && commandStore.hasLocallyRedundantDependencies(update.minWaitingOnTxnId(), current.executeAt(), current.route().participants()))
safeStore.commandStore().removeRedundantDependencies(current.route().participants(), update);
- if (isPreBootstrap) update.removeWaitingOn(redundant); // above operation not guaranteed to remove all redundant dependencies; transaction may
- else update.removeInvalidatedAppliedOrTruncated(redundant);
+ // if we are a range transaction, being redundant for this transaction does not imply we are redundant for all transactions
+ update.removeWaitingOn(redundant);
return safeCommand.updateWaitingOn(update);
}
@@ -1249,7 +1275,20 @@ public class Commands
return progressKey.equals(route.homeKey()) ? Home : Local;
}
- enum EnsureAction { Ignore, Check, Add, TrySet, Set }
+ enum EnsureAction
+ {
+ /** Don't check */
+ Ignore,
+ /** Add, but return false if insufficient for any reason */
+ TryAdd,
+ /** Supplement existing information, asserting that the existing and additional information are independently sufficient,
+ * returning false only if the existing information is absent AND the new information is insufficient. */
+ Add,
+ /** Set, but only return false if insufficient */
+ TrySet,
+ /** Overwrite existing information if sufficient; fail otherwise */
+ Set
+ }
@SuppressWarnings({"unchecked", "rawtypes"})
private static CommonAttributes set(SafeCommandStore safeStore, Command command, CommonAttributes attrs,
@@ -1320,10 +1359,21 @@ public class Commands
* Validate we have sufficient information for the route, partialTxn and partialDeps fields, and if so update them;
* otherwise return false (or throw an exception if an illegal state is encountered)
*/
- private static boolean validate(Status status, CommonAttributes attrs, Ranges existingRanges, Ranges additionalRanges, ProgressShard shard,
+ private static boolean validate(Status status, CommonAttributes attrs,
+ Ranges existingRanges, Ranges additionalRanges, ProgressShard shard,
Route<?> route, EnsureAction ensureRoute,
@Nullable PartialTxn partialTxn, EnsureAction ensurePartialTxn,
@Nullable PartialDeps partialDeps, EnsureAction ensurePartialDeps)
+ {
+ return validate(status, attrs, existingRanges, additionalRanges, shard, route, ensureRoute, partialTxn, ensurePartialTxn, partialDeps, ensurePartialDeps, null);
+ }
+
+ private static boolean validate(Status status, CommonAttributes attrs,
+ Ranges existingRanges, Ranges additionalRanges, ProgressShard shard,
+ Route<?> route, EnsureAction ensureRoute,
+ @Nullable PartialTxn partialTxn, EnsureAction ensurePartialTxn,
+ @Nullable PartialDeps partialDeps, EnsureAction ensurePartialDeps,
+ @Nullable SafeCommandStore permitStaleMissing)
{
if (shard == Unsure)
return false;
@@ -1332,12 +1382,12 @@ public class Commands
switch (ensureRoute)
{
default: throw new AssertionError("Unexpected action: " + ensureRoute);
- case Check:
+ case TryAdd:
+ case Add:
if (!isFullRoute(attrs.route()) && !isFullRoute(route))
return false;
case Ignore:
break;
- case Add:
case Set:
if (!isFullRoute(route))
throw new IllegalArgumentException("Incomplete route (" + route + ") sent to home shard");
@@ -1353,90 +1403,80 @@ public class Commands
Invariants.checkState(status != Accepted && status != AcceptedInvalidate);
// validate new partial txn
- if (!validate(ensurePartialTxn, existingRanges, additionalRanges, covers(attrs.partialTxn()), covers(partialTxn), "txn", partialTxn))
+ if (!validate(ensurePartialTxn, existingRanges, additionalRanges, covers(attrs.partialTxn()), covers(partialTxn), permitStaleMissing, "txn", partialTxn))
return false;
- Invariants.checkState(partialTxn == null || attrs.txnId().rw() == null || attrs.txnId().rw().equals(partialTxn.kind()), "Transaction has different kind to its TxnId");
- Invariants.checkState(!shard.isHome() || ensurePartialTxn == Ignore || hasQuery(attrs.partialTxn()) || hasQuery(partialTxn), "Home transaction should include query");
+ Invariants.checkState(partialTxn == null || attrs.txnId().rw().equals(partialTxn.kind()), "Transaction has different kind to its TxnId");
+ Invariants.checkState(partialTxn == null || !shard.isHome() || ensurePartialTxn == Ignore || hasQuery(attrs.partialTxn()) || hasQuery(partialTxn), "Home transaction should include query");
- return validate(ensurePartialDeps, existingRanges, additionalRanges, covers(attrs.partialDeps()), covers(partialDeps), "deps", partialDeps);
+ return validate(ensurePartialDeps, existingRanges, additionalRanges, covers(attrs.partialDeps()), covers(partialDeps), permitStaleMissing, "deps", partialDeps);
}
// FIXME (immutable-state): has this been removed?
- private static boolean validate(EnsureAction action, Ranges existingRanges, Ranges additionalRanges,
- Ranges existing, Ranges adding, String kind, Object obj)
+ private static boolean validate(EnsureAction action, Ranges existingRanges, Ranges requiredRanges,
+ Ranges existing, Ranges adding, @Nullable SafeCommandStore permitStaleMissing,
+ String kind, Object obj)
{
switch (action)
{
default: throw new IllegalStateException("Unexpected action: " + action);
case Ignore:
- break;
+ return true;
case TrySet:
- if (adding != null)
+ case Set:
+ if (containsAll(adding, requiredRanges, permitStaleMissing))
+ return true;
+
+ if (action == Set)
+ illegalState("Incomplete " + kind + " (" + obj + ") provided; does not cover " + requiredRanges.subtract(adding));
+
+ return false;
+
+ case TryAdd:
+ case Add:
+ if (existing == null)
{
- if (!adding.containsAll(existingRanges))
- return false;
+ if (adding == null)
+ return false; // we don't want to permit a null value for txn/deps, even if we are stale for all participating ranges, as it breaks assumptions elsewhere
- if (additionalRanges != existingRanges && !adding.containsAll(additionalRanges))
+ if (!adding.containsAll(existingRanges))
return false;
- break;
+ return validate(action == TryAdd ? TrySet : Set, existingRanges, requiredRanges, existing, adding, permitStaleMissing, kind, obj);
}
- case Set:
- // failing any of these tests is always an illegal state
- Invariants.checkState(adding != null);
- if (!adding.containsAll(existingRanges))
- throw new IllegalArgumentException("Incomplete " + kind + " (" + obj + ") provided; does not cover " + existingRanges);
- if (additionalRanges != existingRanges && !adding.containsAll(additionalRanges))
- throw new IllegalArgumentException("Incomplete " + kind + " (" + obj + ") provided; does not cover " + additionalRanges);
- break;
+ Invariants.checkState(existing.containsAll(existingRanges), "Existing ranges insufficient");
+ if (requiredRanges == existingRanges)
+ return true;
- case Check:
- case Add:
if (adding == null)
- {
- if (existing == null)
- return false;
+ return permitStaleMissing != null && containsAll(Ranges.EMPTY, requiredRanges.subtract(existing), permitStaleMissing);
- Invariants.checkState(existing.containsAll(existingRanges));
- if (existingRanges != additionalRanges && !existing.containsAll(additionalRanges))
- {
- if (action == Check)
- return false;
+ requiredRanges = requiredRanges.subtract(existing);
+ if (containsAll(adding, requiredRanges, permitStaleMissing))
+ return true;
- throw new IllegalArgumentException("Missing additional " + kind + "; existing does not cover " + additionalRanges.subtract(existingRanges));
- }
- }
- else if (existing != null)
- {
- Ranges covering = adding.with(existing);
- Invariants.checkState(covering.containsAll(existingRanges));
- if (existingRanges != additionalRanges && !covering.containsAll(additionalRanges))
- {
- if (action == Check)
- return false;
+ if (action == Add)
+ illegalState("Incomplete " + kind + " (" + obj + ") provided; does not cover " + requiredRanges.subtract(adding));
- throw new IllegalArgumentException("Incomplete additional " + kind + " (" + obj + ") provided; does not cover " + additionalRanges.subtract(existingRanges));
- }
- }
- else
- {
- if (!adding.containsAll(existingRanges))
- return false;
+ return false;
+ }
+ }
- if (existingRanges != additionalRanges && !adding.containsAll(additionalRanges))
- {
- if (action == Check)
- return false;
+ private static boolean containsAll(Ranges adding, Ranges requiredRanges, @Nullable SafeCommandStore permitStaleMissing)
+ {
+ if (adding.containsAll(requiredRanges))
+ return true;
- throw new IllegalArgumentException("Incomplete additional " + kind + " (" + obj + ") provided; does not cover " + additionalRanges.subtract(existingRanges));
- }
- }
- break;
+ if (permitStaleMissing != null)
+ {
+ Ranges staleRanges = permitStaleMissing.commandStore().redundantBefore().staleRanges();
+ requiredRanges = requiredRanges.subtract(staleRanges);
+ if (adding.containsAll(requiredRanges))
+ return true;
}
- return true;
+ return false;
}
}
diff --git a/accord-core/src/main/java/accord/local/DurableBefore.java b/accord-core/src/main/java/accord/local/DurableBefore.java
index abe8580e..b8c13d98 100644
--- a/accord-core/src/main/java/accord/local/DurableBefore.java
+++ b/accord-core/src/main/java/accord/local/DurableBefore.java
@@ -27,15 +27,15 @@ import accord.local.Status.Durability;
import accord.primitives.Participants;
import accord.primitives.Range;
import accord.primitives.Ranges;
-import accord.primitives.Routables;
import accord.primitives.TxnId;
+import accord.primitives.Unseekables;
import accord.utils.Invariants;
import accord.utils.ReducingIntervalMap;
import accord.utils.ReducingRangeMap;
-import static accord.local.Status.Durability.Majority;
+import static accord.local.Status.Durability.MajorityOrInvalidated;
import static accord.local.Status.Durability.NotDurable;
-import static accord.local.Status.Durability.Universal;
+import static accord.local.Status.Durability.UniversalOrInvalidated;
public class DurableBefore extends ReducingRangeMap<DurableBefore.Entry>
{
@@ -84,7 +84,7 @@ public class DurableBefore extends ReducingRangeMap<DurableBefore.Entry>
public Durability get(TxnId txnId)
{
if (txnId.compareTo(majorityBefore) < 0)
- return txnId.compareTo(universalBefore) < 0 ? Universal : Majority;
+ return txnId.compareTo(universalBefore) < 0 ? UniversalOrInvalidated : MajorityOrInvalidated;
return NotDurable;
}
@@ -175,14 +175,14 @@ public class DurableBefore extends ReducingRangeMap<DurableBefore.Entry>
return ReducingIntervalMap.merge(a, b, DurableBefore.Entry::max, Builder::new);
}
- public Durability min(TxnId txnId, Routables<?> routables)
+ public Durability min(TxnId txnId, Unseekables<?> unseekables)
{
- return notDurableIfNull(foldl(routables, Entry::mergeMin, null, txnId, test -> test == NotDurable));
+ return notDurableIfNull(foldl(unseekables, Entry::mergeMin, null, txnId, test -> test == NotDurable));
}
- public Durability max(TxnId txnId, Routables<?> routables)
+ public Durability max(TxnId txnId, Unseekables<?> unseekables)
{
- return notDurableIfNull(foldl(routables, Entry::mergeMax, null, txnId, test -> test == Universal));
+ return notDurableIfNull(foldl(unseekables, Entry::mergeMax, null, txnId, test -> test == UniversalOrInvalidated));
}
public Durability get(TxnId txnId, RoutingKey participant)
@@ -191,14 +191,14 @@ public class DurableBefore extends ReducingRangeMap<DurableBefore.Entry>
return entry == null ? NotDurable : entry.get(txnId);
}
- public boolean isUniversal(TxnId txnId, Routables<?> participants)
+ public boolean isUniversal(TxnId txnId, Unseekables<?> participants)
{
- return min(txnId, participants) == Universal;
+ return min(txnId, participants) == UniversalOrInvalidated;
}
public boolean isUniversal(TxnId txnId, RoutingKey participant)
{
- return get(txnId, participant) == Universal;
+ return get(txnId, participant) == UniversalOrInvalidated;
}
public boolean isSomeShardDurable(TxnId txnId, Participants<?> participants, Durability durability)
@@ -209,9 +209,9 @@ public class DurableBefore extends ReducingRangeMap<DurableBefore.Entry>
public Durability min(TxnId txnId)
{
if (min.universalBefore.compareTo(txnId) > 0)
- return Universal;
+ return UniversalOrInvalidated;
if (min.majorityBefore.compareTo(txnId) > 0)
- return Majority;
+ return MajorityOrInvalidated;
return NotDurable;
}
diff --git a/accord-core/src/main/java/accord/local/Node.java b/accord-core/src/main/java/accord/local/Node.java
index b7115b1b..2c4b8e68 100644
--- a/accord-core/src/main/java/accord/local/Node.java
+++ b/accord-core/src/main/java/accord/local/Node.java
@@ -85,6 +85,7 @@ import accord.topology.TopologyManager;
import accord.utils.MapReduceConsume;
import accord.utils.RandomSource;
import accord.utils.async.AsyncChain;
+import accord.utils.async.AsyncExecutor;
import accord.utils.async.AsyncResult;
import accord.utils.async.AsyncResults;
import net.nicoulaj.compilecommand.annotations.Inline;
@@ -494,7 +495,7 @@ public class Node implements ConfigurationService.Listener, NodeTimeService
messageSink.send(to, send, executor, callback);
}
- private void checkStore(AgentExecutor executor)
+ private void checkStore(AsyncExecutor executor)
{
CommandStore current = CommandStore.maybeCurrent();
if (current != null && current != executor)
diff --git a/accord-core/src/main/java/accord/local/RedundantBefore.java b/accord-core/src/main/java/accord/local/RedundantBefore.java
index 4359f1c2..b8410f18 100644
--- a/accord-core/src/main/java/accord/local/RedundantBefore.java
+++ b/accord-core/src/main/java/accord/local/RedundantBefore.java
@@ -18,24 +18,32 @@
package accord.local;
+import java.util.Objects;
+import java.util.function.BiFunction;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import accord.api.RoutingKey;
+import accord.primitives.Deps;
import accord.primitives.EpochSupplier;
import accord.primitives.Participants;
import accord.primitives.Range;
import accord.primitives.Ranges;
+import accord.primitives.Seekables;
import accord.primitives.Timestamp;
import accord.primitives.TxnId;
import accord.utils.Invariants;
import accord.utils.ReducingIntervalMap;
import accord.utils.ReducingRangeMap;
+import static accord.local.RedundantBefore.PreBootstrapOrStale.FULLY;
+import static accord.local.RedundantBefore.PreBootstrapOrStale.POST_BOOTSTRAP;
+import static accord.local.RedundantBefore.PreBootstrapOrStale.PARTIALLY;
import static accord.local.RedundantStatus.LIVE;
-import static accord.local.RedundantStatus.NOT_OWNED;
-import static accord.local.RedundantStatus.PRE_BOOTSTRAP;
import static accord.local.RedundantStatus.LOCALLY_REDUNDANT;
+import static accord.local.RedundantStatus.NOT_OWNED;
+import static accord.local.RedundantStatus.PRE_BOOTSTRAP_OR_STALE;
+import static accord.local.RedundantStatus.SHARD_REDUNDANT;
public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
{
@@ -47,19 +55,62 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
}
}
+ public enum PreBootstrapOrStale { NOT_OWNED, FULLY, PARTIALLY, POST_BOOTSTRAP }
+
public static class Entry
{
+ // TODO (desired): we don't need to maintain this now, and can simplify our builder, by migrating to ReducingRangeMap.foldWithBounds
public final Range range;
public final long startEpoch, endEpoch;
- public final @Nonnull TxnId redundantBefore, bootstrappedAt;
- public Entry(Range range, long startEpoch, long endEpoch, @Nonnull TxnId redundantBefore, @Nonnull TxnId bootstrappedAt)
+ /**
+ * Represents the maximum TxnId we know to have fully executed until locally for the range in question.
+ * Unless we are stale or pre-bootstrap, in which case no such guarantees can be made.
+ */
+ public final @Nonnull TxnId locallyAppliedOrInvalidatedBefore;
+
+ /**
+ * Represents the maximum TxnId we know to have fully executed until across all healthy replicas for the range in question.
+ * Unless we are stale or pre-bootstrap, in which case no such guarantees can be made.
+ */
+ public final @Nonnull TxnId shardAppliedOrInvalidatedBefore;
+
+ /**
+ * bootstrappedAt defines the txnId bounds we expect to maintain data for locally.
+ *
+ * We can bootstrap ranges at different times, and have a transaction that participates in both ranges -
+ * in this case one of the portions of the transaction may be totally unordered with respect to other transactions
+ * in that range because both occur prior to the bootstrappedAt point, so their dependencies are entirely erased.
+ * We can also re-bootstrap the same range because bootstrap failed, and leave dangling transactions to execute
+ * which then execute in an unordered fashion.
+ *
+ * See also {@link CommandStore#safeToRead}.
+ */
+ public final @Nonnull TxnId bootstrappedAt;
+
+ /**
+ * The maximum of each of locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore and bootstrappedAt.
+ */
+ public final @Nonnull TxnId minConflict;
+
+ /**
+ * staleUntilAtLeast provides a minimum TxnId until which we know we will be unable to completely execute
+ * transactions locally for the impacted range.
+ *
+ * See also {@link CommandStore#safeToRead}.
+ */
+ public final @Nullable Timestamp staleUntilAtLeast;
+
+ public Entry(Range range, long startEpoch, long endEpoch, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId bootstrappedAt, @Nullable Timestamp staleUntilAtLeast)
{
this.range = range;
this.startEpoch = startEpoch;
this.endEpoch = endEpoch;
- this.redundantBefore = redundantBefore;
+ this.locallyAppliedOrInvalidatedBefore = locallyAppliedOrInvalidatedBefore;
+ this.shardAppliedOrInvalidatedBefore = shardAppliedOrInvalidatedBefore;
this.bootstrappedAt = bootstrappedAt;
+ this.minConflict = Timestamp.max(Timestamp.max(locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore), bootstrappedAt);
+ this.staleUntilAtLeast = staleUntilAtLeast;
}
public static Entry reduce(Entry a, Entry b)
@@ -69,28 +120,42 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
private static Entry merge(Range range, Entry a, Entry b)
{
+ if (a.startEpoch > b.endEpoch)
+ return a;
+
+ if (b.startEpoch > a.endEpoch)
+ return b;
+
long startEpoch = Long.max(a.startEpoch, b.startEpoch);
long endEpoch = Long.min(a.endEpoch, b.endEpoch);
- int cr = a.redundantBefore.compareTo(b.redundantBefore);
+ int cl = a.locallyAppliedOrInvalidatedBefore.compareTo(b.locallyAppliedOrInvalidatedBefore);
+ int cs = a.shardAppliedOrInvalidatedBefore.compareTo(b.shardAppliedOrInvalidatedBefore);
int cb = a.bootstrappedAt.compareTo(b.bootstrappedAt);
+ int csu = compareStaleUntilAtLeast(a.staleUntilAtLeast, b.staleUntilAtLeast);
- if (range.equals(a.range) && startEpoch == a.startEpoch && endEpoch == a.endEpoch && cr >= 0 && cb >= 0)
+ if (range.equals(a.range) && startEpoch == a.startEpoch && endEpoch == a.endEpoch && cl >= 0 && cb >= 0 && cs >= 0 && csu >= 0)
return a;
- if (range.equals(b.range) && startEpoch == b.startEpoch && endEpoch == b.endEpoch && cr <= 0 && cb <= 0)
+ if (range.equals(b.range) && startEpoch == b.startEpoch && endEpoch == b.endEpoch && cl <= 0 && cb <= 0 && cs <= 0 && csu <= 0)
return b;
- TxnId redundantBefore = cr >= 0 ? a.redundantBefore : b.redundantBefore;
+ TxnId locallyAppliedOrInvalidatedBefore = cl >= 0 ? a.locallyAppliedOrInvalidatedBefore : b.locallyAppliedOrInvalidatedBefore;
+ TxnId shardAppliedOrInvalidatedBefore = cs >= 0 ? a.shardAppliedOrInvalidatedBefore : b.shardAppliedOrInvalidatedBefore;
TxnId bootstrappedAt = cb >= 0 ? a.bootstrappedAt : b.bootstrappedAt;
+ Timestamp staleUntilAtLeast = csu >= 0 ? a.staleUntilAtLeast : b.staleUntilAtLeast;
// if our redundantBefore predates bootstrappedAt, we should clear it to avoid erroneously treating
// transactions prior as locally redundant when they may simply have not applied yet, since we may
// permit the sync point that defines redundancy to apply locally without waiting for these earlier
// transactions, since we now consider them to be bootstrapping
// TODO (desired): revisit later as semantics here evolve
- if (bootstrappedAt.compareTo(redundantBefore) >= 0)
- redundantBefore = TxnId.NONE;
-
- return new Entry(range, startEpoch, endEpoch, redundantBefore, bootstrappedAt);
+ if (bootstrappedAt.compareTo(locallyAppliedOrInvalidatedBefore) >= 0)
+ locallyAppliedOrInvalidatedBefore = TxnId.NONE;
+ if (bootstrappedAt.compareTo(shardAppliedOrInvalidatedBefore) >= 0)
+ shardAppliedOrInvalidatedBefore = TxnId.NONE;
+ if (staleUntilAtLeast != null && bootstrappedAt.compareTo(staleUntilAtLeast) >= 0)
+ staleUntilAtLeast = null;
+
+ return new Entry(range, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, bootstrappedAt, staleUntilAtLeast);
}
static @Nonnull RedundantStatus getAndMerge(Entry entry, @Nonnull RedundantStatus prev, TxnId txnId, EpochSupplier executeAt)
@@ -107,36 +172,99 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
return entry.get(txnId);
}
+ static PreBootstrapOrStale getAndMerge(Entry entry, @Nonnull PreBootstrapOrStale prev, TxnId txnId, EpochSupplier executeAt)
+ {
+ if (prev == PARTIALLY || entry == null || entry.outOfBounds(txnId, executeAt))
+ return prev;
+
+ boolean isPreBootstrapOrStale = entry.staleUntilAtLeast != null || entry.bootstrappedAt.compareTo(txnId) > 0;
+ return isPreBootstrapOrStale ? prev == POST_BOOTSTRAP ? PARTIALLY : FULLY
+ : prev == FULLY ? PARTIALLY : POST_BOOTSTRAP;
+ }
+
+ static Timestamp getAndMergeMinConflict(Entry entry, @Nonnull Timestamp prev)
+ {
+ if (entry == null)
+ return prev;
+ return Timestamp.max(prev, entry.minConflict);
+ }
+
+ static <T extends Deps> Deps.AbstractBuilder<T> collectDep(Entry entry, @Nonnull Deps.AbstractBuilder<T> prev, @Nonnull EpochSupplier minEpoch, @Nonnull EpochSupplier executeAt)
+ {
+ if (entry == null || entry.outOfBounds(minEpoch, executeAt))
+ return prev;
+
+ if (entry.shardAppliedOrInvalidatedBefore.compareTo(Timestamp.NONE) > 0)
+ prev.add(entry.range, entry.shardAppliedOrInvalidatedBefore);
+
+ return prev;
+ }
+
+ static Ranges validateSafeToRead(Entry entry, @Nonnull Ranges safeToRead, Timestamp bootstrapAt, Object ignore)
+ {
+ if (entry == null)
+ return safeToRead;
+
+ if (bootstrapAt.compareTo(entry.bootstrappedAt) < 0 || (entry.staleUntilAtLeast != null && bootstrapAt.compareTo(entry.staleUntilAtLeast) < 0))
+ return safeToRead.subtract(Ranges.of(entry.range));
+
+ return safeToRead;
+ }
+
+ static Ranges expectToExecute(Entry entry, @Nonnull Ranges executeRanges, TxnId txnId, @Nullable Timestamp executeAt)
+ {
+ if (entry == null || (executeAt == null ? entry.outOfBounds(txnId) : entry.outOfBounds(txnId, executeAt)))
+ return executeRanges;
+
+ if (txnId.compareTo(entry.bootstrappedAt) < 0 || entry.staleUntilAtLeast != null)
+ return executeRanges.subtract(Ranges.of(entry.range));
+
+ return executeRanges;
+ }
+
RedundantStatus get(TxnId txnId)
{
- if (redundantBefore.compareTo(txnId) > 0)
+ // we have to first check bootstrappedAt, since we are not locally redundant for the covered range
+ // if the txnId is partially pre-bootstrap (since we may not have applied it for this range)
+ if (staleUntilAtLeast != null || bootstrappedAt.compareTo(txnId) > 0)
+ return PRE_BOOTSTRAP_OR_STALE;
+ if (locallyAppliedOrInvalidatedBefore.compareTo(txnId) > 0)
+ {
+ if (shardAppliedOrInvalidatedBefore.compareTo(txnId) > 0)
+ return SHARD_REDUNDANT;
return LOCALLY_REDUNDANT;
- if (bootstrappedAt.compareTo(txnId) > 0)
- return PRE_BOOTSTRAP;
+ }
return LIVE;
}
- public final boolean isLocalRedundancyViaBootstrap()
+ private static int compareStaleUntilAtLeast(@Nullable Timestamp a, @Nullable Timestamp b)
{
- return redundantBefore.compareTo(bootstrappedAt) < 0;
+ boolean aIsNull = a == null, bIsNull = b == null;
+ if (aIsNull != bIsNull) return aIsNull ? -1 : 1;
+ return aIsNull ? 0 : a.compareTo(b);
}
- public final TxnId locallyRedundantBefore()
+ public final TxnId shardRedundantBefore()
{
- return TxnId.min(bootstrappedAt, redundantBefore);
+ return shardAppliedOrInvalidatedBefore;
}
// TODO (required, consider): this admits the range of epochs that cross the two timestamps, which matches our
// behaviour elsewhere but we probably want to only interact with the two point epochs in which we participate,
// but note hasRedundantDependencies which really does want to scan a range
- private boolean outOfBounds(Timestamp lb, EpochSupplier ub)
+ private boolean outOfBounds(EpochSupplier lb, EpochSupplier ub)
{
return ub.epoch() < startEpoch || lb.epoch() >= endEpoch;
}
+ private boolean outOfBounds(Timestamp lb)
+ {
+ return lb.epoch() >= endEpoch;
+ }
+
Entry withRange(Range range)
{
- return new Entry(range, startEpoch, endEpoch, redundantBefore, bootstrappedAt);
+ return new Entry(range, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, bootstrappedAt, staleUntilAtLeast);
}
public boolean equals(Object that)
@@ -153,7 +281,10 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
{
return this.startEpoch == that.startEpoch
&& this.endEpoch == that.endEpoch
- && this.redundantBefore.equals(that.redundantBefore);
+ && this.locallyAppliedOrInvalidatedBefore.equals(that.locallyAppliedOrInvalidatedBefore)
+ && this.shardAppliedOrInvalidatedBefore.equals(that.shardAppliedOrInvalidatedBefore)
+ && this.bootstrappedAt.equals(that.bootstrappedAt)
+ && Objects.equals(this.staleUntilAtLeast, that.staleUntilAtLeast);
}
@Override
@@ -162,27 +293,68 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
return "("
+ (startEpoch == Long.MIN_VALUE ? "-\u221E" : Long.toString(startEpoch)) + ","
+ (endEpoch == Long.MAX_VALUE ? "\u221E" : Long.toString(endEpoch)) + ","
- + (redundantBefore.compareTo(bootstrappedAt) >= 0 ? redundantBefore + ")" : bootstrappedAt + "*)");
+ + (locallyAppliedOrInvalidatedBefore.compareTo(bootstrappedAt) >= 0 ? locallyAppliedOrInvalidatedBefore + ")" : bootstrappedAt + "*)");
}
}
public static RedundantBefore EMPTY = new RedundantBefore();
+ private final Ranges staleRanges;
+
private RedundantBefore()
{
+ staleRanges = Ranges.EMPTY;
}
RedundantBefore(boolean inclusiveEnds, RoutingKey[] starts, Entry[] values)
{
super(inclusiveEnds, starts, values);
+ staleRanges = extractStaleRanges(values);
}
- public static RedundantBefore create(Ranges ranges, long startEpoch, long endEpoch, @Nonnull TxnId redundantBefore, @Nonnull TxnId bootstrappedAt)
+ private static Ranges extractStaleRanges(Entry[] values)
+ {
+ int countStaleRanges = 0;
+ for (Entry entry : values)
+ {
+ if (entry != null && entry.staleUntilAtLeast != null)
+ ++countStaleRanges;
+ }
+
+ if (countStaleRanges == 0)
+ return Ranges.EMPTY;
+
+ Range[] staleRanges = new Range[countStaleRanges];
+ countStaleRanges = 0;
+ for (Entry entry : values)
+ {
+ if (entry != null && entry.staleUntilAtLeast != null)
+ staleRanges[countStaleRanges++] = entry.range;
+ }
+ return Ranges.ofSortedAndDeoverlapped(staleRanges).mergeTouching();
+ }
+
+ public static RedundantBefore create(Ranges ranges, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId bootstrappedAt)
+ {
+ return create(ranges, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, bootstrappedAt, null);
+ }
+
+ public static RedundantBefore create(Ranges ranges, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId bootstrappedAt, @Nullable Timestamp staleUntilAtLeast)
+ {
+ return create(ranges, Long.MIN_VALUE, Long.MAX_VALUE, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, bootstrappedAt, staleUntilAtLeast);
+ }
+
+ public static RedundantBefore create(Ranges ranges, long startEpoch, long endEpoch, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId bootstrappedAt)
+ {
+ return create(ranges, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, bootstrappedAt, null);
+ }
+
+ public static RedundantBefore create(Ranges ranges, long startEpoch, long endEpoch, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId bootstrappedAt, @Nullable Timestamp staleUntilAtLeast)
{
if (ranges.isEmpty())
return new RedundantBefore();
- Entry entry = new Entry(null, startEpoch, endEpoch, redundantBefore, bootstrappedAt);
+ Entry entry = new Entry(null, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, bootstrappedAt, staleUntilAtLeast);
Builder builder = new Builder(ranges.get(0).endInclusive(), ranges.size() * 2);
for (int i = 0 ; i < ranges.size() ; ++i)
{
@@ -211,6 +383,57 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
return foldl(participants, Entry::getAndMerge, NOT_OWNED, txnId, executeAt, ignore -> false);
}
+ /**
+ * RedundantStatus.REDUNDANT overrides PRE_BOOTSTRAP; to avoid complicating that state machine,
+ * for cases where we care independently about the overall pre-bootstrap state we have a separate mechanism
+ */
+ public PreBootstrapOrStale preBootstrapOrStale(TxnId txnId, @Nullable EpochSupplier executeAt, Participants<?> participants)
+ {
+ if (executeAt == null) executeAt = txnId;
+ return foldl(participants, Entry::getAndMerge, PreBootstrapOrStale.NOT_OWNED, txnId, executeAt, r -> r == PARTIALLY);
+ }
+
+ public Timestamp minConflict(Seekables<?, ?> participants)
+ {
+ return foldl(participants, Entry::getAndMergeMinConflict, Timestamp.NONE, ignore -> false);
+ }
+
+ public <T extends Deps> Deps.AbstractBuilder<T> collectDeps(Seekables<?, ?> participants, Deps.AbstractBuilder<T> builder, EpochSupplier minEpoch, EpochSupplier executeAt)
+ {
+ return foldl(participants, Entry::collectDep, builder, minEpoch, executeAt, ignore -> false);
+ }
+
+ public Ranges validateSafeToRead(Timestamp forBootstrapAt, Ranges ranges)
+ {
+ return foldl(ranges, Entry::validateSafeToRead, ranges, forBootstrapAt, null, r -> false);
+ }
+
+ /**
+ * Subtract any ranges we consider stale or pre-bootstrap
+ */
+ public Ranges expectToExecute(TxnId txnId, @Nonnull Timestamp executeAt, Ranges ranges)
+ {
+ Invariants.checkArgument(executeAt != null, "executeAt must not be null");
+ return foldl(ranges, Entry::expectToExecute, ranges, txnId, executeAt, r -> false);
+ }
+
+ /**
+ * Subtract any ranges we consider stale or pre-bootstrap at any point
+ */
+ public Ranges everExpectToExecute(TxnId txnId, Ranges ranges)
+ {
+ return foldl(ranges, Entry::expectToExecute, ranges, txnId, null, r -> false);
+ }
+
+ /**
+ * RedundantStatus.REDUNDANT overrides PRE_BOOTSTRAP; to avoid complicating that state machine,
+ * for cases where we care independently about the overall pre-bootstrap state we have a separate mechanism
+ */
+ public Ranges staleRanges()
+ {
+ return staleRanges;
+ }
+
static class Builder extends ReducingIntervalMap.Builder<RoutingKey, Entry, RedundantBefore>
{
protected Builder(boolean inclusiveEnds, int capacity)
@@ -224,6 +447,23 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
return a.equalsIgnoreRange(b);
}
+ // we maintain the range that an Entry applies to for ease of some other functionality - this logic is applied here in append and above by overriding equals.
+ // entries may be null for ranges we do not have any knowledge for; the final "entry" we append is expected to be null by the ReducingIntervalMap.Builder
+ // TODO (desired): consider migrating to foldlWithBounds
+ @Override
+ public void append(RoutingKey start, @Nullable Entry value, BiFunction<Entry, Entry, Entry> reduce)
+ {
+ if (value != null && value.range.start().compareTo(start) < 0)
+ value = value.withRange(value.range.newRange(start, value.range.end()));
+
+ int tailIdx = values.size() - 1;
+ super.append(start, value, reduce);
+
+ Entry tailValue; // TODO (desired): clean up maintenance of accurate range bounds
+ if (values.size() - 2 == tailIdx && tailIdx >= 0 && (tailValue = values.get(tailIdx)) != null && tailValue.range.end().compareTo(start) > 0)
+ values.set(tailIdx, tailValue.withRange(tailValue.range.newRange(tailValue.range.start(), start)));
+ }
+
@Override
protected Entry mergeEqual(Entry a, Entry b)
{
@@ -231,7 +471,7 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
return new Entry(a.range.newRange(
a.range.start().compareTo(b.range.start()) <= 0 ? a.range.start() : b.range.start(),
a.range.end().compareTo(b.range.end()) >= 0 ? a.range.end() : b.range.end()
- ), a.startEpoch, a.endEpoch, a.redundantBefore, a.bootstrappedAt);
+ ), a.startEpoch, a.endEpoch, a.locallyAppliedOrInvalidatedBefore, a.shardAppliedOrInvalidatedBefore, a.bootstrappedAt, a.staleUntilAtLeast);
}
@Override
diff --git a/accord-core/src/main/java/accord/local/RedundantStatus.java b/accord-core/src/main/java/accord/local/RedundantStatus.java
index 15a5ccc8..63eecd34 100644
--- a/accord-core/src/main/java/accord/local/RedundantStatus.java
+++ b/accord-core/src/main/java/accord/local/RedundantStatus.java
@@ -24,22 +24,52 @@ import accord.utils.Invariants;
public enum RedundantStatus
{
- /** None of the relevant ranges are owned by the command store */
+ /**
+ * None of the relevant ranges are owned by the command store
+ */
NOT_OWNED,
- /** Some of the relevant ranges are owned by the command store and valid for execution */
+ /**
+ * Some of the relevant ranges are owned by the command store and valid for execution
+ */
LIVE,
- /** The relevant owned ranges are part live and part pre-bootstrap */
- PARTIALLY_PRE_BOOTSTRAP,
+ /**
+ * The relevant owned ranges are part live and part pre-bootstrap or stale
+ */
+ PARTIALLY_PRE_BOOTSTRAP_OR_STALE,
- /** The relevant owned ranges are ALL pre-bootstrap, meaning we are fetching the transaction's entire result
- * from another node's snapshot already */
- PRE_BOOTSTRAP,
+ /**
+ * The relevant owned ranges are ALL pre-bootstrap or stale, meaning we are either
+ * 1) fetching the transaction's entire result from another node's snapshot already; or
+ * 2) the range is stale and _must be bootstrapped_
+ */
+ PRE_BOOTSTRAP_OR_STALE,
- /** The relevant owned ranges are fully redundant, meaning they are known applied or invalidated via a sync point
- * that has applied locally */
+ /**
+ * Some of the relevant owned ranges are redundant, meaning any intersecting transaction is known to be either applied
+ * or invalidated via a sync point that has applied locally, but other ranges are known to be pre-bootstrap or stale
+ * so that we cannot treat the transaction as having been fully applied locally.
+ *
+ * Note that this status overrides PRE_BOOTSTRAP_OR_STALE, since it implies the transaction has applied.
+ */
+ REDUNDANT_PRE_BOOTSTRAP_OR_STALE,
+
+ /**
+ * The relevant owned ranges are redundant, meaning any intersecting transaction is known to be either applied or invalidated
+ * via a sync point that has applied locally.
+ *
+ * Note that this status overrides PRE_BOOTSTRAP_OR_STALE, since it implies the transaction has applied.
+ */
LOCALLY_REDUNDANT,
+
+ /**
+ * The relevant owned ranges are redundant, meaning any intersecting transaction is known to be either applied or invalidated
+ * via a sync point that has applied locally and on all healthy shards.
+ *
+ * Note that this status overrides PRE_BOOTSTRAP_OR_STALE, since it implies the transaction has applied.
+ */
+ SHARD_REDUNDANT,
;
private EnumMap<RedundantStatus, RedundantStatus> merge;
@@ -49,33 +79,59 @@ public enum RedundantStatus
NOT_OWNED.merge = new EnumMap<>(RedundantStatus.class);
NOT_OWNED.merge.put(NOT_OWNED, NOT_OWNED);
NOT_OWNED.merge.put(LIVE, LIVE);
- NOT_OWNED.merge.put(PARTIALLY_PRE_BOOTSTRAP, PARTIALLY_PRE_BOOTSTRAP);
- NOT_OWNED.merge.put(PRE_BOOTSTRAP, PRE_BOOTSTRAP);
+ NOT_OWNED.merge.put(PARTIALLY_PRE_BOOTSTRAP_OR_STALE, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
+ NOT_OWNED.merge.put(PRE_BOOTSTRAP_OR_STALE, PRE_BOOTSTRAP_OR_STALE);
+ NOT_OWNED.merge.put(REDUNDANT_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
NOT_OWNED.merge.put(LOCALLY_REDUNDANT, LOCALLY_REDUNDANT);
+ NOT_OWNED.merge.put(SHARD_REDUNDANT, SHARD_REDUNDANT);
LIVE.merge = new EnumMap<>(RedundantStatus.class);
LIVE.merge.put(NOT_OWNED, LIVE);
LIVE.merge.put(LIVE, LIVE);
- LIVE.merge.put(PARTIALLY_PRE_BOOTSTRAP, PARTIALLY_PRE_BOOTSTRAP);
- LIVE.merge.put(PRE_BOOTSTRAP, PARTIALLY_PRE_BOOTSTRAP);
- LIVE.merge.put(LOCALLY_REDUNDANT, LOCALLY_REDUNDANT);
- PARTIALLY_PRE_BOOTSTRAP.merge = new EnumMap<>(RedundantStatus.class);
- PARTIALLY_PRE_BOOTSTRAP.merge.put(NOT_OWNED, PARTIALLY_PRE_BOOTSTRAP);
- PARTIALLY_PRE_BOOTSTRAP.merge.put(LIVE, PARTIALLY_PRE_BOOTSTRAP);
- PARTIALLY_PRE_BOOTSTRAP.merge.put(PARTIALLY_PRE_BOOTSTRAP, PARTIALLY_PRE_BOOTSTRAP);
- PARTIALLY_PRE_BOOTSTRAP.merge.put(PRE_BOOTSTRAP, PARTIALLY_PRE_BOOTSTRAP);
- PARTIALLY_PRE_BOOTSTRAP.merge.put(LOCALLY_REDUNDANT, LOCALLY_REDUNDANT);
- PRE_BOOTSTRAP.merge = new EnumMap<>(RedundantStatus.class);
- PRE_BOOTSTRAP.merge.put(NOT_OWNED, PRE_BOOTSTRAP);
- PRE_BOOTSTRAP.merge.put(LIVE, PARTIALLY_PRE_BOOTSTRAP);
- PRE_BOOTSTRAP.merge.put(PARTIALLY_PRE_BOOTSTRAP, PARTIALLY_PRE_BOOTSTRAP);
- PRE_BOOTSTRAP.merge.put(PRE_BOOTSTRAP, PRE_BOOTSTRAP);
- PRE_BOOTSTRAP.merge.put(LOCALLY_REDUNDANT, LOCALLY_REDUNDANT);
+ LIVE.merge.put(PARTIALLY_PRE_BOOTSTRAP_OR_STALE, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
+ LIVE.merge.put(PRE_BOOTSTRAP_OR_STALE, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
+ LIVE.merge.put(REDUNDANT_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+ LIVE.merge.put(LOCALLY_REDUNDANT, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+ LIVE.merge.put(SHARD_REDUNDANT, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+ PARTIALLY_PRE_BOOTSTRAP_OR_STALE.merge = new EnumMap<>(RedundantStatus.class);
+ PARTIALLY_PRE_BOOTSTRAP_OR_STALE.merge.put(NOT_OWNED, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
+ PARTIALLY_PRE_BOOTSTRAP_OR_STALE.merge.put(LIVE, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
+ PARTIALLY_PRE_BOOTSTRAP_OR_STALE.merge.put(PARTIALLY_PRE_BOOTSTRAP_OR_STALE, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
+ PARTIALLY_PRE_BOOTSTRAP_OR_STALE.merge.put(PRE_BOOTSTRAP_OR_STALE, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
+ PARTIALLY_PRE_BOOTSTRAP_OR_STALE.merge.put(REDUNDANT_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+ PARTIALLY_PRE_BOOTSTRAP_OR_STALE.merge.put(LOCALLY_REDUNDANT, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+ PARTIALLY_PRE_BOOTSTRAP_OR_STALE.merge.put(SHARD_REDUNDANT, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+ PRE_BOOTSTRAP_OR_STALE.merge = new EnumMap<>(RedundantStatus.class);
+ PRE_BOOTSTRAP_OR_STALE.merge.put(NOT_OWNED, PRE_BOOTSTRAP_OR_STALE);
+ PRE_BOOTSTRAP_OR_STALE.merge.put(LIVE, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
+ PRE_BOOTSTRAP_OR_STALE.merge.put(PARTIALLY_PRE_BOOTSTRAP_OR_STALE, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
+ PRE_BOOTSTRAP_OR_STALE.merge.put(PRE_BOOTSTRAP_OR_STALE, PRE_BOOTSTRAP_OR_STALE);
+ PRE_BOOTSTRAP_OR_STALE.merge.put(REDUNDANT_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+ PRE_BOOTSTRAP_OR_STALE.merge.put(LOCALLY_REDUNDANT, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+ PRE_BOOTSTRAP_OR_STALE.merge.put(SHARD_REDUNDANT, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+ REDUNDANT_PRE_BOOTSTRAP_OR_STALE.merge = new EnumMap<>(RedundantStatus.class);
+ REDUNDANT_PRE_BOOTSTRAP_OR_STALE.merge.put(NOT_OWNED, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+ REDUNDANT_PRE_BOOTSTRAP_OR_STALE.merge.put(LIVE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE); // TODO (expected): should this be an invalid combination?
+ REDUNDANT_PRE_BOOTSTRAP_OR_STALE.merge.put(PARTIALLY_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+ REDUNDANT_PRE_BOOTSTRAP_OR_STALE.merge.put(PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+ REDUNDANT_PRE_BOOTSTRAP_OR_STALE.merge.put(REDUNDANT_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+ REDUNDANT_PRE_BOOTSTRAP_OR_STALE.merge.put(LOCALLY_REDUNDANT, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+ REDUNDANT_PRE_BOOTSTRAP_OR_STALE.merge.put(SHARD_REDUNDANT, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
LOCALLY_REDUNDANT.merge = new EnumMap<>(RedundantStatus.class);
LOCALLY_REDUNDANT.merge.put(NOT_OWNED, LOCALLY_REDUNDANT);
- LOCALLY_REDUNDANT.merge.put(LIVE, LOCALLY_REDUNDANT);
- LOCALLY_REDUNDANT.merge.put(PARTIALLY_PRE_BOOTSTRAP, LOCALLY_REDUNDANT);
- LOCALLY_REDUNDANT.merge.put(PRE_BOOTSTRAP, LOCALLY_REDUNDANT);
+ LOCALLY_REDUNDANT.merge.put(LIVE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE); // TODO (expected): should this be an invalid combination?
+ LOCALLY_REDUNDANT.merge.put(PARTIALLY_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+ LOCALLY_REDUNDANT.merge.put(PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+ LOCALLY_REDUNDANT.merge.put(REDUNDANT_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
LOCALLY_REDUNDANT.merge.put(LOCALLY_REDUNDANT, LOCALLY_REDUNDANT);
+ LOCALLY_REDUNDANT.merge.put(SHARD_REDUNDANT, LOCALLY_REDUNDANT);
+ SHARD_REDUNDANT.merge = new EnumMap<>(RedundantStatus.class);
+ SHARD_REDUNDANT.merge.put(NOT_OWNED, SHARD_REDUNDANT);
+ SHARD_REDUNDANT.merge.put(LIVE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE); // TODO (expected): should this be an invalid combination?
+ SHARD_REDUNDANT.merge.put(PARTIALLY_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+ SHARD_REDUNDANT.merge.put(PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+ SHARD_REDUNDANT.merge.put(REDUNDANT_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+ SHARD_REDUNDANT.merge.put(LOCALLY_REDUNDANT, LOCALLY_REDUNDANT);
+ SHARD_REDUNDANT.merge.put(SHARD_REDUNDANT, SHARD_REDUNDANT);
}
public RedundantStatus merge(RedundantStatus that)
diff --git a/accord-core/src/main/java/accord/local/SafeCommand.java b/accord-core/src/main/java/accord/local/SafeCommand.java
index a02e294a..69413246 100644
--- a/accord-core/src/main/java/accord/local/SafeCommand.java
+++ b/accord-core/src/main/java/accord/local/SafeCommand.java
@@ -57,8 +57,6 @@ public abstract class SafeCommand
private <C extends Command> C update(C update)
{
- // TODO (now): reenable this check
-// Invariants.checkState(current() == null || !CommandStore.current().isTruncated(current()));
set(update);
return update;
}
diff --git a/accord-core/src/main/java/accord/local/SafeCommandStore.java b/accord-core/src/main/java/accord/local/SafeCommandStore.java
index eb9574e1..c1eb61d7 100644
--- a/accord-core/src/main/java/accord/local/SafeCommandStore.java
+++ b/accord-core/src/main/java/accord/local/SafeCommandStore.java
@@ -33,16 +33,15 @@ import accord.primitives.Ranges;
import accord.primitives.Seekable;
import accord.primitives.Seekables;
import accord.primitives.Timestamp;
-import accord.primitives.Txn.Kind;
+import accord.primitives.Txn.Kind.Kinds;
import accord.primitives.TxnId;
import accord.primitives.Unseekables;
import javax.annotation.Nullable;
+import com.google.common.base.Predicates;
+
import static accord.local.Commands.Cleanup.NO;
-import static accord.local.RedundantStatus.PRE_BOOTSTRAP;
-import static accord.primitives.Txn.Kind.ExclusiveSyncPoint;
-import static accord.primitives.Txn.Kind.Read;
-import static accord.primitives.Txn.Kind.Write;
+import static accord.local.RedundantBefore.PreBootstrapOrStale.FULLY;
/**
* A CommandStore with exclusive access; a reference to this should not be retained outside of the scope of the method
@@ -52,9 +51,9 @@ import static accord.primitives.Txn.Kind.Write;
*/
public abstract class SafeCommandStore
{
- public interface CommandFunction<I, O>
+ public interface CommandFunction<P1, I, O>
{
- O apply(Seekable keyOrRange, TxnId txnId, Timestamp executeAt, Status status, I in);
+ O apply(P1 p1, Seekable keyOrRange, TxnId txnId, Timestamp executeAt, Status status, I in);
}
public enum TestTimestamp
@@ -65,57 +64,6 @@ public abstract class SafeCommandStore
EXECUTES_AFTER
}
public enum TestDep { WITH, WITHOUT, ANY_DEPS }
- public enum TestKind implements Predicate<Kind>
- {
- Ws, RorWs, WsOrSyncPoint, SyncPoints, Any;
-
- @Override
- public boolean test(Kind kind)
- {
- switch (this)
- {
- default: throw new AssertionError();
- case Any: return true;
- case WsOrSyncPoint: return kind == Write || kind == Kind.SyncPoint || kind == ExclusiveSyncPoint;
- case SyncPoints: return kind == Kind.SyncPoint || kind == ExclusiveSyncPoint;
- case Ws: return kind == Write;
- case RorWs: return kind == Read || kind == Write;
- }
- }
-
- public static TestKind conflicts(Kind kind)
- {
- switch (kind)
- {
- default: throw new AssertionError();
- case Read:
- case NoOp:
- return Ws;
- case Write:
- return RorWs;
- case SyncPoint:
- case ExclusiveSyncPoint:
- return Any;
- }
- }
-
- public static TestKind shouldHaveWitnessed(Kind kind)
- {
- switch (kind)
- {
- default: throw new AssertionError();
- case Read:
- return WsOrSyncPoint;
- case Write:
- return Any;
- case SyncPoint:
- case ExclusiveSyncPoint:
- case NoOp:
- return SyncPoints;
- }
- }
-
- }
/**
* If the transaction exists (with some associated data) in the CommandStore, return it. Otherwise return null.
@@ -212,16 +160,18 @@ public abstract class SafeCommandStore
* Visits keys first and then ranges, both in ascending order.
* Within each key or range visits TxnId in ascending order of queried timestamp.
*/
- public abstract <T> T mapReduceWithTerminate(Seekables<?, ?> keysOrRanges, Ranges slice,
- TestKind testKind, TestTimestamp testTimestamp, Timestamp timestamp,
- TestDep testDep, @Nullable TxnId depId,
- @Nullable Status minStatus, @Nullable Status maxStatus,
- CommandFunction<T, T> map, T accumulate, Predicate<T> terminate);
- public abstract <T> T mapReduce(Seekables<?, ?> keysOrRanges, Ranges slice,
- TestKind testKind, TestTimestamp testTimestamp, Timestamp timestamp,
- TestDep testDep, @Nullable TxnId depId, @Nullable Status minStatus, @Nullable Status maxStatus,
- CommandFunction<T, T> map, T accumulate, T terminalValue);
+ public <P1, T> T mapReduce(Seekables<?, ?> keys, Ranges slice,
+ Kinds testKind, TestTimestamp testTimestamp, Timestamp timestamp,
+ TestDep testDep, @Nullable TxnId depId, @Nullable Status minStatus, @Nullable Status maxStatus,
+ CommandFunction<P1, T, T> map, P1 p1, T initialValue)
+ {
+ return mapReduce(keys, slice, testKind, testTimestamp, timestamp, testDep, depId, minStatus, maxStatus, map, p1, initialValue, Predicates.alwaysFalse());
+ }
+ public abstract <P1, T> T mapReduce(Seekables<?, ?> keys, Ranges slice,
+ Kinds testKind, TestTimestamp testTimestamp, Timestamp timestamp,
+ TestDep testDep, @Nullable TxnId depId, @Nullable Status minStatus, @Nullable Status maxStatus,
+ CommandFunction<P1, T, T> map, P1 p1, T initialValue, Predicate<? super T> terminate);
protected abstract void register(Seekables<?, ?> keysOrRanges, Ranges slice, Command command);
protected abstract void register(Seekable keyOrRange, Ranges slice, Command command);
@@ -247,9 +197,9 @@ public abstract class SafeCommandStore
// if we have to re-bootstrap (due to failed bootstrap or catching up on a range) then we may
// have dangling redundant commands; these can safely be executed locally because we are a timestamp store
- final boolean isFullyPreBootstrap(Command command, Participants<?> forKeys)
+ final boolean isFullyPreBootstrapOrStale(Command command, Participants<?> forKeys)
{
- return commandStore().redundantBefore().status(command.txnId(), command.executeAtOrTxnId(), forKeys) == PRE_BOOTSTRAP;
+ return commandStore().redundantBefore().preBootstrapOrStale(command.txnId(), command.executeAtOrTxnId(), forKeys) == FULLY;
}
public void notifyListeners(SafeCommand safeCommand)
diff --git a/accord-core/src/main/java/accord/local/SaveStatus.java b/accord-core/src/main/java/accord/local/SaveStatus.java
index b547fbb0..1043c321 100644
--- a/accord-core/src/main/java/accord/local/SaveStatus.java
+++ b/accord-core/src/main/java/accord/local/SaveStatus.java
@@ -23,7 +23,9 @@ import accord.primitives.Ballot;
import accord.primitives.Timestamp;
import accord.primitives.TxnId;
+import static accord.local.SaveStatus.LocalExecution.CleaningUp;
import static accord.local.SaveStatus.LocalExecution.NotReady;
+import static accord.local.Status.Definition.DefinitionErased;
import static accord.local.Status.Definition.DefinitionKnown;
import static accord.local.Status.Definition.DefinitionUnknown;
import static accord.local.Status.Known.Apply;
@@ -32,7 +34,9 @@ import static accord.local.Status.Known.ExecuteAtOnly;
import static accord.local.Status.Known.Nothing;
import static accord.local.Status.KnownDeps.*;
import static accord.local.Status.KnownExecuteAt.*;
-import static accord.local.Status.NotDefined;
+import static accord.local.Status.KnownRoute.Covering;
+import static accord.local.Status.KnownRoute.Full;
+import static accord.local.Status.KnownRoute.Maybe;
import static accord.local.Status.Outcome.Unknown;
import static accord.local.Status.Truncated;
@@ -49,29 +53,39 @@ public enum SaveStatus
// TODO (expected): erase Uninitialised in Context once command finishes
// TODO (expected): we can use Uninitialised in several places to simplify/better guarantee correct behaviour with truncation
Uninitialised (Status.NotDefined),
+ // TODO (expected): reify PreAcceptedNotDefined and NotDefinedWithSomeRoute (latter to semantically represent outcome of InformHome)
NotDefined (Status.NotDefined),
PreAccepted (Status.PreAccepted),
+ // note: AcceptedInvalidate and AcceptedInvalidateWithDefinition clear any proposed Deps.
+ // This means voters recovering an earlier transaction will not consider the record when excluding the possibility of a fast-path commit.
+ // This is safe, because any Accept that may override the AcceptedInvalidate will construct new Deps that must now witness the recovering transaction.
AcceptedInvalidate (Status.AcceptedInvalidate),
- AcceptedInvalidateWithDefinition(Status.AcceptedInvalidate, DefinitionKnown, ExecuteAtUnknown, DepsUnknown, Unknown),
+ AcceptedInvalidateWithDefinition(Status.AcceptedInvalidate, Full, DefinitionKnown, ExecuteAtUnknown, DepsUnknown, Unknown),
Accepted (Status.Accepted),
- AcceptedWithDefinition (Status.Accepted, DefinitionKnown, ExecuteAtProposed, DepsProposed, Unknown),
- PreCommitted (Status.PreCommitted),
- PreCommittedWithAcceptedDeps (Status.PreCommitted, DefinitionUnknown, ExecuteAtKnown, DepsProposed, Unknown),
- PreCommittedWithDefinition (Status.PreCommitted, DefinitionKnown, ExecuteAtKnown, DepsUnknown, Unknown),
- PreCommittedWithDefinitionAndAcceptedDeps(Status.PreCommitted, DefinitionKnown, ExecuteAtKnown, DepsProposed, Unknown),
- Committed (Status.Committed),
- ReadyToExecute (Status.ReadyToExecute, LocalExecution.ReadyToExecute),
- PreApplied (Status.PreApplied),
- Applying (Status.PreApplied),
- Applied (Status.Applied, LocalExecution.Applied),
+ AcceptedWithDefinition (Status.Accepted, Full, DefinitionKnown, ExecuteAtProposed, DepsProposed, Unknown),
+ PreCommitted (Status.PreCommitted, LocalExecution.ReadyToExclude),
+ PreCommittedWithAcceptedDeps (Status.PreCommitted, Covering, DefinitionUnknown, ExecuteAtKnown, DepsProposed, Unknown, LocalExecution.ReadyToExclude),
+ PreCommittedWithDefinition (Status.PreCommitted, Full, DefinitionKnown, ExecuteAtKnown, DepsUnknown, Unknown, LocalExecution.ReadyToExclude),
+ PreCommittedWithDefinitionAndAcceptedDeps(Status.PreCommitted, Full, DefinitionKnown, ExecuteAtKnown, DepsProposed, Unknown, LocalExecution.ReadyToExclude),
+ Committed (Status.Committed, LocalExecution.WaitingToExecute),
+ ReadyToExecute (Status.ReadyToExecute, LocalExecution.ReadyToExecute),
+ PreApplied (Status.PreApplied, LocalExecution.WaitingToApply),
+ Applying (Status.PreApplied, LocalExecution.Applying),
+ // similar to Truncated, but doesn't imply we have any global knowledge about application
+ Applied (Status.Applied, LocalExecution.Applied),
// TruncatedApplyWithDeps is a state never adopted within a single replica; it is however a useful state we may enter by combining state from multiple replicas
- TruncatedApplyWithDeps (Status.Truncated, DefinitionUnknown, ExecuteAtKnown, DepsKnown, Outcome.Apply, LocalExecution.CleaningUp),
- TruncatedApplyWithOutcome (Status.Truncated, DefinitionUnknown, ExecuteAtKnown, DepsUnknown, Outcome.Apply, LocalExecution.CleaningUp),
- TruncatedApply (Status.Truncated, DefinitionUnknown, ExecuteAtKnown, DepsUnknown, Outcome.WasApply, LocalExecution.CleaningUp),
- Erased (Status.Truncated, DefinitionUnknown, ExecuteAtUnknown, DepsUnknown, Outcome.Erased, LocalExecution.CleaningUp),
- Invalidated (Status.Invalidated, LocalExecution.CleaningUp),
+ // TODO (desired): if we migrate away from SaveStatus in CheckStatusOk towards Known we can remove this TruncatedApplyWithDeps
+ TruncatedApplyWithDeps (Status.Truncated, Full, DefinitionErased, ExecuteAtKnown, DepsKnown, Outcome.Apply, CleaningUp),
+ TruncatedApplyWithOutcome (Status.Truncated, Full, DefinitionErased, ExecuteAtKnown, DepsErased, Outcome.Apply, CleaningUp),
+ TruncatedApply (Status.Truncated, Full, DefinitionErased, ExecuteAtKnown, DepsErased, Outcome.WasApply, CleaningUp),
+ // Expunged means the command is either entirely pre-bootstrap or stale and we don't have enough information to move it through the normal redundant->truncation path (i.e. it might be truncated, it might be applied)
+ Erased (Status.Truncated, Maybe, DefinitionErased, ExecuteAtErased, DepsErased, Outcome.Erased, CleaningUp),
+ Invalidated (Status.Invalidated, CleaningUp),
;
+ /**
+ * Note that this is a LOCAL concept ONLY, and should not be used to infer anything remotely.
+ */
public enum LocalExecution
{
NotReady(Nothing),
@@ -143,14 +157,14 @@ public enum SaveStatus
this(status, phase, status.minKnown, execution);
}
- SaveStatus(Status status, Definition definition, KnownExecuteAt executeAt, KnownDeps deps, Outcome outcome)
+ SaveStatus(Status status, KnownRoute route, Definition definition, KnownExecuteAt executeAt, KnownDeps deps, Outcome outcome)
{
- this(status, definition, executeAt, deps, outcome, NotReady);
+ this(status, route, definition, executeAt, deps, outcome, NotReady);
}
- SaveStatus(Status status, Definition definition, KnownExecuteAt executeAt, KnownDeps deps, Outcome outcome, LocalExecution execution)
+ SaveStatus(Status status, KnownRoute route, Definition definition, KnownExecuteAt executeAt, KnownDeps deps, Outcome outcome, LocalExecution execution)
{
- this(status, status.phase, new Known(definition, executeAt, deps, outcome), execution);
+ this(status, status.phase, new Known(route, definition, executeAt, deps, outcome), execution);
}
SaveStatus(Status status, Phase phase, Known known, LocalExecution execution)
@@ -188,30 +202,34 @@ public enum SaveStatus
}
}
+ // TODO (expected): merge Known only, and ensure 1:1 mapping so can reconstruct composite
// TODO (expected, testing): exhaustive testing, particularly around PreCommitted
public static SaveStatus get(Status status, Known known)
{
+ if (known.isInvalidated())
+ return Invalidated;
+
switch (status)
{
default: throw new AssertionError("Unexpected status: " + status);
- case NotDefined: return NotDefined;
- case PreAccepted: return PreAccepted;
+ case NotDefined: return known.executeAt.isDecidedAndKnownToExecute() ? PreCommitted : NotDefined;
+ case PreAccepted: return known.executeAt.isDecidedAndKnownToExecute() ? PreCommittedWithDefinition : PreAccepted;
case AcceptedInvalidate:
// AcceptedInvalidate logically clears any proposed deps and executeAt
- if (!known.executeAt.hasDecidedExecuteAt())
+ if (!known.executeAt.isDecidedAndKnownToExecute())
return known.isDefinitionKnown() ? AcceptedInvalidateWithDefinition : AcceptedInvalidate;
// If we know the executeAt decision then we do not clear it, and fall-through to PreCommitted
// however, we still clear the deps, as any deps we might have previously seen proposed are now expired
// TODO (expected, consider): consider clearing Command.partialDeps in this case also
known = known.with(DepsUnknown);
case Accepted:
- if (!known.executeAt.hasDecidedExecuteAt())
+ if (!known.executeAt.isDecidedAndKnownToExecute())
return known.isDefinitionKnown() ? AcceptedWithDefinition : Accepted;
// if the decision is known, we're really PreCommitted
case PreCommitted:
if (known.isDefinitionKnown())
- return known.deps.hasProposedOrDecidedDeps() ? PreCommittedWithDefinitionAndAcceptedDeps : PreCommittedWithDefinition;
- return known.deps.hasProposedOrDecidedDeps() ? PreCommittedWithAcceptedDeps : PreCommitted;
+ return known.deps == DepsProposed ? PreCommittedWithDefinitionAndAcceptedDeps : PreCommittedWithDefinition;
+ return known.deps == DepsProposed ? PreCommittedWithAcceptedDeps : PreCommitted;
case Committed: return Committed;
case ReadyToExecute: return ReadyToExecute;
case PreApplied: return PreApplied;
@@ -225,12 +243,14 @@ public enum SaveStatus
switch (status.status)
{
// most statuses already know everything they can
+ case NotDefined:
+ case PreAccepted:
case Accepted:
case AcceptedInvalidate:
case PreCommitted:
if (known.isSatisfiedBy(status.known))
return status;
- return get(status.status, status.known.merge(known));
+ return get(status.status, status.known.atLeast(known));
case Truncated:
switch (status)
@@ -259,6 +279,7 @@ public enum SaveStatus
return status;
}
+ // TODO (expected): tighten up distinction of "preferKnowledge" and its interaction with CheckStatus
public static SaveStatus merge(SaveStatus a, Ballot acceptedA, SaveStatus b, Ballot acceptedB, boolean preferKnowledge)
{
// we first enrich cleanups with the knowledge of the other, to avoid counter-intuitive situations where
diff --git a/accord-core/src/main/java/accord/local/SerializerSupport.java b/accord-core/src/main/java/accord/local/SerializerSupport.java
index abdd7f19..6816d4e2 100644
--- a/accord-core/src/main/java/accord/local/SerializerSupport.java
+++ b/accord-core/src/main/java/accord/local/SerializerSupport.java
@@ -133,7 +133,7 @@ public class SerializerSupport
{
Propagate propagate = messageProvider.propagateCommit();
txn = propagate.partialTxn;
- deps = propagate.partialDeps;
+ deps = propagate.committedDeps;
}
else
{
@@ -175,7 +175,7 @@ public class SerializerSupport
{
Propagate propagate = messageProvider.propagateApply();
txn = propagate.partialTxn;
- deps = propagate.partialDeps;
+ deps = propagate.committedDeps;
writes = propagate.writes;
result = propagate.result;
}
@@ -221,7 +221,7 @@ public class SerializerSupport
{
Propagate propagateCommit = messageProvider.propagateCommit();
txn = propagateCommit.partialTxn;
- deps = propagateCommit.partialDeps;
+ deps = propagateCommit.committedDeps;
}
else if (witnessed.contains(COMMIT_MINIMAL_REQ))
{
@@ -340,7 +340,7 @@ public class SerializerSupport
else if (witnessed.contains(PROPAGATE_COMMIT_MSG))
{
Propagate propagate = messageProvider.propagateCommit();
- return new TxnAndDeps(propagate.partialTxn, propagate.partialDeps);
+ return new TxnAndDeps(propagate.partialTxn, propagate.committedDeps);
}
else
{
@@ -359,7 +359,7 @@ public class SerializerSupport
else if (witnessed.contains(PROPAGATE_APPLY_MSG))
{
Propagate propagate = messageProvider.propagateApply();
- return new TxnAndDeps(propagate.partialTxn, propagate.partialDeps);
+ return new TxnAndDeps(propagate.partialTxn, propagate.committedDeps);
}
else if (witnessed.contains(COMMIT_MAXIMAL_REQ))
{
@@ -369,7 +369,7 @@ public class SerializerSupport
else if (witnessed.contains(PROPAGATE_COMMIT_MSG))
{
Propagate propagate = messageProvider.propagateCommit();
- return new TxnAndDeps(propagate.partialTxn, propagate.partialDeps);
+ return new TxnAndDeps(propagate.partialTxn, propagate.committedDeps);
}
else if (witnessed.contains(COMMIT_MINIMAL_REQ))
{
diff --git a/accord-core/src/main/java/accord/local/Status.java b/accord-core/src/main/java/accord/local/Status.java
index bf0f3bef..1f67b4b4 100644
--- a/accord-core/src/main/java/accord/local/Status.java
+++ b/accord-core/src/main/java/accord/local/Status.java
@@ -30,21 +30,25 @@ import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.Stream;
+import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static accord.local.Status.Definition.*;
import static accord.local.Status.Known.*;
import static accord.local.Status.KnownDeps.*;
import static accord.local.Status.KnownExecuteAt.*;
+import static accord.local.Status.KnownRoute.Covering;
+import static accord.local.Status.KnownRoute.Full;
+import static accord.local.Status.KnownRoute.Maybe;
import static accord.local.Status.Outcome.*;
import static accord.local.Status.Phase.*;
public enum Status
{
NotDefined (None, Nothing),
- PreAccepted (PreAccept, DefinitionOnly),
- AcceptedInvalidate(Accept, DefinitionUnknown, ExecuteAtUnknown, DepsUnknown, Unknown), // may or may not have witnessed
- Accepted (Accept, DefinitionUnknown, ExecuteAtProposed, DepsProposed, Unknown), // may or may not have witnessed
+ PreAccepted (PreAccept, DefinitionAndRoute),
+ AcceptedInvalidate(Accept, Maybe, DefinitionUnknown, ExecuteAtUnknown, DepsUnknown, Unknown), // may or may not have witnessed
+ Accepted (Accept, Covering, DefinitionUnknown, ExecuteAtProposed, DepsProposed, Unknown), // may or may not have witnessed
/**
* PreCommitted is a peculiar state, half-way between Accepted and Committed.
@@ -71,16 +75,16 @@ public enum Status
* To solve this problem we simply permit the executeAt we discover for B to be propagated to A* without
* its dependencies. Though this does complicate the state machine a little.
*/
- PreCommitted (Accept, DefinitionUnknown, ExecuteAtKnown, DepsUnknown, Unknown),
+ PreCommitted (Accept, Maybe, DefinitionUnknown, ExecuteAtKnown, DepsUnknown, Unknown),
- Committed (Commit, DefinitionKnown, ExecuteAtKnown, DepsKnown, Unknown),
+ Committed (Commit, Full, DefinitionKnown, ExecuteAtKnown, DepsKnown, Unknown),
// TODO (expected): do we need ReadyToExecute here, or can we keep it to SaveStatus only?
- ReadyToExecute (Commit, DefinitionKnown, ExecuteAtKnown, DepsKnown, Unknown),
+ ReadyToExecute (Commit, Full, DefinitionKnown, ExecuteAtKnown, DepsKnown, Unknown),
// TODO (expected): do we need both PreApplied and Applied here, or can we keep them to SaveStatus only?
- PreApplied (Persist, DefinitionKnown, ExecuteAtKnown, DepsKnown, Outcome.Apply),
- Applied (Persist, DefinitionKnown, ExecuteAtKnown, DepsKnown, Outcome.Apply),
- Truncated (Cleanup, DefinitionUnknown, ExecuteAtUnknown, DepsUnknown, Unknown),
- Invalidated (Persist, NoOp, NoExecuteAt, NoDeps, Outcome.Invalidated),
+ PreApplied (Persist, Full, DefinitionKnown, ExecuteAtKnown, DepsKnown, Outcome.Apply),
+ Applied (Persist, Full, DefinitionKnown, ExecuteAtKnown, DepsKnown, Outcome.Apply),
+ Truncated (Cleanup, Maybe, DefinitionErased, ExecuteAtErased, DepsErased, Outcome.Erased),
+ Invalidated (Persist, Maybe, NoOp, NoExecuteAt, NoDeps, Outcome.Invalidated),
;
/**
@@ -101,40 +105,89 @@ public enum Status
* A vector of various facets of knowledge about, or required for, processing a transaction.
* Each property is treated independently, there is no precedence relationship between them.
* Each property's values are however ordered with respect to each other.
+ *
+ * This information does not need to be consistent with
*/
public static class Known
{
- public static final Known Nothing = new Known(DefinitionUnknown, ExecuteAtUnknown, DepsUnknown, Unknown);
- public static final Known DefinitionOnly = new Known(DefinitionKnown, ExecuteAtUnknown, DepsUnknown, Unknown);
- public static final Known ExecuteAtOnly = new Known(DefinitionUnknown, ExecuteAtKnown, DepsUnknown, Unknown);
- public static final Known Decision = new Known(DefinitionKnown, ExecuteAtKnown, DepsKnown, Unknown);
- public static final Known Apply = new Known(DefinitionUnknown, ExecuteAtKnown, DepsKnown, Outcome.Apply);
- public static final Known Invalidated = new Known(DefinitionUnknown, ExecuteAtUnknown, DepsUnknown, Outcome.Invalidated);
-
+ public static final Known Nothing = new Known(Maybe, DefinitionUnknown, ExecuteAtUnknown, DepsUnknown, Unknown);
+ // TODO (expected): deprecate DefinitionOnly
+ public static final Known DefinitionOnly = new Known(Maybe, DefinitionKnown, ExecuteAtUnknown, DepsUnknown, Unknown);
+ public static final Known DefinitionAndRoute = new Known(Full, DefinitionKnown, ExecuteAtUnknown, DepsUnknown, Unknown);
+ public static final Known ExecuteAtOnly = new Known(Maybe, DefinitionUnknown, ExecuteAtKnown, DepsUnknown, Unknown);
+ public static final Known Decision = new Known(Full, DefinitionKnown, ExecuteAtKnown, DepsKnown, Unknown);
+ public static final Known Apply = new Known(Full, DefinitionUnknown, ExecuteAtKnown, DepsKnown, Outcome.Apply);
+ public static final Known Invalidated = new Known(Maybe, DefinitionUnknown, ExecuteAtUnknown, DepsUnknown, Outcome.Invalidated);
+
+ public final KnownRoute route;
public final Definition definition;
public final KnownExecuteAt executeAt;
public final KnownDeps deps;
public final Outcome outcome;
- public Known(Definition definition, KnownExecuteAt executeAt, KnownDeps deps, Outcome outcome)
+ public Known(Known copy)
{
+ this.route = copy.route;
+ this.definition = copy.definition;
+ this.executeAt = copy.executeAt;
+ this.deps = copy.deps;
+ this.outcome = copy.outcome;
+ }
+
+ public Known(KnownRoute route, Definition definition, KnownExecuteAt executeAt, KnownDeps deps, Outcome outcome)
+ {
+ this.route = route;
this.definition = definition;
this.executeAt = executeAt;
this.deps = deps;
this.outcome = outcome;
}
- public Known merge(Known with)
+ public Known atLeast(Known that)
{
- Definition maxDefinition = definition.compareTo(with.definition) >= 0 ? definition : with.definition;
- KnownExecuteAt maxExecuteAt = executeAt.compareTo(with.executeAt) >= 0 ? executeAt : with.executeAt;
- KnownDeps maxDeps = deps.compareTo(with.deps) >= 0 ? deps : with.deps;
- Outcome maxOutcome = outcome.compareTo(with.outcome) >= 0 ? outcome : with.outcome;
- if (maxDefinition == definition && maxExecuteAt == executeAt && maxDeps == deps && maxOutcome == outcome)
- return this;
- if (maxDefinition == with.definition && maxExecuteAt == with.executeAt && maxDeps == with.deps && maxOutcome == with.outcome)
+ // TODO (expected): validate no inconsistent combos
+ KnownRoute maxRoute = route.atLeast(that.route);
+ Definition maxDefinition = definition.atLeast(that.definition);
+ KnownExecuteAt maxExecuteAt = executeAt.atLeast(that.executeAt);
+ KnownDeps maxDeps = deps.atLeast(that.deps);
+ Outcome maxOutcome = outcome.atLeast(that.outcome);
+ return selectOrCreate(that, maxRoute, maxDefinition, maxExecuteAt, maxDeps, maxOutcome);
+ }
+
+ public Known reduce(Known that)
+ {
+ KnownRoute maxRoute = route.reduce(that.route);
+ Definition minDefinition = definition.reduce(that.definition);
+ KnownExecuteAt maxExecuteAt = executeAt.reduce(that.executeAt);
+ KnownDeps minDeps = deps.reduce(that.deps);
+ Outcome maxOutcome = outcome.reduce(that.outcome);
+ return selectOrCreate(that, maxRoute, minDefinition, maxExecuteAt, minDeps, maxOutcome);
+ }
+
+ public Known validForAll()
+ {
+ KnownRoute maxRoute = route.validForAll();
+ Definition minDefinition = definition.validForAll();
+ KnownExecuteAt maxExecuteAt = executeAt.validForAll();
+ KnownDeps minDeps = deps.validForAll();
+ Outcome maxOutcome = outcome.validForAll();
+ return selectOrCreate(maxRoute, minDefinition, maxExecuteAt, minDeps, maxOutcome);
+ }
+
+ @Nonnull
+ private Known selectOrCreate(Known with, KnownRoute maxRoute, Definition maxDefinition, KnownExecuteAt maxExecuteAt, KnownDeps maxDeps, Outcome maxOutcome)
+ {
+ if (maxRoute == with.route && maxDefinition == with.definition && maxExecuteAt == with.executeAt && maxDeps == with.deps && maxOutcome == with.outcome)
return with;
- return new Known(maxDefinition, maxExecuteAt, maxDeps, maxOutcome);
+ return selectOrCreate(maxRoute, maxDefinition, maxExecuteAt, maxDeps, maxOutcome);
+ }
+
+ @Nonnull
+ private Known selectOrCreate(KnownRoute maxRoute, Definition maxDefinition, KnownExecuteAt maxExecuteAt, KnownDeps maxDeps, Outcome maxOutcome)
+ {
+ if (maxRoute == route && maxDefinition == definition && maxExecuteAt == executeAt && maxDeps == deps && maxOutcome == outcome)
+ return this;
+ return new Known(maxRoute, maxDefinition, maxExecuteAt, maxDeps, maxOutcome);
}
public boolean isSatisfiedBy(Known that)
@@ -172,17 +225,52 @@ public enum Status
{
if (outcome == newOutcome)
return this;
- return new Known(definition, executeAt, deps, newOutcome);
+ return new Known(route, definition, executeAt, deps, newOutcome);
}
public Known with(KnownDeps newDeps)
{
if (deps == newDeps)
return this;
- return new Known(definition, executeAt, newDeps, outcome);
+ return new Known(route, definition, executeAt, newDeps, outcome);
+ }
+
+ /**
+ * Convert this Known to one that represents the knowledge that can be propagated to another replica without
+ * further information available to that replica, e.g. we cannot apply without also knowing the definition.
+ */
+ // TODO (expected): merge propagates and propagatesStatus
+ public Known propagates()
+ {
+ if (isInvalidated())
+ return Invalidated;
+
+ if (definition == DefinitionUnknown)
+ return executeAt.isDecidedAndKnownToExecute() ? ExecuteAtOnly : Nothing;
+
+ KnownExecuteAt executeAt = this.executeAt;
+ if (!executeAt.isDecidedAndKnownToExecute())
+ return DefinitionOnly;
+
+ // cannot propagate proposed deps; and cannot propagate known deps without executeAt
+ KnownDeps deps = this.deps;
+ if (!deps.hasDecidedDeps())
+ return SaveStatus.PreCommittedWithDefinition.known;
+
+ switch (outcome)
+ {
+ default: throw new AssertionError("Unhandled outcome: " + outcome);
+ case Unknown:
+ case WasApply:
+ case Erased:
+ return Committed.minKnown;
+
+ case Apply:
+ return PreApplied.minKnown;
+ }
}
- public Status propagate()
+ public Status propagatesStatus()
{
switch (outcome)
{
@@ -195,14 +283,14 @@ public enum Status
case Apply:
case WasApply:
- if (executeAt.hasDecidedExecuteAt() && definition.isKnown() && deps.hasDecidedDeps())
+ if (executeAt.isDecidedAndKnownToExecute() && definition.isKnown() && deps.hasDecidedDeps())
return PreApplied;
case Unknown:
- if (executeAt.hasDecidedExecuteAt() && definition.isKnown() && deps.hasDecidedDeps())
+ if (executeAt.isDecidedAndKnownToExecute() && definition.isKnown() && deps.hasDecidedDeps())
return Committed;
- if (executeAt.hasDecidedExecuteAt())
+ if (executeAt.isDecidedAndKnownToExecute())
return PreCommitted;
if (definition.isKnown())
@@ -212,7 +300,7 @@ public enum Status
throw new IllegalStateException();
}
- return NotDefined;
+ return outcome == WasApply ? Truncated : NotDefined;
}
public boolean isDefinitionKnown()
@@ -220,6 +308,11 @@ public enum Status
return definition.isKnown();
}
+ public boolean hasDefinitionBeenKnown()
+ {
+ return definition.isKnown() || outcome.isDecided();
+ }
+
/**
* The command may have an incomplete route when this is false
*/
@@ -228,25 +321,141 @@ public enum Status
return definition.isKnown() || outcome.isOrWasApply();
}
+ public boolean isTruncated()
+ {
+ switch (outcome)
+ {
+ default: throw new AssertionError("Unhandled outcome: " + outcome);
+ case Invalidated:
+ case Unknown:
+ return false;
+ case Apply:
+ // since Apply is universal, we can
+ return deps == DepsErased;
+ case Erased:
+ case WasApply:
+ return true;
+ }
+ }
+
public boolean canProposeInvalidation()
{
return deps.canProposeInvalidation() && executeAt.canProposeInvalidation() && outcome.canProposeInvalidation();
}
+ public Known subtract(Known subtract)
+ {
+ if (!subtract.isSatisfiedBy(this))
+ return Known.Nothing;
+
+ Definition newDefinition = subtract.definition.compareTo(definition) >= 0 ? DefinitionUnknown : definition;
+ KnownExecuteAt newExecuteAt = subtract.executeAt.compareTo(executeAt) >= 0 ? ExecuteAtUnknown : executeAt;
+ KnownDeps newDeps = subtract.deps.compareTo(deps) >= 0 ? DepsUnknown : deps;
+ Outcome newOutcome = outcome.subtract(subtract.outcome);
+ return new Known(route, newDefinition, newExecuteAt, newDeps, newOutcome);
+ }
+
+ public boolean isDecided()
+ {
+ return executeAt.isDecided() || outcome.isDecided();
+ }
+
+ public boolean isDecidedToExecute()
+ {
+ return executeAt.isDecidedAndKnownToExecute() || outcome.isOrWasApply();
+ }
+
public String toString()
{
return Stream.of(definition.isKnown() ? "Definition" : null,
- executeAt.hasDecidedExecuteAt() ? "ExecuteAt" : null,
+ executeAt.isDecidedAndKnownToExecute() ? "ExecuteAt" : null,
deps.hasDecidedDeps() ? "Deps" : null,
- outcome.isOrWasApply() ? "Outcome" : null
+ outcome.isDecided() ? outcome.toString() : null
).filter(Objects::nonNull).collect(Collectors.joining(",", "[", "]"));
}
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ Known that = (Known) o;
+ return route == that.route && definition == that.definition && executeAt == that.executeAt && deps == that.deps && outcome == that.outcome;
+ }
+
+ @Override
+ public int hashCode()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ public boolean hasDefinition()
+ {
+ return definition.isKnown();
+ }
+
+ public boolean hasDecidedDeps()
+ {
+ return deps.hasDecidedDeps();
+ }
+
+ public boolean isInvalidated()
+ {
+ return outcome.isInvalidated();
+ }
+ }
+
+ public enum KnownRoute
+ {
+ /**
+ * A route may or may not be known, but it may not cover (or even intersect) this shard.
+ * The route should be relied upon only if it is a FullRoute.
+ */
+ Maybe,
+
+ /**
+ * A route is known that covers the ranges this shard participates in.
+ * Note that if the status is less than Committed, this may not be the final set of owned ranges,
+ * and the route may not cover whatever this is decided as.
+ *
+ * This status primarily exists to communicate semantically to the reader.
+ */
+ Covering,
+
+ /**
+ * The full route is known. <i>Generally</i> this coincides with knowing the Definition.
+ */
+ Full
+ ;
+
+ public boolean hasFull()
+ {
+ return this == Full;
+ }
+
+ public KnownRoute reduce(KnownRoute that)
+ {
+ if (this == that) return this;
+ if (this == Full || that == Full) return Full;
+ return Maybe;
+ }
+
+ public KnownRoute validForAll()
+ {
+ return this == Covering ? Maybe : this;
+ }
+
+ public KnownRoute atLeast(KnownRoute that)
+ {
+ return this.compareTo(that) >= 0 ? this : that;
+ }
}
public enum KnownExecuteAt
{
/**
- * No decision is known to have been reached
+ * No decision is known to have been reached. If executeAt is not null, it represents either when
+ * the transaction was witnessed, or some earlier ExecuteAtProposed that was invalidated by AcceptedInvalidate
*/
ExecuteAtUnknown,
@@ -255,6 +464,11 @@ public enum Status
*/
ExecuteAtProposed,
+ /**
+ * A decision to execute or invalidate the transaction is known to have been reached and since been cleaned up
+ */
+ ExecuteAtErased,
+
/**
* A decision to execute the transaction is known to have been reached, and the associated executeAt timestamp
*/
@@ -266,19 +480,35 @@ public enum Status
NoExecuteAt
;
- public boolean hasDecidedExecuteAt()
+ /**
+ * Is known to have agreed to execute or not; but the decision is not known (maybe erased)
+ */
+ public boolean isDecided()
{
- return compareTo(ExecuteAtKnown) >= 0;
+ return compareTo(ExecuteAtErased) >= 0;
}
- public boolean hasDecidedNonZeroExecuteAt()
+ /**
+ * Is known to execute, and when.
+ */
+ public boolean isDecidedAndKnownToExecute()
{
return this == ExecuteAtKnown;
}
- public final Timestamp executeAtIfKnownElseTxnId(TxnId txnId, Timestamp executeAt)
+ public KnownExecuteAt atLeast(KnownExecuteAt that)
+ {
+ return compareTo(that) >= 0 ? this : that;
+ }
+
+ public KnownExecuteAt reduce(KnownExecuteAt that)
+ {
+ return atLeast(that);
+ }
+
+ public KnownExecuteAt validForAll()
{
- return this == ExecuteAtKnown ? executeAt : txnId;
+ return compareTo(ExecuteAtErased) <= 0 ? ExecuteAtUnknown : this;
}
public boolean canProposeInvalidation()
@@ -300,6 +530,12 @@ public enum Status
*/
DepsProposed,
+ /**
+ * A decision to execute or invalidate the transaction is known to have been reached, and any associated
+ * dependencies for the shard(s) in question have been cleaned up.
+ */
+ DepsErased,
+
/**
* A decision to execute the transaction is known to have been reached, and the associated dependencies
* for the shard(s) in question are known for the coordination and execution epochs.
@@ -322,11 +558,6 @@ public enum Status
return this == DepsKnown;
}
- public boolean isDecided()
- {
- return compareTo(DepsKnown) >= 0;
- }
-
public boolean canProposeInvalidation()
{
return this == DepsUnknown;
@@ -336,6 +567,21 @@ public enum Status
{
return this == DepsProposed || this == DepsKnown;
}
+
+ public KnownDeps atLeast(KnownDeps that)
+ {
+ return compareTo(that) >= 0 ? this : that;
+ }
+
+ public KnownDeps reduce(KnownDeps that)
+ {
+ return compareTo(that) <= 0 ? this : that;
+ }
+
+ public KnownDeps validForAll()
+ {
+ return this == NoDeps ? NoDeps : DepsUnknown;
+ }
}
/**
@@ -349,26 +595,47 @@ public enum Status
DefinitionUnknown,
/**
- * The definition is known
- *
- * TODO (expected, clarity): distinguish between known for coordination epoch and known for commit/execute
+ * The definition was known, but has been erased
*/
- DefinitionKnown,
+ DefinitionErased,
/**
* The definition is irrelevant, as the transaction has been invalidated and may be treated as a no-op
*/
- NoOp;
+ NoOp,
- public boolean canProposeInvalidation()
+ /**
+ * The definition is known
+ *
+ * TODO (expected, clarity): distinguish between known for coordination epoch and known for commit/execute
+ */
+ DefinitionKnown;
+
+ public boolean isKnown()
{
- return this == DefinitionUnknown;
+ return this == DefinitionKnown;
}
- public boolean isKnown()
+ public boolean isOrWasKnown()
{
return this == DefinitionKnown;
}
+
+ public Definition atLeast(Definition that)
+ {
+ return compareTo(that) >= 0 ? this : that;
+ }
+
+ // combine info about two shards into a composite representation
+ public Definition reduce(Definition that)
+ {
+ return compareTo(that) <= 0 ? this : that;
+ }
+
+ public Definition validForAll()
+ {
+ return this == NoOp ? NoOp : DefinitionUnknown;
+ }
}
/**
@@ -382,26 +649,25 @@ public enum Status
Unknown,
/**
- * The outcome is known
+ * The transaction has been *completely cleaned up* - this means it has been made
+ * durable at every live replica of every shard we contacted
*/
- Apply,
+ Erased,
/**
* The transaction has been cleaned-up, but was applied and the relevant portion of its outcome has been cleaned up
- * TODO (expected): is this state helpful? Feels like we can do without it
*/
WasApply,
/**
- * The transaction is known to have been invalidated
+ * The outcome is known
*/
- Invalidated,
+ Apply,
/**
- * The transaction has been *completely cleaned up* - this means it has been made
- * durable at every live replica of every shard we contacted
+ * The transaction is known to have been invalidated
*/
- Erased
+ Invalidated
;
public boolean isOrWasApply()
@@ -441,9 +707,33 @@ public enum Status
return this == Invalidated;
}
- public boolean isTruncated()
+ public Outcome atLeast(Outcome that)
+ {
+ return this.compareTo(that) >= 0 ? this : that;
+ }
+
+ // outcomes are universal - any replica of any shard may propagate its outcome to any other replica of any other shard
+ public Outcome reduce(Outcome that)
{
- return this == Erased || this == WasApply;
+ return atLeast(that);
+ }
+
+ /**
+ * Do not imply truncation where none has happened
+ */
+ public Outcome validForAll()
+ {
+ return compareTo(WasApply) <= 0 ? Unknown : this;
+ }
+
+ public Outcome subtract(Outcome that)
+ {
+ return this.compareTo(that) <= 0 ? Unknown : this;
+ }
+
+ public boolean isDecided()
+ {
+ return this != Unknown;
}
}
@@ -456,22 +746,51 @@ public enum Status
* Represents the durability of a transaction's Persist phase.
* NotDurable: the outcome has not been durably recorded
* Local: the outcome has been durably recorded at least locally
+ * ShardUniversalOrInvalidated: the outcome has been durably recorded at all healthy replicas of the shard, or is invalidated
+ * ShardUniversal: the outcome has been durably recorded at all healthy replicas of the shard
+ * MajorityOrInvalidated: the outcome has been durably recorded to a majority of each participating shard
* Majority: the outcome has been durably recorded to a majority of each participating shard
- * Universal: the outcome has been durably recorded to every participating replica
- * DurableOrInvalidated: the outcome was either invalidated, or has been durably recorded to every replica
+ * Universal: the outcome has been durably recorded to every healthy replica
+ * DurableOrInvalidated: the outcome was either invalidated, or has been durably recorded to every healthy replica
*/
public enum Durability
{
- NotDurable, Local, Majority, Universal, DurableOrInvalidated;
+ NotDurable, Local, ShardUniversal,
+ MajorityOrInvalidated, Majority,
+ UniversalOrInvalidated, Universal;
public boolean isDurable()
{
- return compareTo(Majority) >= 0 && compareTo(DurableOrInvalidated) < 0;
+ return this == Majority || this == Universal;
}
public boolean isDurableOrInvalidated()
{
- return compareTo(Majority) >= 0;
+ return compareTo(MajorityOrInvalidated) >= 0;
+ }
+
+ public boolean isMaybeInvalidated()
+ {
+ return this == NotDurable || this == MajorityOrInvalidated || this == UniversalOrInvalidated;
+ }
+
+ public static Durability merge(Durability a, Durability b)
+ {
+ int c = a.compareTo(b);
+ if (c < 0) { Durability tmp = a; a = b; b = tmp; }
+ // if we know we are applied, we can remove the OrInvalidated qualifier
+ if (a == UniversalOrInvalidated && (b == Majority || b == ShardUniversal || b == Local)) a = Universal;
+ if ((a == ShardUniversal) && (b == Local || b == NotDurable)) a = Local;
+ if (b == NotDurable && a.compareTo(MajorityOrInvalidated) < 0) a = NotDurable;
+ return a;
+ }
+
+ public static Durability mergeAtLeast(Durability a, Durability b)
+ {
+ int c = a.compareTo(b);
+ if (c < 0) { Durability tmp = a; a = b; b = tmp; }
+ if (a == UniversalOrInvalidated && (b == Majority || b == ShardUniversal || b == Local)) a = Universal;
+ return a;
}
}
@@ -484,10 +803,10 @@ public enum Status
this.minKnown = minKnown;
}
- Status(Phase phase, Definition definition, KnownExecuteAt executeAt, KnownDeps deps, Status.Outcome outcome)
+ Status(Phase phase, KnownRoute route, Definition definition, KnownExecuteAt executeAt, KnownDeps deps, Status.Outcome outcome)
{
this.phase = phase;
- this.minKnown = new Known(definition, executeAt, deps, outcome);
+ this.minKnown = new Known(route, definition, executeAt, deps, outcome);
}
// TODO (desired, clarity): investigate all uses of hasBeen, and migrate as many as possible to testing
@@ -512,7 +831,8 @@ public enum Status
Ballot accepted = getAccepted.apply(item);
boolean update = max == null
|| maxStatus.phase.compareTo(status.phase) < 0
- || (status.phase.equals(Phase.Accept) && maxAccepted.compareTo(accepted) < 0);
+ || (status.phase == Accept && maxAccepted.compareTo(accepted) < 0)
+ || status.phase != Accept && maxStatus.phase == status.phase && maxStatus.compareTo(status) > 0;
if (!update)
continue;
@@ -534,9 +854,4 @@ public enum Status
return a;
return b;
}
-
- public static Status simpleMax(Status a, Status b)
- {
- return a.compareTo(b) >= 0 ? a : b;
- }
}
diff --git a/accord-core/src/main/java/accord/messages/AbstractExecute.java b/accord-core/src/main/java/accord/messages/AbstractExecute.java
index 804c101a..6784f88a 100644
--- a/accord-core/src/main/java/accord/messages/AbstractExecute.java
+++ b/accord-core/src/main/java/accord/messages/AbstractExecute.java
@@ -38,6 +38,7 @@ import accord.primitives.Timestamp;
import accord.primitives.TxnId;
import accord.topology.Topologies;
+import static accord.local.SaveStatus.LocalExecution.ReadyToExecute;
import static accord.local.SaveStatus.LocalExecution.WaitingToExecute;
import static accord.local.Status.Committed;
import static accord.messages.ReadData.ReadNack.NotCommitted;
@@ -210,7 +211,11 @@ public abstract class AbstractExecute extends ReadData implements Command.Transi
safeStore.progressLog().waiting(safeCommand, WaitingToExecute, null, readScope);
if (status == Committed) return null;
- else return NotCommitted;
+ else
+ {
+ safeStore.progressLog().waiting(safeCommand, ReadyToExecute, null, readScope);
+ return NotCommitted;
+ }
case OBSOLETE:
state = State.OBSOLETE;
return Redundant;
@@ -268,6 +273,8 @@ public abstract class AbstractExecute extends ReadData implements Command.Transi
throw new IllegalStateException("ReadOk was sent, yet ack called again");
case OBSOLETE:
logger.debug("After the read completed for txn {}, the result was marked obsolete", txnId);
+ if (fail != null)
+ node.agent().onUncaughtException(fail);
break;
case PENDING:
state = State.RETURNED;
@@ -283,18 +290,18 @@ public abstract class AbstractExecute extends ReadData implements Command.Transi
return new ReadOk(unavailable, data);
}
- private void removeListener(SafeCommandStore safeStore, TxnId txnId)
- {
- SafeCommand safeCommand = safeStore.ifInitialised(txnId);
- safeCommand.removeListener(this);
- }
-
@Override
protected void cancel()
{
node.commandStores().mapReduceConsume(this, waitingOn.stream(), forEach(in -> removeListener(in, txnId), node.agent()));
}
+ private void removeListener(SafeCommandStore safeStore, TxnId txnId)
+ {
+ SafeCommand safeCommand = safeStore.ifInitialised(txnId);
+ safeCommand.removeListener(this);
+ }
+
@Override
public String toString()
{
diff --git a/accord-core/src/main/java/accord/messages/Accept.java b/accord-core/src/main/java/accord/messages/Accept.java
index 44abe4fd..5e069ab5 100644
--- a/accord-core/src/main/java/accord/messages/Accept.java
+++ b/accord-core/src/main/java/accord/messages/Accept.java
@@ -29,6 +29,7 @@ import accord.local.SafeCommand;
import accord.local.SafeCommandStore;
import accord.primitives.Ballot;
import accord.primitives.Deps;
+import accord.primitives.EpochSupplier;
import accord.primitives.FullRoute;
import accord.primitives.PartialDeps;
import accord.primitives.PartialRoute;
@@ -109,7 +110,8 @@ public class Accept extends TxnRequest.WithUnsynced<Accept.AcceptReply>
private PartialDeps calculatePartialDeps(SafeCommandStore safeStore)
{
- return PreAccept.calculatePartialDeps(safeStore, txnId, keys, executeAt, safeStore.ranges().allBetween(minUnsyncedEpoch, executeAt));
+ Ranges ranges = safeStore.ranges().allBetween(minUnsyncedEpoch, txnId);
+ return PreAccept.calculatePartialDeps(safeStore, txnId, keys, EpochSupplier.constant(minUnsyncedEpoch), executeAt, ranges);
}
@Override
diff --git a/accord-core/src/main/java/accord/messages/Apply.java b/accord-core/src/main/java/accord/messages/Apply.java
index a2e874be..02983598 100644
--- a/accord-core/src/main/java/accord/messages/Apply.java
+++ b/accord-core/src/main/java/accord/messages/Apply.java
@@ -42,6 +42,7 @@ import accord.primitives.TxnId;
import accord.primitives.Unseekables;
import accord.primitives.Writes;
import accord.topology.Topologies;
+import accord.utils.Invariants;
public class Apply extends TxnRequest<ApplyReply>
{
@@ -72,6 +73,7 @@ public class Apply extends TxnRequest<ApplyReply>
protected Apply(Kind kind, Id to, Topologies participates, Topologies executes, TxnId txnId, Route<?> route, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result)
{
super(to, participates, route, txnId);
+ Invariants.checkState(txnId.rw() != Txn.Kind.Write || writes != null);
Ranges slice = kind == Kind.Maximal || executes == participates ? scope.covering() : executes.computeRangesForNode(to);
// TODO (desired): it's wasteful to encode the full set of ranges owned by the recipient node;
// often it will be cheaper to include the FullRoute for Deps scope (or come up with some other safety-preserving encoding scheme)
diff --git a/accord-core/src/main/java/accord/messages/ApplyThenWaitUntilApplied.java b/accord-core/src/main/java/accord/messages/ApplyThenWaitUntilApplied.java
index 02b4fc19..e590f7e5 100644
--- a/accord-core/src/main/java/accord/messages/ApplyThenWaitUntilApplied.java
+++ b/accord-core/src/main/java/accord/messages/ApplyThenWaitUntilApplied.java
@@ -114,6 +114,12 @@ public class ApplyThenWaitUntilApplied extends WaitUntilApplied
node.agent().onLocalBarrier(partialTxnKeys, txnId);
}
+ @Override
+ public MessageType type()
+ {
+ return MessageType.APPLY_THEN_WAIT_UNTIL_APPLIED_REQ;
+ }
+
@Override
public String toString()
{
diff --git a/accord-core/src/main/java/accord/messages/BeginInvalidation.java b/accord-core/src/main/java/accord/messages/BeginInvalidation.java
index 5a3b0dbe..790a4115 100644
--- a/accord-core/src/main/java/accord/messages/BeginInvalidation.java
+++ b/accord-core/src/main/java/accord/messages/BeginInvalidation.java
@@ -187,11 +187,6 @@ public class BeginInvalidation extends AbstractEpochRequest<BeginInvalidation.In
return Status.max(invalidateReplies, r -> r.status, r -> r.accepted, invalidateReply -> true);
}
- public static InvalidateReply maxNotTruncated(List<InvalidateReply> invalidateReplies)
- {
- return Status.max(invalidateReplies, r -> r.status, r -> r.accepted, invalidateReply -> true);
- }
-
public static RoutingKey findHomeKey(List<InvalidateReply> invalidateOks)
{
for (InvalidateReply ok : invalidateOks)
diff --git a/accord-core/src/main/java/accord/messages/BeginRecovery.java b/accord-core/src/main/java/accord/messages/BeginRecovery.java
index e23a764c..56adf49a 100644
--- a/accord-core/src/main/java/accord/messages/BeginRecovery.java
+++ b/accord-core/src/main/java/accord/messages/BeginRecovery.java
@@ -46,7 +46,6 @@ import accord.utils.Invariants;
import static accord.local.SafeCommandStore.TestDep.WITH;
import static accord.local.SafeCommandStore.TestDep.WITHOUT;
-import static accord.local.SafeCommandStore.TestKind.shouldHaveWitnessed;
import static accord.local.SafeCommandStore.TestTimestamp.EXECUTES_AFTER;
import static accord.local.SafeCommandStore.TestTimestamp.STARTED_AFTER;
import static accord.local.SafeCommandStore.TestTimestamp.STARTED_BEFORE;
@@ -118,7 +117,8 @@ public class BeginRecovery extends TxnRequest<BeginRecovery.RecoverReply>
PartialDeps deps = command.partialDeps();
if (!command.known().deps.hasProposedOrDecidedDeps())
{
- deps = calculatePartialDeps(safeStore, txnId, partialTxn.keys(), txnId, safeStore.ranges().coordinates(txnId));
+ // TODO (required): consider whether we are safe ignoring the concept of minUnsyncedEpoch here
+ deps = calculatePartialDeps(safeStore, txnId, partialTxn.keys(), txnId, txnId, safeStore.ranges().coordinates(txnId));
}
boolean rejectsFastPath;
@@ -335,32 +335,32 @@ public class BeginRecovery extends TxnRequest<BeginRecovery.RecoverReply>
}
}
- private static Deps acceptedStartedBeforeWithoutWitnessing(SafeCommandStore commandStore, TxnId startedBefore, Ranges ranges, Seekables<?, ?> keys)
+ private static Deps acceptedStartedBeforeWithoutWitnessing(SafeCommandStore safeStore, TxnId startedBefore, Ranges ranges, Seekables<?, ?> keys)
{
try (Deps.Builder builder = Deps.builder())
{
// any transaction that started
- commandStore.mapReduce(keys, ranges, shouldHaveWitnessed(startedBefore.rw()), STARTED_BEFORE, startedBefore, WITHOUT, startedBefore, Accepted, PreCommitted,
- (keyOrRange, txnId, executeAt, saveStatus, prev) -> {
- if (executeAt.compareTo(startedBefore) > 0)
- builder.add(keyOrRange, txnId);
- return builder;
- }, builder, null);
+ safeStore.mapReduce(keys, ranges, startedBefore.rw().witnesses(), STARTED_BEFORE, startedBefore, WITHOUT, startedBefore, Accepted, PreCommitted,
+ (startedBefore0, keyOrRange, txnId, executeAt, status, prev) -> {
+ if (executeAt.compareTo(startedBefore0) > 0)
+ prev.add(keyOrRange, txnId);
+ return prev;
+ }, startedBefore, builder);
return builder.build();
}
}
- private static Deps committedStartedBeforeAndWitnessed(SafeCommandStore commandStore, TxnId startedBefore, Ranges ranges, Seekables<?, ?> keys)
+ private static Deps committedStartedBeforeAndWitnessed(SafeCommandStore safeStore, TxnId startedBefore, Ranges ranges, Seekables<?, ?> keys)
{
try (Deps.Builder builder = Deps.builder())
{
- commandStore.mapReduce(keys, ranges, shouldHaveWitnessed(startedBefore.rw()), STARTED_BEFORE, startedBefore, WITH, startedBefore, Committed, null,
- (keyOrRange, txnId, executeAt, saveStatus, prev) -> builder.add(keyOrRange, txnId), builder, (Deps.AbstractBuilder<Deps>)null);
+ safeStore.mapReduce(keys, ranges, startedBefore.rw().witnesses(), STARTED_BEFORE, startedBefore, WITH, startedBefore, Committed, null,
+ (p1, keyOrRange, txnId, executeAt, status, prev) -> prev.add(keyOrRange, txnId), null, (Deps.AbstractBuilder<Deps>)builder);
return builder.build();
}
}
- private static boolean hasAcceptedStartedAfterWithoutWitnessing(SafeCommandStore commandStore, TxnId startedAfter, Ranges ranges, Seekables<?, ?> keys)
+ private static boolean hasAcceptedStartedAfterWithoutWitnessing(SafeCommandStore safeStore, TxnId startedAfter, Ranges ranges, Seekables<?, ?> keys)
{
/*
* The idea here is to discover those transactions that were started after us and have been Accepted
@@ -368,12 +368,14 @@ public class BeginRecovery extends TxnRequest<BeginRecovery.RecoverReply>
* the fast path. This is central to safe recovery, as if every transaction that executes later has
* witnessed us we are safe to propose the pre-accept timestamp regardless, whereas if any transaction
* has not witnessed us we can safely invalidate (us).
+ *
+ * TODO (required): consider carefully how _adding_ ranges to a CommandStore affects this
*/
- return commandStore.mapReduce(keys, ranges, shouldHaveWitnessed(startedAfter.rw()), STARTED_AFTER, startedAfter, WITHOUT, startedAfter, Accepted, PreCommitted,
- (keyOrRange, txnId, executeAt, saveStatus, prev) -> true, false, true);
+ return safeStore.mapReduce(keys, ranges, startedAfter.rw().witnesses(), STARTED_AFTER, startedAfter, WITHOUT, startedAfter, Accepted, PreCommitted,
+ (p1, keyOrRange, txnId, executeAt, status, prev) -> true, null, false, i -> i);
}
- private static boolean hasCommittedExecutesAfterWithoutWitnessing(SafeCommandStore commandStore, TxnId startedAfter, Ranges ranges, Seekables<?, ?> keys)
+ private static boolean hasCommittedExecutesAfterWithoutWitnessing(SafeCommandStore safeStore, TxnId startedAfter, Ranges ranges, Seekables<?, ?> keys)
{
/*
* The idea here is to discover those transactions that have been decided to execute after us
@@ -382,7 +384,7 @@ public class BeginRecovery extends TxnRequest<BeginRecovery.RecoverReply>
* witnessed us we are safe to propose the pre-accept timestamp regardless, whereas if any transaction
* has not witnessed us we can safely invalidate it.
*/
- return commandStore.mapReduce(keys, ranges, shouldHaveWitnessed(startedAfter.rw()), EXECUTES_AFTER, startedAfter, WITHOUT, startedAfter, Committed, null,
- (keyOrRange, txnId, executeAt, saveStatus, prev) -> true,false, true);
+ return safeStore.mapReduce(keys, ranges, startedAfter.rw().witnesses(), EXECUTES_AFTER, startedAfter, WITHOUT, startedAfter, Committed, null,
+ (p1, keyOrRange, txnId, executeAt, status, prev) -> true, null, false, i -> i);
}
}
diff --git a/accord-core/src/main/java/accord/messages/CheckStatus.java b/accord-core/src/main/java/accord/messages/CheckStatus.java
index 4c8ae207..5feec410 100644
--- a/accord-core/src/main/java/accord/messages/CheckStatus.java
+++ b/accord-core/src/main/java/accord/messages/CheckStatus.java
@@ -18,11 +18,16 @@
package accord.messages;
+import java.util.Arrays;
+import java.util.Objects;
+import java.util.function.Predicate;
import javax.annotation.Nullable;
import accord.api.Result;
import accord.api.RoutingKey;
import accord.coordinate.Infer;
+import accord.coordinate.Infer.InvalidIfNot;
+import accord.coordinate.Infer.IsPreempted;
import accord.local.Command;
import accord.local.Commands;
import accord.local.Node;
@@ -32,14 +37,16 @@ import accord.local.SafeCommand;
import accord.local.SafeCommandStore;
import accord.local.SaveStatus;
import accord.local.Status;
+import accord.primitives.AbstractKeys;
import accord.primitives.Ballot;
import accord.primitives.EpochSupplier;
import accord.primitives.PartialDeps;
import accord.primitives.PartialRoute;
import accord.primitives.PartialTxn;
-import accord.primitives.Participants;
import accord.primitives.ProgressToken;
+import accord.primitives.Range;
import accord.primitives.Ranges;
+import accord.primitives.Routables;
import accord.primitives.Route;
import accord.primitives.Timestamp;
import accord.primitives.TxnId;
@@ -48,17 +55,25 @@ import accord.primitives.Writes;
import accord.topology.Topologies;
import accord.utils.Invariants;
import accord.utils.MapReduceConsume;
+import accord.utils.ReducingIntervalMap;
+import accord.utils.ReducingRangeMap;
+import javax.annotation.Nonnull;
+
+import static accord.coordinate.Infer.InvalidIfNot.NotKnownToBeInvalid;
+import static accord.coordinate.Infer.IsPreempted.NotPreempted;
+import static accord.coordinate.Infer.IsPreempted.Preempted;
import static accord.local.Status.Committed;
-import static accord.local.Status.Definition;
import static accord.local.Status.Durability;
+import static accord.local.Status.Durability.Local;
+import static accord.local.Status.Durability.Majority;
+import static accord.local.Status.Durability.ShardUniversal;
+import static accord.local.Status.Durability.Universal;
import static accord.local.Status.Known;
-import static accord.local.Status.KnownDeps;
-import static accord.local.Status.KnownExecuteAt;
+import static accord.local.Status.KnownDeps.DepsErased;
import static accord.local.Status.NotDefined;
-import static accord.local.Status.Outcome;
-import static accord.local.Status.Phase;
import static accord.local.Status.Truncated;
+import static accord.messages.CheckStatus.WithQuorum.HasQuorum;
import static accord.messages.TxnRequest.computeScope;
import static accord.primitives.Route.castToRoute;
import static accord.primitives.Route.isRoute;
@@ -70,18 +85,19 @@ public class CheckStatus extends AbstractEpochRequest<CheckStatus.CheckStatusRep
public static class SerializationSupport
{
- public static CheckStatusOk createOk(Ranges truncated, Status invalidIfNotAtLeast, SaveStatus status, SaveStatus maxStatus, Ballot promised, Ballot accepted, @Nullable Timestamp executeAt,
+ public static CheckStatusOk createOk(FoundKnownMap map, SaveStatus maxKnowledgeStatus, SaveStatus maxStatus, Ballot promised, Ballot accepted, @Nullable Timestamp executeAt,
boolean isCoordinating, Durability durability,
@Nullable Route<?> route, @Nullable RoutingKey homeKey)
{
- return new CheckStatusOk(truncated, invalidIfNotAtLeast, status, maxStatus, promised, accepted, executeAt, isCoordinating, durability, route, homeKey);
+ return new CheckStatusOk(map, maxKnowledgeStatus, maxStatus, promised, accepted, executeAt, isCoordinating, durability, route, homeKey);
}
- public static CheckStatusOk createOk(Ranges truncated, Status invalidIfNotAtLeast, SaveStatus status, SaveStatus maxStatus, Ballot promised, Ballot accepted, @Nullable Timestamp executeAt,
+ public static CheckStatusOk createOk(FoundKnownMap map, SaveStatus maxKnowledgeStatus, SaveStatus maxStatus,
+ Ballot promised, Ballot accepted, @Nullable Timestamp executeAt,
boolean isCoordinating, Durability durability,
@Nullable Route<?> route, @Nullable RoutingKey homeKey,
PartialTxn partialTxn, PartialDeps committedDeps, Writes writes, Result result)
{
- return new CheckStatusOkFull(truncated, invalidIfNotAtLeast, status, maxStatus, promised, accepted, executeAt, isCoordinating, durability, route, homeKey,
+ return new CheckStatusOkFull(map, maxKnowledgeStatus, maxStatus, promised, accepted, executeAt, isCoordinating, durability, route, homeKey,
partialTxn, committedDeps, writes, result);
}
}
@@ -142,17 +158,23 @@ public class CheckStatus extends AbstractEpochRequest<CheckStatus.CheckStatusRep
if (!command.has(Known.DefinitionOnly) && Route.isRoute(query) && safeStore.ranges().allAt(txnId.epoch()).contains(Route.castToRoute(query).homeKey()))
Commands.informHome(safeStore, safeCommand, Route.castToRoute(query));
- Ranges truncated = command.hasBeen(Truncated) ? safeStore.ranges().allBetween(command.txnId().epoch(), command.executeAtOrTxnId().epoch()) : Ranges.EMPTY;
- Status invalidIfNotAtLeast = invalidIfNotAtLeast(safeStore);
+ Ranges ranges = safeStore.ranges().allBetween(command.txnId().epoch(), command.executeAtIfKnownOrTxnId().epoch());
+ InvalidIfNot invalidIfNotAtLeast = invalidIfNotAtLeast(safeStore);
boolean isCoordinating = isCoordinating(node, command);
+ Durability durability = command.durability();
+ Route<?> route = command.route();
+ if (Route.isFullRoute(route))
+ durability = Durability.mergeAtLeast(durability, safeStore.commandStore().durableBefore().min(txnId, route));
+
switch (includeInfo)
{
default: throw new IllegalStateException();
case No:
case Route:
- return new CheckStatusOk(truncated, isCoordinating, invalidIfNotAtLeast, includeInfo == IncludeInfo.No ? null : command.route(), command);
+ Route<?> respondWithRoute = includeInfo == IncludeInfo.No ? null : route;
+ return new CheckStatusOk(ranges, isCoordinating, invalidIfNotAtLeast, respondWithRoute, durability, command);
case All:
- return new CheckStatusOkFull(truncated, isCoordinating, invalidIfNotAtLeast, command);
+ return new CheckStatusOkFull(ranges, isCoordinating, invalidIfNotAtLeast, durability, command);
}
}
@@ -181,7 +203,7 @@ public class CheckStatus extends AbstractEpochRequest<CheckStatus.CheckStatusRep
else node.reply(replyTo, replyContext, ok, null);
}
- private Status invalidIfNotAtLeast(SafeCommandStore safeStore)
+ private InvalidIfNot invalidIfNotAtLeast(SafeCommandStore safeStore)
{
return Infer.invalidIfNotAtLeast(safeStore, txnId, query);
}
@@ -191,40 +213,326 @@ public class CheckStatus extends AbstractEpochRequest<CheckStatus.CheckStatusRep
boolean isOk();
}
+ /**
+ * This is slightly different to Known, in that it represents the knowledge we have obtained from any shard that
+ * can be applied to any other shard, along with any information we might use to infer invalidation.
+ */
+ public static class FoundKnown extends Known
+ {
+ public static final FoundKnown Nothing = new FoundKnown(Known.Nothing, NotKnownToBeInvalid, NotPreempted);
+ public static final FoundKnown Invalidated = new FoundKnown(Known.Invalidated, NotKnownToBeInvalid, NotPreempted);
+
+ public final InvalidIfNot invalidIfNot;
+ public final IsPreempted isPreempted;
+
+ public FoundKnown(Known known, InvalidIfNot invalidIfNot, IsPreempted isPreempted)
+ {
+ super(known);
+ this.invalidIfNot = invalidIfNot;
+ this.isPreempted = isPreempted;
+ }
+
+ public FoundKnown atLeast(FoundKnown with)
+ {
+ Known known = super.atLeast(with);
+ if (known == this)
+ return this;
+ return new FoundKnown(known, invalidIfNot.atLeast(with.invalidIfNot), isPreempted.merge(with.isPreempted));
+ }
+
+ public FoundKnown reduce(FoundKnown with)
+ {
+ Known known = super.reduce(with);
+ if (known == this)
+ return this;
+ return new FoundKnown(known, invalidIfNot.reduce(with.invalidIfNot), isPreempted.validForBoth(with.isPreempted));
+ }
+
+ public FoundKnown validForAll()
+ {
+ Known known = super.validForAll();
+ if (known == this)
+ return this;
+ return new FoundKnown(known, NotKnownToBeInvalid, NotPreempted);
+ }
+
+ public boolean isTruncated()
+ {
+ switch (outcome)
+ {
+ default: throw new AssertionError("Unhandled outcome: " + outcome);
+ case Invalidated:
+ case Unknown:
+ return false;
+ case Apply:
+ // since Apply is universal, we can
+ return deps == DepsErased;
+ case Erased:
+ case WasApply:
+ return true;
+ }
+ }
+
+ public boolean canProposeInvalidation()
+ {
+ return deps.canProposeInvalidation() && executeAt.canProposeInvalidation() && outcome.canProposeInvalidation();
+ }
+
+ public boolean inferInvalidWithQuorum()
+ {
+ return invalidIfNot.inferInvalidWithQuorum(isPreempted, this);
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ FoundKnown that = (FoundKnown) o;
+ return route == that.route && definition == that.definition && executeAt == that.executeAt && deps == that.deps && outcome == that.outcome && invalidIfNot == that.invalidIfNot && isPreempted == that.isPreempted;
+ }
+ }
+
+ // TODO (expected, consider): build only with keys/ranges found in command stores, not the covering ranges of command stores?
+ // by including the whole covering ranges we can infer
+ public static class FoundKnownMap extends ReducingRangeMap<FoundKnown>
+ {
+ public static class SerializerSupport
+ {
+ public static FoundKnownMap create(boolean inclusiveEnds, RoutingKey[] ends, FoundKnown[] values)
+ {
+ return new FoundKnownMap(inclusiveEnds, ends, values);
+ }
+ }
+
+ private transient final FoundKnown validForAll;
+
+ private FoundKnownMap()
+ {
+ this.validForAll = FoundKnown.Nothing;
+ }
+
+ private FoundKnownMap(boolean inclusiveEnds, RoutingKey[] starts, FoundKnown[] values)
+ {
+ this(inclusiveEnds, starts, values, FoundKnown.Nothing);
+ }
+
+ private FoundKnownMap(boolean inclusiveEnds, RoutingKey[] starts, FoundKnown[] values, FoundKnown validForAll)
+ {
+ super(inclusiveEnds, starts, values);
+ this.validForAll = validForAll;
+ }
+
+ public static FoundKnownMap create(Unseekables<?> keysOrRanges, SaveStatus saveStatus, InvalidIfNot invalidIfNot, Ballot promised)
+ {
+ FoundKnown known = new FoundKnown(saveStatus.known, invalidIfNot, promised.equals(Ballot.ZERO) ? NotPreempted : Preempted);
+ if (keysOrRanges.isEmpty())
+ return new FoundKnownMap();
+
+ switch (keysOrRanges.domain())
+ {
+ default: throw new AssertionError("Unhandled domain type: " + keysOrRanges.domain());
+ case Range:
+ {
+ Ranges ranges = (Ranges)keysOrRanges;
+ Builder builder = new Builder(ranges.get(0).endInclusive(), 2 * ranges.size());
+ for (Range range : ranges)
+ {
+ builder.append(range.start(), known, FoundKnown::atLeast);
+ builder.append(range.end(), null, FoundKnown::atLeast);
+ }
+ return builder.build();
+ }
+ case Key:
+ {
+ AbstractKeys<RoutingKey> keys = (AbstractKeys<RoutingKey>) keysOrRanges;
+ Builder builder = new Builder(keys.get(0).asRange().endInclusive(), 2 * keys.size());
+ for (RoutingKey key : keys)
+ {
+ Range range = key.asRange();
+ builder.append(range.start(), known, FoundKnown::atLeast);
+ builder.append(range.end(), null, FoundKnown::atLeast);
+ }
+ return builder.build();
+ }
+ }
+ }
+
+ public static FoundKnownMap merge(FoundKnownMap a, FoundKnownMap b)
+ {
+ return ReducingRangeMap.merge(a, b, FoundKnown::atLeast, Builder::new);
+ }
+
+ private FoundKnownMap finish(Unseekables<?> routeOrParticipants, WithQuorum withQuorum)
+ {
+ FoundKnown validForAll;
+ switch (withQuorum)
+ {
+ default: throw new AssertionError("Unhandled WithQuorum: " + withQuorum);
+ case HasQuorum: validForAll = foldlWithDefault(routeOrParticipants, FoundKnownMap::reduceInferredOrKnownForWithQuorum, FoundKnown.Nothing, null, i -> false); break;
+ case NoQuorum: validForAll = foldlWithDefault(routeOrParticipants, FoundKnownMap::reduceKnownFor, FoundKnown.Nothing, null, i -> false); break;
+ }
+ validForAll = this.validForAll.atLeast(validForAll).validForAll();
+ return with(validForAll);
+ }
+
+ private FoundKnownMap with(FoundKnown validForAll)
+ {
+ if (validForAll.equals(this.validForAll))
+ return this;
+
+ int i = 0;
+ for (; i < size(); ++i)
+ {
+ FoundKnown pre = values[i];
+ if (pre == null)
+ continue;
+
+ FoundKnown post = pre.atLeast(validForAll);
+ if (!pre.equals(post))
+ break;
+ }
+
+ if (i == size())
+ return new FoundKnownMap(inclusiveEnds(), starts, values, validForAll);
+
+ RoutingKey[] newStarts = new RoutingKey[size() + 1];
+ FoundKnown[] newValues = new FoundKnown[size()];
+ System.arraycopy(starts, 0, newStarts, 0, i);
+ System.arraycopy(values, 0, newValues, 0, i);
+ int count = i;
+ while (i < size())
+ {
+ FoundKnown pre = values[i++];
+ FoundKnown post = pre == null ? null : pre.atLeast(validForAll);
+ if (count == 0 || !Objects.equals(post, newValues[count - 1]))
+ {
+ newStarts[count] = starts[i-1];
+ newValues[count++] = post;
+ }
+ }
+ newStarts[count] = starts[size()];
+ if (count != newValues.length)
+ {
+ newValues = Arrays.copyOf(newValues, count);
+ newStarts = Arrays.copyOf(newStarts, count + 1);
+ }
+ return new FoundKnownMap(inclusiveEnds(), newStarts, newValues, validForAll);
+ }
+
+ public boolean hasTruncated(Routables<?> routables)
+ {
+ return foldlWithDefault(routables, (known, prev) -> known.isTruncated(), FoundKnown.Nothing, false, i -> i);
+ }
+
+ public boolean hasTruncated()
+ {
+ return foldl((known, prev) -> known.isTruncated(), false, i -> i);
+ }
+
+ public Known knownFor(Routables<?> routables)
+ {
+ return validForAll.atLeast(foldlWithDefault(routables, FoundKnownMap::reduceKnownFor, FoundKnown.Nothing, null, i -> false));
+ }
+
+ public Ranges matchingRanges(Predicate<FoundKnown> match)
+ {
+ return foldlWithBounds((known, ranges, start, end) -> match.test(known) ? ranges.with(Ranges.of(start.asRange().newRange(start, end))) : ranges, Ranges.EMPTY, i -> false);
+ }
+
+ private static FoundKnown reduceInferredOrKnownForWithQuorum(FoundKnown foundKnown, @Nullable FoundKnown prev)
+ {
+ if (foundKnown.inferInvalidWithQuorum())
+ foundKnown = foundKnown.atLeast(FoundKnown.Invalidated);
+
+ if (prev == null)
+ return foundKnown;
+
+ return prev.reduce(foundKnown);
+ }
+
+ private static Known reduceKnownFor(FoundKnown foundKnown, @Nullable Known prev)
+ {
+ if (prev == null)
+ return foundKnown;
+
+ return prev.reduce(foundKnown);
+ }
+
+ private static FoundKnown reduceKnownFor(FoundKnown foundKnown, @Nullable FoundKnown prev)
+ {
+ if (prev == null)
+ return foundKnown;
+
+ return prev.reduce(foundKnown);
+ }
+
+ public Ranges knownFor(Known required, Ranges expect)
+ {
+ // TODO (desired): implement and use foldlWithDefaultAndBounds so can subtract rather than add
+ return foldlWithBounds(expect, (known, prev, start, end) -> {
+ if (!required.isSatisfiedBy(known))
+ return prev;
+
+ return prev.with(Ranges.of(start.asRange().newRange(start, end)));
+ }, Ranges.EMPTY, i -> false);
+ }
+
+ static class Builder extends ReducingIntervalMap.Builder<RoutingKey, FoundKnown, FoundKnownMap>
+ {
+ protected Builder(boolean inclusiveEnds, int capacity)
+ {
+ super(inclusiveEnds, capacity);
+ }
+
+ @Override
+ protected FoundKnownMap buildInternal()
+ {
+ return new FoundKnownMap(inclusiveEnds, starts.toArray(new RoutingKey[0]), values.toArray(new FoundKnown[0]));
+ }
+ }
+ }
+
public static class CheckStatusOk implements CheckStatusReply
{
- public final Ranges truncated;
- public final Status invalidIfNotAtLeast;
- public final SaveStatus saveStatus; // the maximum non-truncated status; or truncated if all responses are truncated
- public final SaveStatus maxSaveStatus;
- public final Ballot promised;
- public final Ballot accepted;
+ public final FoundKnownMap map;
+ // TODO (required): tighten up constraints here to ensure we only report truncated when the range is Durable
+ // TODO (expected): stop using saveStatus and maxSaveStatus - move to only Known
+ // care needed when merging Accepted and AcceptedInvalidate; might be easier to retain saveStatus only for merging these cases
+ public final SaveStatus maxKnowledgeSaveStatus, maxSaveStatus;
+ public final Ballot promised, accepted;
public final @Nullable Timestamp executeAt; // not set if invalidating or invalidated
public final boolean isCoordinating;
- public final Durability durability; // i.e. on all shards
+ public final Durability durability;
public final @Nullable Route<?> route;
public final @Nullable RoutingKey homeKey;
- public CheckStatusOk(Ranges truncated, boolean isCoordinating, Status invalidIfNotAtLeast, Command command)
+ public CheckStatusOk(Ranges ranges, boolean isCoordinating, InvalidIfNot invalidIfNot, Durability durability, Command command)
{
- this(truncated, isCoordinating, invalidIfNotAtLeast, command.route(), command);
+ this(ranges, isCoordinating, invalidIfNot, command.route(), durability, command);
}
- public CheckStatusOk(Ranges truncated, boolean isCoordinating, Status invalidIfNotAtLeast, Route<?> route, Command command)
+ public CheckStatusOk(Ranges ranges, boolean isCoordinating, InvalidIfNot invalidIfNot, Route<?> route, Durability durability, Command command)
{
- this(truncated, invalidIfNotAtLeast, command.saveStatus(), command.saveStatus(), command.promised(), command.accepted(), command.executeAt(),
- isCoordinating, command.durability(), route, command.homeKey());
+ this(ranges, invalidIfNot, command.saveStatus(), command.promised(), command.accepted(), command.executeAt(),
+ isCoordinating, durability, route, command.homeKey());
}
- private CheckStatusOk(Ranges truncated, Status invalidIfNotAtLeast, SaveStatus saveStatus, SaveStatus maxSaveStatus,
- Ballot promised, Ballot accepted, @Nullable Timestamp executeAt,
+ private CheckStatusOk(Ranges ranges, InvalidIfNot invalidIfNot, SaveStatus saveStatus, Ballot promised,
+ Ballot accepted, @Nullable Timestamp executeAt,
boolean isCoordinating, Durability durability,
@Nullable Route<?> route, @Nullable RoutingKey homeKey)
{
- this.truncated = truncated;
- this.invalidIfNotAtLeast = invalidIfNotAtLeast;
- this.saveStatus = saveStatus;
+ this(FoundKnownMap.create(ranges, saveStatus, invalidIfNot, promised), saveStatus, saveStatus, promised, accepted, executeAt, isCoordinating, durability, route, homeKey);
+ }
+
+ private CheckStatusOk(FoundKnownMap map, SaveStatus maxKnowledgeSaveStatus, SaveStatus maxSaveStatus, Ballot promised, Ballot accepted,
+ @Nullable Timestamp executeAt, boolean isCoordinating, Durability durability,
+ @Nullable Route<?> route, @Nullable RoutingKey homeKey)
+ {
+ this.map = map;
this.maxSaveStatus = maxSaveStatus;
+ this.maxKnowledgeSaveStatus = maxKnowledgeSaveStatus;
this.promised = promised;
this.accepted = accepted;
this.executeAt = executeAt;
@@ -240,22 +548,96 @@ public class CheckStatus extends AbstractEpochRequest<CheckStatus.CheckStatusRep
return new ProgressToken(durability, status, promised, accepted);
}
- public boolean isTruncated()
+ public Timestamp executeAtIfKnown()
+ {
+ if (maxKnown().executeAt.isDecidedAndKnownToExecute())
+ return executeAt;
+ return null;
+ }
+
+ public CheckStatusOk finish(Unseekables<?> routeOrParticipants, WithQuorum withQuorum)
+ {
+ CheckStatusOk finished = this;
+ if (withQuorum == HasQuorum)
+ {
+ Durability durability = this.durability;
+ if (durability == Local) durability = Majority;
+ else if (durability == ShardUniversal) durability = Universal;
+ finished = merge(durability);
+ }
+ if (Route.isRoute(routeOrParticipants))
+ {
+ finished = finished.merge(Route.castToRoute(routeOrParticipants));
+ return finished.with(finished.map.finish(finished.route, withQuorum));
+ }
+ else
+ {
+ return finished.with(finished.map.finish(Unseekables.merge(routeOrParticipants, (Unseekables) finished.route), withQuorum));
+ }
+ }
+
+ /**
+ * NOTE: if the response is *incomplete* this does not detect possible truncation, it only indicates if the
+ * combination of the responses we received represents truncation
+ */
+ public boolean isTruncatedResponse()
+ {
+ return map.hasTruncated();
+ }
+
+ public boolean isTruncatedResponse(Routables<?> routables)
+ {
+ return map.hasTruncated(routables);
+ }
+
+ public Ranges truncatedResponse()
+ {
+ return map.matchingRanges(FoundKnown::isTruncated);
+ }
+
+ public CheckStatusOk merge(@Nonnull Route<?> route)
{
- return maxSaveStatus.phase == Phase.Cleanup;
+ Route<?> mergedRoute = Route.merge((Route)this.route, route);
+ if (mergedRoute == this.route)
+ return this;
+ return new CheckStatusOk(map, maxKnowledgeSaveStatus, maxSaveStatus, promised, accepted, executeAt,
+ isCoordinating, durability, mergedRoute, homeKey);
}
- public boolean inferInvalidated(WithQuorum withQuorum)
+ public CheckStatusOk merge(@Nonnull Durability durability)
{
- return Infer.inferInvalidated(withQuorum, invalidIfNotAtLeast, saveStatus, maxSaveStatus);
+ durability = Durability.merge(durability, this.durability);
+ if (durability == this.durability)
+ return this;
+ return new CheckStatusOk(map, maxKnowledgeSaveStatus, maxSaveStatus, promised, accepted, executeAt,
+ isCoordinating, durability, route, homeKey);
}
- public Known inferredOrKnown(WithQuorum withQuorum)
+ CheckStatusOk with(@Nonnull FoundKnownMap newMap)
{
- if (inferInvalidated(withQuorum))
- return Known.Invalidated;
+ if (newMap == this.map)
+ return this;
+ return new CheckStatusOk(newMap, maxKnowledgeSaveStatus, maxSaveStatus, promised, accepted, executeAt,
+ isCoordinating, durability, route, homeKey);
+ }
- return saveStatus.known;
+ // TODO (required): harden markShardStale against unnecessary actions by utilising inferInvalidated==MAYBE and performing a global query
+ public Known knownFor(Routables<?> participants)
+ {
+ Known known = map.knownFor(participants);
+ Invariants.checkState(!known.hasFullRoute() || Route.isFullRoute(route));
+ Invariants.checkState(!known.outcome.isInvalidated() || (!maxKnowledgeSaveStatus.known.isDecidedToExecute() && !maxSaveStatus.known.isDecidedToExecute()));
+ Invariants.checkState(!(maxSaveStatus.known.outcome.isInvalidated() || maxKnowledgeSaveStatus.known.outcome.isInvalidated()) || !known.isDecidedToExecute());
+ // TODO (desired): make sure these match identically, rather than only ensuring Route.isFullRoute (either by coercing it here or by ensuring it at callers)
+ return known;
+ }
+
+ // it is assumed that we are invoking this for a transaction that will execute;
+ // the result may be erroneous if the transaction is invalidated, as logically this can apply to all ranges
+ public Ranges knownFor(Known required, Ranges expect)
+ {
+ Invariants.checkState(maxSaveStatus != SaveStatus.Invalidated);
+ return map.knownFor(required, expect);
}
@Override
@@ -268,7 +650,9 @@ public class CheckStatus extends AbstractEpochRequest<CheckStatus.CheckStatusRep
public String toString()
{
return "CheckStatusOk{" +
- "status:" + saveStatus +
+ "map:" + map +
+ "maxNotTruncatedSaveStatus:" + maxKnowledgeSaveStatus +
+ "maxSaveStatus:" + maxSaveStatus +
", promised:" + promised +
", accepted:" + accepted +
", executeAt:" + executeAt +
@@ -281,10 +665,10 @@ public class CheckStatus extends AbstractEpochRequest<CheckStatus.CheckStatusRep
boolean preferSelf(CheckStatusOk that)
{
- if ((this.saveStatus.is(Truncated) && !this.saveStatus.is(NotDefined)) || (that.saveStatus.is(Truncated) && !that.saveStatus.is(NotDefined)))
- return this.saveStatus.compareTo(that.saveStatus) <= 0;
+ if ((this.maxKnowledgeSaveStatus.is(Truncated) && !this.maxKnowledgeSaveStatus.is(NotDefined)) || (that.maxKnowledgeSaveStatus.is(Truncated) && !that.maxKnowledgeSaveStatus.is(NotDefined)))
+ return this.maxKnowledgeSaveStatus.compareTo(that.maxKnowledgeSaveStatus) <= 0;
- return this.saveStatus.compareTo(that.saveStatus) >= 0;
+ return this.maxKnowledgeSaveStatus.compareTo(that.maxKnowledgeSaveStatus) >= 0;
}
public CheckStatusOk merge(CheckStatusOk that)
@@ -300,31 +684,43 @@ public class CheckStatus extends AbstractEpochRequest<CheckStatus.CheckStatusRep
CheckStatusOk defer = prefer == this ? that : this;
// then select the max along each criteria, preferring the coordinator
- CheckStatusOk maxStatus = SaveStatus.max(prefer, prefer.saveStatus, prefer.accepted, defer, defer.saveStatus, defer.accepted, true);
- SaveStatus mergeStatus = SaveStatus.merge(prefer.saveStatus, prefer.accepted, defer.saveStatus, defer.accepted, true);
- SaveStatus mergeMaxStatus = SaveStatus.merge(prefer.saveStatus, prefer.accepted, defer.saveStatus, defer.accepted, false);
+ FoundKnownMap mergeMap = FoundKnownMap.merge(prefer.map, defer.map);
+ CheckStatusOk maxStatus = SaveStatus.max(prefer, prefer.maxKnowledgeSaveStatus, prefer.accepted, defer, defer.maxKnowledgeSaveStatus, defer.accepted, true);
+ SaveStatus mergeMaxKnowledgeStatus = SaveStatus.merge(prefer.maxKnowledgeSaveStatus, prefer.accepted, defer.maxKnowledgeSaveStatus, defer.accepted, true);
+ SaveStatus mergeMaxStatus = SaveStatus.merge(prefer.maxSaveStatus, prefer.accepted, defer.maxSaveStatus, defer.accepted, false);
CheckStatusOk maxPromised = prefer.promised.compareTo(defer.promised) >= 0 ? prefer : defer;
CheckStatusOk maxAccepted = prefer.accepted.compareTo(defer.accepted) >= 0 ? prefer : defer;
- CheckStatusOk maxDurability = prefer.durability.compareTo(defer.durability) >= 0 ? prefer : defer;
CheckStatusOk maxHomeKey = prefer.homeKey != null || defer.homeKey == null ? prefer : defer;
- CheckStatusOk maxExecuteAt = prefer.saveStatus.known.executeAt.compareTo(defer.saveStatus.known.executeAt) >= 0 ? prefer : defer;
+ CheckStatusOk maxExecuteAt = prefer.maxKnown().executeAt.compareTo(defer.maxKnown().executeAt) >= 0 ? prefer : defer;
Route<?> mergedRoute = Route.merge(prefer.route, (Route)defer.route);
- Ranges mergedTruncated = prefer.truncated.with(defer.truncated);
- Status mergedInvalidIfNotAtLeast = Status.simpleMax(prefer.invalidIfNotAtLeast, defer.invalidIfNotAtLeast);
+ Durability mergedDurability = Durability.merge(prefer.durability, defer.durability);
// if the maximum (or preferred equal) is the same on all dimensions, return it
- if (mergeStatus == maxStatus.saveStatus && maxStatus == maxPromised && maxStatus == maxAccepted && maxStatus == maxDurability
- && maxStatus == maxHomeKey && maxStatus == maxExecuteAt && maxStatus.route == mergedRoute
- && maxStatus.truncated.equals(mergedTruncated) && maxStatus.invalidIfNotAtLeast == mergedInvalidIfNotAtLeast)
+ if (mergeMaxKnowledgeStatus == maxStatus.maxKnowledgeSaveStatus
+ && mergeMaxStatus == maxStatus.maxSaveStatus
+ && maxStatus == maxPromised && maxStatus == maxAccepted
+ && maxStatus == maxHomeKey && maxStatus == maxExecuteAt
+ && maxStatus.route == mergedRoute
+ && maxStatus.map.equals(mergeMap)
+ && maxStatus.durability == mergedDurability)
{
return maxStatus;
}
// otherwise assemble the maximum of each, and propagate isCoordinating from the origin we selected the promise from
boolean isCoordinating = maxPromised == prefer ? prefer.isCoordinating : defer.isCoordinating;
- return new CheckStatusOk(mergedTruncated, mergedInvalidIfNotAtLeast, mergeStatus, mergeMaxStatus,
- maxPromised.promised, maxAccepted.accepted, maxExecuteAt.executeAt,
- isCoordinating, maxDurability.durability, mergedRoute, maxHomeKey.homeKey);
+ return new CheckStatusOk(mergeMap, mergeMaxKnowledgeStatus, mergeMaxStatus, maxPromised.promised, maxAccepted.accepted, maxExecuteAt.executeAt,
+ isCoordinating, mergedDurability, mergedRoute, maxHomeKey.homeKey);
+ }
+
+ public Known maxKnown()
+ {
+ return map.foldl(Known::atLeast, Known.Nothing, i -> false);
+ }
+
+ public InvalidIfNot maxInvalidIfNot()
+ {
+ return map.foldl((known, prev) -> known.invalidIfNot.atLeast(prev), NotKnownToBeInvalid, InvalidIfNot::isMax);
}
@Override
@@ -341,26 +737,57 @@ public class CheckStatus extends AbstractEpochRequest<CheckStatus.CheckStatusRep
public final Writes writes;
public final Result result;
- public CheckStatusOkFull(Ranges truncated, boolean isCoordinating, Status invalidIfNotAtLeast, Command command)
+ public CheckStatusOkFull(Ranges ranges, boolean isCoordinating, InvalidIfNot invalidIfNot, Durability durability, Command command)
{
- super(truncated, isCoordinating, invalidIfNotAtLeast, command);
+ super(ranges, isCoordinating, invalidIfNot, durability, command);
this.partialTxn = command.partialTxn();
this.committedDeps = command.status().compareTo(Committed) >= 0 ? command.partialDeps() : null;
this.writes = command.writes();
this.result = command.result();
}
- protected CheckStatusOkFull(Ranges truncated, Status invalidIfNotCommitted, SaveStatus status, SaveStatus maxStatus, Ballot promised, Ballot accepted, Timestamp executeAt,
- boolean isCoordinating, Durability durability, Route<?> route,
- RoutingKey homeKey, PartialTxn partialTxn, PartialDeps committedDeps, Writes writes, Result result)
+ protected CheckStatusOkFull(FoundKnownMap map, SaveStatus maxNotTruncatedSaveStatus, SaveStatus maxSaveStatus, Ballot promised, Ballot accepted, Timestamp executeAt,
+ boolean isCoordinating, Durability durability, Route<?> route,
+ RoutingKey homeKey, PartialTxn partialTxn, PartialDeps committedDeps, Writes writes, Result result)
{
- super(truncated, invalidIfNotCommitted, status, maxStatus, promised, accepted, executeAt, isCoordinating, durability, route, homeKey);
+ super(map, maxNotTruncatedSaveStatus, maxSaveStatus, promised, accepted, executeAt, isCoordinating, durability, route, homeKey);
this.partialTxn = partialTxn;
this.committedDeps = committedDeps;
this.writes = writes;
this.result = result;
}
+ public CheckStatusOkFull finish(Route<?> route, WithQuorum withQuorum)
+ {
+ return (CheckStatusOkFull) super.finish(route, withQuorum);
+ }
+
+ public CheckStatusOkFull merge(@Nonnull Route<?> route)
+ {
+ Route<?> mergedRoute = Route.merge((Route)this.route, route);
+ if (mergedRoute == this.route)
+ return this;
+ return new CheckStatusOkFull(map, maxKnowledgeSaveStatus, maxSaveStatus, promised, accepted, executeAt,
+ isCoordinating, durability, mergedRoute, homeKey, partialTxn, committedDeps, writes, result);
+ }
+
+ public CheckStatusOkFull merge(@Nonnull Durability durability)
+ {
+ durability = Durability.merge(durability, this.durability);
+ if (durability == this.durability)
+ return this;
+ return new CheckStatusOkFull(map, maxKnowledgeSaveStatus, maxSaveStatus, promised, accepted, executeAt,
+ isCoordinating, durability, route, homeKey, partialTxn, committedDeps, writes, result);
+ }
+
+ CheckStatusOk with(@Nonnull FoundKnownMap newMap)
+ {
+ if (newMap == this.map)
+ return this;
+ return new CheckStatusOkFull(newMap, maxKnowledgeSaveStatus, maxSaveStatus, promised, accepted, executeAt,
+ isCoordinating, durability, route, homeKey, partialTxn, committedDeps, writes, result);
+ }
+
/**
* This method assumes parameter is of the same type and has the same additional info (modulo partial replication).
* If parameters have different info, it is undefined which properties will be returned.
@@ -383,7 +810,7 @@ public class CheckStatus extends AbstractEpochRequest<CheckStatus.CheckStatusRep
CheckStatusOk minSrc = maxSrc == this ? that : this;
if (!(minSrc instanceof CheckStatusOkFull))
{
- return new CheckStatusOkFull(max.truncated, max.invalidIfNotAtLeast, max.saveStatus, max.maxSaveStatus, max.promised, max.accepted, fullMax.executeAt, max.isCoordinating, max.durability, max.route,
+ return new CheckStatusOkFull(max.map, max.maxKnowledgeSaveStatus, max.maxSaveStatus, max.promised, max.accepted, fullMax.executeAt, max.isCoordinating, max.durability, max.route,
max.homeKey, fullMax.partialTxn, fullMax.committedDeps, fullMax.writes, fullMax.result);
}
@@ -397,71 +824,35 @@ public class CheckStatus extends AbstractEpochRequest<CheckStatus.CheckStatusRep
Writes writes = (fullMax.writes != null ? fullMax : fullMin).writes;
Result result = (fullMax.result != null ? fullMax : fullMin).result;
- return new CheckStatusOkFull(max.truncated, max.invalidIfNotAtLeast, max.saveStatus, max.maxSaveStatus, max.promised, max.accepted, max.executeAt, max.isCoordinating, max.durability, max.route,
+ return new CheckStatusOkFull(max.map, max.maxKnowledgeSaveStatus, max.maxSaveStatus, max.promised, max.accepted, max.executeAt, max.isCoordinating, max.durability, max.route,
max.homeKey, partialTxn, committedDeps, writes, result);
}
- public Known sufficientFor(Participants<?> participants, WithQuorum withQuorum)
- {
- if (inferInvalidated(withQuorum))
- return Known.Invalidated;
-
- return sufficientFor(participants, saveStatus, route, partialTxn, committedDeps, writes, result);
- }
-
- private static Known sufficientFor(Participants<?> participants, SaveStatus saveStatus, Route<?> route, PartialTxn partialTxn, PartialDeps committedDeps, Writes writes, Result result)
+ /**
+ * Reduce what is Known about all intersecting shards into a summary Known. This will be the maximal knowledge
+ * we have, i.e. if we have some outcome/decision on one shard but it is truncated on another intersecting shard,
+ * we will get the outcome/decision; if we only have it truncated on one shard and unknown on another, it will
+ * be shown as truncated.
+ *
+ * If a non-intersecting shard has information that can be propagated to this shard, i.e. the executeAt or outcome,
+ * then this will be merged as though it were an intersecting shard, however no record of truncation will be so propagated,
+ * nor any knowledge that does not transfer (i.e. Definition or Deps).
+ */
+ @Override
+ public Known knownFor(Routables<?> participants)
{
- Status.Definition definition = saveStatus.known.definition;
- switch (definition)
- {
- default: throw new AssertionError();
- case DefinitionKnown:
- if (partialTxn != null && partialTxn.covers(participants) && route.kind().isFullRoute())
- break;
- definition = Definition.DefinitionUnknown;
- case DefinitionUnknown:
- case NoOp:
- }
-
- KnownExecuteAt executeAt = saveStatus.known.executeAt;
- KnownDeps deps = saveStatus.known.deps;
- switch (deps)
- {
- default: throw new AssertionError();
- case DepsKnown:
- if (committedDeps != null && committedDeps.covers(participants))
- break;
- case DepsProposed:
- case NoDeps:
- deps = KnownDeps.DepsUnknown;
- case DepsUnknown:
- }
-
- Status.Outcome outcome = saveStatus.known.outcome;
- switch (outcome)
- {
- default: throw new AssertionError();
- case WasApply:
- if (writes != null && result != null)
- outcome = Outcome.Apply;
-
- case Apply:
- if (writes == null || result == null)
- outcome = Outcome.WasApply;
-
- case Invalidated:
- case Unknown:
- case Erased:
- }
-
- return new Known(definition, executeAt, deps, outcome);
+ Known known = super.knownFor(participants);
+ Invariants.checkState(!known.hasDefinition() || (partialTxn != null && partialTxn.covering().containsAll(participants)));
+ Invariants.checkState(!known.hasDecidedDeps() || (committedDeps != null && committedDeps.covering.containsAll(participants)));
+ return known;
}
@Override
public String toString()
{
return "CheckStatusOk{" +
- "status:" + saveStatus +
+ "map:" + map +
+ ", maxSaveStatus:" + maxSaveStatus +
", promised:" + promised +
", accepted:" + accepted +
", executeAt:" + executeAt +
diff --git a/accord-core/src/main/java/accord/messages/Commit.java b/accord-core/src/main/java/accord/messages/Commit.java
index a1b0404b..6f4bfd76 100644
--- a/accord-core/src/main/java/accord/messages/Commit.java
+++ b/accord-core/src/main/java/accord/messages/Commit.java
@@ -96,7 +96,7 @@ public class Commit extends TxnRequest<ReadNack>
if (kind == Kind.Maximal)
{
// boolean isHome = coordinateTopology.rangesForNode(to).contains(route.homeKey());
- // TODO (expected): only includeQuery if isHome; this affects state eviction and is low priority given size in C*
+ // TODO (desired): only includeQuery if isHome; this affects state eviction and is low priority given size in C*
partialTxn = txn.slice(scope.covering(), true);
sendRoute = route;
}
diff --git a/accord-core/src/main/java/accord/messages/GetDeps.java b/accord-core/src/main/java/accord/messages/GetDeps.java
index 083c15c8..28348ef2 100644
--- a/accord-core/src/main/java/accord/messages/GetDeps.java
+++ b/accord-core/src/main/java/accord/messages/GetDeps.java
@@ -33,6 +33,7 @@ import accord.topology.Topologies;
import accord.utils.Invariants;
import static accord.messages.PreAccept.calculatePartialDeps;
+import static accord.primitives.EpochSupplier.constant;
public class GetDeps extends TxnRequest.WithUnsynced<PartialDeps>
{
@@ -71,7 +72,7 @@ public class GetDeps extends TxnRequest.WithUnsynced<PartialDeps>
public PartialDeps apply(SafeCommandStore instance)
{
Ranges ranges = instance.ranges().allBetween(minUnsyncedEpoch, executeAt);
- return calculatePartialDeps(instance, txnId, keys, executeAt, ranges);
+ return calculatePartialDeps(instance, txnId, keys, constant(minUnsyncedEpoch), executeAt, ranges);
}
@Override
diff --git a/accord-core/src/main/java/accord/messages/MessageType.java b/accord-core/src/main/java/accord/messages/MessageType.java
index a3fe592a..56ea2ed4 100644
--- a/accord-core/src/main/java/accord/messages/MessageType.java
+++ b/accord-core/src/main/java/accord/messages/MessageType.java
@@ -66,7 +66,7 @@ public class MessageType
public static final MessageType SET_GLOBALLY_DURABLE_REQ = mt(REMOTE, true );
public static final MessageType QUERY_DURABLE_BEFORE_REQ = mt(REMOTE, false);
public static final MessageType QUERY_DURABLE_BEFORE_RSP = mt(REMOTE, false);
- public static final MessageType APPLY_AND_WAIT_UNTIL_APPLIED_REQ = mt(REMOTE, true );
+ public static final MessageType APPLY_THEN_WAIT_UNTIL_APPLIED_REQ= mt(REMOTE, true );
public static final MessageType PROPAGATE_PRE_ACCEPT_MSG = mt(LOCAL, true );
public static final MessageType PROPAGATE_COMMIT_MSG = mt(LOCAL, true );
diff --git a/accord-core/src/main/java/accord/messages/PreAccept.java b/accord-core/src/main/java/accord/messages/PreAccept.java
index ce72cebc..45e9df5a 100644
--- a/accord-core/src/main/java/accord/messages/PreAccept.java
+++ b/accord-core/src/main/java/accord/messages/PreAccept.java
@@ -20,6 +20,7 @@ package accord.messages;
import java.util.List;
import java.util.Objects;
+import java.util.function.Function;
import javax.annotation.Nullable;
import org.slf4j.Logger;
@@ -30,7 +31,6 @@ import accord.local.Commands;
import accord.local.Node.Id;
import accord.local.SafeCommand;
import accord.local.SafeCommandStore;
-import accord.local.SafeCommandStore.TestKind;
import accord.messages.TxnRequest.WithUnsynced;
import accord.primitives.Deps;
import accord.primitives.EpochSupplier;
@@ -105,10 +105,12 @@ public class PreAccept extends WithUnsynced<PreAccept.PreAcceptReply> implements
protected PreAcceptReply applyIfDoesNotCoordinate(SafeCommandStore safeStore)
{
// we only preaccept in the coordination epoch, but we might contact other epochs for dependencies
+ // TODO (required): for recovery we need to update CommandsForKeys et al, even if we don't participate in coordination.
+ // must consider cleanup though. (alternative is to make recovery more complicated)
Ranges ranges = safeStore.ranges().allBetween(minUnsyncedEpoch, txnId);
if (txnId.rw() == ExclusiveSyncPoint)
safeStore.commandStore().markExclusiveSyncPoint(safeStore, txnId, ranges);
- return new PreAcceptOk(txnId, txnId, calculatePartialDeps(safeStore, txnId, partialTxn.keys(), txnId, ranges));
+ return new PreAcceptOk(txnId, txnId, calculatePartialDeps(safeStore, txnId, partialTxn.keys(), EpochSupplier.constant(minUnsyncedEpoch), txnId, ranges));
}
@Override
@@ -126,15 +128,18 @@ public class PreAccept extends WithUnsynced<PreAccept.PreAcceptReply> implements
{
default:
case Success:
- case Redundant: // we might hit 'Redundant' if we have to contact later epochs and partially re-contact a node we already contacted
+ // we might hit 'Redundant' if we have to contact later epochs and partially re-contact a node we already contacted
+ // TODO (expected): consider dedicated special case, or rename
+ case Redundant:
Command command = safeCommand.current();
// for efficiency, we don't usually return dependencies newer than txnId as they aren't necessarily needed
// for recovery, and it's better to persist less data than more. However, for exclusive sync points we
// don't need to perform an Accept round, nor do we need to persist this state to aid recovery. We just
// want the issuer of the sync point to know which transactions to wait for before it can safely treat
// all transactions with lower txnId as expired.
+ Ranges ranges = safeStore.ranges().allBetween(minUnsyncedEpoch, txnId);
return new PreAcceptOk(txnId, command.executeAt(),
- calculatePartialDeps(safeStore, txnId, partialTxn.keys(), txnId, safeStore.ranges().allBetween(minUnsyncedEpoch, txnId)));
+ calculatePartialDeps(safeStore, txnId, partialTxn.keys(), EpochSupplier.constant(minUnsyncedEpoch), txnId, ranges));
case Truncated:
case RejectedBallot:
@@ -247,29 +252,31 @@ public class PreAccept extends WithUnsynced<PreAccept.PreAcceptReply> implements
}
}
- static PartialDeps calculatePartialDeps(SafeCommandStore commandStore, TxnId txnId, Seekables<?, ?> keys, Timestamp executeAt, Ranges ranges)
+ static PartialDeps calculatePartialDeps(SafeCommandStore safeStore, TxnId txnId, Seekables<?, ?> keys, EpochSupplier minEpoch, Timestamp executeAt, Ranges ranges)
{
- try (PartialDeps.Builder builder = PartialDeps.builder(ranges))
- {
- return calculateDeps(commandStore, txnId, keys, executeAt, ranges, builder);
- }
+ // TODO (expected): do not build covering ranges; no longer especially valuable given use of FullRoute
+ return calculateDeps(safeStore, txnId, keys, minEpoch, executeAt, ranges, PartialDeps::builder);
}
- private static <T extends Deps> T calculateDeps(SafeCommandStore commandStore, TxnId txnId, Seekables<?, ?> keys, Timestamp executeAt, Ranges ranges, Deps.AbstractBuilder<T> builder)
+ private static <T extends Deps> T calculateDeps(SafeCommandStore safeStore, TxnId txnId, Seekables<?, ?> keys, EpochSupplier minEpoch, Timestamp executeAt, Ranges ranges, Function<Ranges, Deps.AbstractBuilder<T>> builderFactory)
{
- TestKind testKind = TestKind.conflicts(txnId.rw());
// could use MAY_EXECUTE_BEFORE to prune those we know execute later.
// NOTE: ExclusiveSyncPoint *relies* on STARTED_BEFORE to ensure it reports a dependency on *every* earlier TxnId that may execute after it.
// This is necessary for reporting to a bootstrapping replica which TxnId it must not prune from dependencies
// i.e. the source replica reports to the target replica those TxnId that STARTED_BEFORE and EXECUTES_AFTER.
- commandStore.mapReduce(keys, ranges, testKind, STARTED_BEFORE, executeAt, ANY_DEPS, null, null, null,
- (keyOrRange, testTxnId, testExecuteAt, saveStatus, in) -> {
- // TODO (easy, efficiency): either pass txnId as parameter or encode this behaviour in a specialised builder to avoid extra allocations
- if (!testTxnId.equals(txnId))
- in.add(keyOrRange, testTxnId);
- return in;
- }, builder, null);
- return builder.build();
+ try (Deps.AbstractBuilder<T> depBuilder = builderFactory.apply(ranges); Deps.AbstractBuilder<T> redundantBuilder = builderFactory.apply(ranges))
+ {
+ // TODO (expected): this can be neater
+ safeStore.mapReduce(keys, ranges, txnId.rw().witnesses(), STARTED_BEFORE, executeAt, ANY_DEPS, null, null, null,
+ (txnId0, keyOrRange, testTxnId, testExecuteAt, testStatus, in) -> {
+ if (!testTxnId.equals(txnId0))
+ in.add(keyOrRange, testTxnId);
+ return in;
+ }, txnId, depBuilder);
+ T deps = depBuilder.build();
+ T redundant = safeStore.commandStore().redundantBefore().collectDeps(keys, redundantBuilder, minEpoch, executeAt).build();
+ return (T)deps.with(redundant);
+ }
}
/**
diff --git a/accord-core/src/main/java/accord/messages/Propagate.java b/accord-core/src/main/java/accord/messages/Propagate.java
index 68a63df1..a0821bcc 100644
--- a/accord-core/src/main/java/accord/messages/Propagate.java
+++ b/accord-core/src/main/java/accord/messages/Propagate.java
@@ -27,6 +27,8 @@ import accord.local.SafeCommand;
import accord.local.SafeCommandStore;
import accord.local.SaveStatus;
import accord.local.Status;
+import accord.messages.CheckStatus.FoundKnownMap;
+import accord.messages.CheckStatus.WithQuorum;
import accord.primitives.EpochSupplier;
import accord.primitives.Keys;
import accord.primitives.PartialDeps;
@@ -44,34 +46,40 @@ import accord.utils.MapReduceConsume;
import javax.annotation.Nullable;
import java.util.function.BiConsumer;
+import static accord.coordinate.Infer.InvalidIfNot.NotKnownToBeInvalid;
+import static accord.local.RedundantStatus.LOCALLY_REDUNDANT;
import static accord.local.Status.NotDefined;
import static accord.local.Status.Phase.Cleanup;
import static accord.local.Status.PreApplied;
+import static accord.messages.CheckStatus.WithQuorum.HasQuorum;
import static accord.primitives.Routables.Slice.Minimal;
public class Propagate implements MapReduceConsume<SafeCommandStore, Void>, EpochSupplier, LocalMessage
{
public static class SerializerSupport
{
- public static Propagate create(TxnId txnId, Route<?> route, SaveStatus saveStatus, SaveStatus maxSaveStatus, Status.Durability durability, RoutingKey homeKey, RoutingKey progressKey, Status.Known achieved, PartialTxn partialTxn, PartialDeps partialDeps, long toEpoch, Timestamp executeAt, Writes writes, Result result)
+ public static Propagate create(TxnId txnId, Route<?> route, SaveStatus maxKnowledgeSaveStatus, SaveStatus maxSaveStatus, Status.Durability durability, RoutingKey homeKey, RoutingKey progressKey, Status.Known achieved, FoundKnownMap known, boolean isTruncated, PartialTxn partialTxn, PartialDeps committedDeps, long toEpoch, Timestamp executeAt, Writes writes, Result result)
{
- return new Propagate(txnId, route, saveStatus, maxSaveStatus, durability, homeKey, progressKey, achieved, partialTxn, partialDeps, toEpoch, executeAt, writes, result, null);
+ return new Propagate(txnId, route, maxKnowledgeSaveStatus, maxSaveStatus, durability, homeKey, progressKey, achieved, known, isTruncated, partialTxn, committedDeps, toEpoch, executeAt, writes, result, null);
}
}
public final TxnId txnId;
public final Route<?> route;
- public final SaveStatus saveStatus;
+ // TODO (expected): remove dependency on these two SaveStatus
+ public final SaveStatus maxKnowledgeSaveStatus;
public final SaveStatus maxSaveStatus;
public final Status.Durability durability;
@Nullable public final RoutingKey homeKey;
@Nullable public final RoutingKey progressKey;
// this is a WHOLE NODE measure, so if commit epoch has more ranges we do not count as committed if we can only commit in coordination epoch
public final Status.Known achieved;
+ public final FoundKnownMap known;
+ public final boolean isTruncated;
@Nullable public final PartialTxn partialTxn;
- @Nullable public final PartialDeps partialDeps;
+ @Nullable public final PartialDeps committedDeps;
public final long toEpoch;
- @Nullable public final Timestamp executeAt;
+ @Nullable public final Timestamp committedExecuteAt;
@Nullable public final Writes writes;
@Nullable public final Result result;
@@ -80,104 +88,107 @@ public class Propagate implements MapReduceConsume<SafeCommandStore, Void>, Epoc
Propagate(
TxnId txnId,
Route<?> route,
- SaveStatus saveStatus,
+ SaveStatus maxKnowledgeSaveStatus,
SaveStatus maxSaveStatus,
Status.Durability durability,
@Nullable RoutingKey homeKey,
@Nullable RoutingKey progressKey,
Status.Known achieved,
+ FoundKnownMap known,
+ boolean isTruncated,
@Nullable PartialTxn partialTxn,
- @Nullable PartialDeps partialDeps,
+ @Nullable PartialDeps committedDeps,
long toEpoch,
- @Nullable Timestamp executeAt,
+ @Nullable Timestamp committedExecuteAt,
@Nullable Writes writes,
@Nullable Result result,
BiConsumer<Status.Known, Throwable> callback)
{
this.txnId = txnId;
this.route = route;
- this.saveStatus = saveStatus;
+ this.maxKnowledgeSaveStatus = maxKnowledgeSaveStatus;
this.maxSaveStatus = maxSaveStatus;
this.durability = durability;
this.homeKey = homeKey;
this.progressKey = progressKey;
this.achieved = achieved;
+ this.known = known;
+ this.isTruncated = isTruncated;
this.partialTxn = partialTxn;
- this.partialDeps = partialDeps;
+ this.committedDeps = committedDeps;
this.toEpoch = toEpoch;
- this.executeAt = executeAt;
+ this.committedExecuteAt = committedExecuteAt;
this.writes = writes;
this.result = result;
this.callback = callback;
}
@SuppressWarnings({"rawtypes", "unchecked"})
- public static void propagate(Node node, TxnId txnId, long sourceEpoch, CheckStatus.WithQuorum withQuorum, Route route, @Nullable Status.Known target, CheckStatus.CheckStatusOkFull full, BiConsumer<Status.Known, Throwable> callback)
+ public static void propagate(Node node, TxnId txnId, long sourceEpoch, WithQuorum withQuorum, Route route, @Nullable Status.Known target, CheckStatus.CheckStatusOkFull full, BiConsumer<Status.Known, Throwable> callback)
{
- if (full.saveStatus.status == NotDefined && full.invalidIfNotAtLeast == NotDefined)
+ if (full.maxKnowledgeSaveStatus.status == NotDefined && full.maxInvalidIfNot() == NotKnownToBeInvalid)
{
callback.accept(Status.Known.Nothing, null);
return;
}
- Invariants.checkState(sourceEpoch == txnId.epoch() || (full.executeAt != null && sourceEpoch == full.executeAt.epoch()));
- Route<?> maxRoute = Route.merge(route, full.route);
+ Invariants.checkState(sourceEpoch == txnId.epoch() || (full.executeAt != null && sourceEpoch == full.executeAt.epoch()) || full.maxKnowledgeSaveStatus == SaveStatus.Erased);
+
+ full = full.finish(route, withQuorum);
+ route = Invariants.nonNull(full.route);
// TODO (required): permit individual shards that are behind to catch up by themselves
long toEpoch = sourceEpoch;
Ranges sliceRanges = node.topology().localRangesForEpochs(txnId.epoch(), toEpoch);
- if (!maxRoute.covers(sliceRanges))
- {
- callback.accept(Status.Known.Nothing, null);
- return;
- }
- RoutingKey progressKey = node.trySelectProgressKey(txnId, maxRoute);
+ RoutingKey progressKey = node.trySelectProgressKey(txnId, route);
- Ranges covering = maxRoute.sliceCovering(sliceRanges, Minimal);
- Participants<?> participatingKeys = maxRoute.participants().slice(covering, Minimal);
- Status.Known achieved = full.sufficientFor(participatingKeys, withQuorum);
- if (achieved.executeAt.hasDecidedExecuteAt() && full.executeAt.epoch() > toEpoch)
+ Ranges covering = route.sliceCovering(sliceRanges, Minimal);
+ Participants<?> participatingKeys = route.participants().slice(covering, Minimal);
+ Status.Known achieved = full.knownFor(participatingKeys);
+ if (achieved.executeAt.isDecidedAndKnownToExecute() && full.executeAt.epoch() > toEpoch)
{
Ranges acceptRanges;
if (!node.topology().hasEpoch(full.executeAt.epoch()) ||
- (!maxRoute.covers(acceptRanges = node.topology().localRangesForEpochs(txnId.epoch(), full.executeAt.epoch()))))
+ (!route.covers(acceptRanges = node.topology().localRangesForEpochs(txnId.epoch(), full.executeAt.epoch()))))
{
// we don't know what the execution epoch requires, so we cannot be sure we can replicate it locally
// we *could* wait until we have the local epoch before running this
Status.Outcome outcome = achieved.outcome.propagatesBetweenShards() ? achieved.outcome : Status.Outcome.Unknown;
- achieved = new Status.Known(achieved.definition, achieved.executeAt, Status.KnownDeps.DepsUnknown, outcome);
+ achieved = new Status.Known(achieved.route, achieved.definition, achieved.executeAt, Status.KnownDeps.DepsUnknown, outcome);
}
else
{
// TODO (expected): this should only be the two precise epochs, not the full range of epochs
sliceRanges = acceptRanges;
- covering = maxRoute.sliceCovering(sliceRanges, Minimal);
- participatingKeys = maxRoute.participants().slice(covering, Minimal);
- Status.Known knownForExecution = full.sufficientFor(participatingKeys, withQuorum);
- if ((target != null && target.isSatisfiedBy(knownForExecution)) || knownForExecution.isSatisfiedBy(achieved))
+ covering = route.sliceCovering(sliceRanges, Minimal);
+ participatingKeys = route.participants().slice(covering, Minimal);
+ Status.Known knownForExecution = full.knownFor(participatingKeys);
+ if ((target != null && target.isSatisfiedBy(knownForExecution)) || achieved.isSatisfiedBy(knownForExecution))
{
achieved = knownForExecution;
toEpoch = full.executeAt.epoch();
}
else
- {
+ { // TODO (expected): does downgrading this ever block progress?
Invariants.checkState(sourceEpoch == txnId.epoch(), "%d != %d", sourceEpoch, txnId.epoch());
- achieved = new Status.Known(achieved.definition, achieved.executeAt, knownForExecution.deps, knownForExecution.outcome);
+ achieved = new Status.Known(achieved.route, achieved.definition, achieved.executeAt, knownForExecution.deps, knownForExecution.outcome);
}
}
}
- PartialTxn partialTxn = null;
+ boolean isTruncated = withQuorum == HasQuorum && full.isTruncatedResponse(covering);
+
+ PartialTxn partialTxn = full.partialTxn;
if (achieved.definition.isKnown())
partialTxn = full.partialTxn.slice(sliceRanges, true).reconstitutePartial(covering);
- PartialDeps partialDeps = null;
+ PartialDeps committedDeps = full.committedDeps;
if (achieved.deps.hasDecidedDeps())
- partialDeps = full.committedDeps.slice(sliceRanges).reconstitutePartial(covering);
+ committedDeps = full.committedDeps.slice(sliceRanges).reconstitutePartial(covering);
Propagate propagate =
- new Propagate(txnId, maxRoute, full.saveStatus, full.maxSaveStatus, full.durability, full.homeKey, progressKey, achieved, partialTxn, partialDeps, toEpoch, full.executeAt, full.writes, full.result, callback);
+ new Propagate(txnId, route, full.maxKnowledgeSaveStatus, full.maxSaveStatus, full.durability, full.homeKey, progressKey, achieved, full.map, isTruncated, partialTxn, committedDeps, toEpoch, full.executeAtIfKnown(), full.writes, full.result, callback);
node.localMessage(propagate);
}
@@ -191,10 +202,12 @@ public class Propagate implements MapReduceConsume<SafeCommandStore, Void>, Epoc
@Override
public Seekables<?, ?> keys()
{
+ // TODO (now): this is insufficient, as we may applyOrUpgradeTruncated
+ // since we don't calculateDeps, we might not need to load CFK to register with it
if (achieved.definition.isKnown())
return partialTxn.keys();
else if (achieved.deps.hasProposedOrDecidedDeps())
- return partialDeps.keyDeps.keys();
+ return committedDeps.keyDeps.keys();
else
return Keys.EMPTY;
}
@@ -210,40 +223,61 @@ public class Propagate implements MapReduceConsume<SafeCommandStore, Void>, Epoc
{
SafeCommand safeCommand = safeStore.get(txnId, this, route);
Command command = safeCommand.current();
- if (command.saveStatus().phase.compareTo(Status.Phase.Persist) >= 0)
- return null;
- Status propagate = achieved.propagate();
+ PartialTxn partialTxn = achieved.hasDefinition() ? this.partialTxn : null;
+ PartialDeps committedDeps = achieved.hasDecidedDeps() ? this.committedDeps : null;
+ switch (command.saveStatus().phase)
+ {
+ // Already know the outcome, waiting on durability so maybe update with new durability information which can also trigger cleanup
+ case Persist: return updateDurability(safeStore, safeCommand);
+ case Cleanup: return null;
+ }
+
+ Timestamp executeAtIfKnown = command.executeAtIfKnown(committedExecuteAt);
+ Status.Known achieved = this.achieved;
+ if (isTruncated)
+ {
+ // TODO (required): do not markShardStale for reads; in general optimise handling of case where we cannot recover a known no-op transaction
+
+ achieved = applyOrUpgradeTruncated(safeStore, safeCommand, command, executeAtIfKnown);
+ if (achieved == null)
+ return null;
+
+ Ranges needed = safeStore.ranges().allBetween(txnId.epoch(), (executeAtIfKnown == null ? txnId : executeAtIfKnown).epoch());
+ if (achieved.isDefinitionKnown() && partialTxn == null)
+ {
+ PartialTxn existing = command.partialTxn();
+ Ranges neededForDefinition = existing == null ? needed : needed.subtract(existing.covering());
+ partialTxn = this.partialTxn.slice(needed, true).reconstitutePartial(neededForDefinition);
+ }
+
+ if (achieved.hasDecidedDeps() && committedDeps == null)
+ {
+ Invariants.checkState(executeAtIfKnown != null);
+ // we don't subtract existing partialDeps, as they cannot be committed deps; we only permit committing deps covering all participating ranges
+ committedDeps = this.committedDeps.slice(needed).reconstitutePartial(needed);
+ }
+ }
+
+ Status propagate = achieved.atLeast(command.known()).propagatesStatus();
if (command.hasBeen(propagate))
{
- if (maxSaveStatus.phase == Cleanup && durability.isDurableOrInvalidated() && Infer.safeToCleanup(safeStore, command, route, executeAt))
+ if (maxSaveStatus.phase == Cleanup && durability.isDurableOrInvalidated() && Infer.safeToCleanup(safeStore, command, route, executeAtIfKnown))
Commands.setTruncatedApply(safeStore, safeCommand);
- return null;
+
+ // TODO (expected): maybe stale?
+ return updateDurability(safeStore, safeCommand);
}
switch (propagate)
{
default: throw new IllegalStateException("Unexpected status: " + propagate);
+ case Truncated: throw new IllegalStateException("Status expected to be handled elsewhere: " + propagate);
case Accepted:
case AcceptedInvalidate:
// we never "propagate" accepted statuses as these are essentially votes,
// and contribute nothing to our local state machine
- throw new IllegalStateException("Invalid states to propagate: " + achieved.propagate());
-
- case Truncated:
- // if our peers have truncated this command, then either:
- // 1) we have already applied it locally; 2) the command doesn't apply locally; 3) we are stale; or 4) the command is invalidated
- if (command.hasBeen(PreApplied) || command.saveStatus().isUninitialised())
- break;
-
- if (Infer.safeToCleanup(safeStore, command, route, executeAt))
- {
- Commands.setErased(safeStore, safeCommand);
- break;
- }
-
- // TODO (required): check if we are stale
- // otherwise we are either stale, or the command didn't reach consensus
+ throw new IllegalStateException("Invalid states to propagate: " + propagate);
case Invalidated:
Commands.commitInvalidate(safeStore, safeCommand, route);
@@ -251,20 +285,21 @@ public class Propagate implements MapReduceConsume<SafeCommandStore, Void>, Epoc
case Applied:
case PreApplied:
- Invariants.checkState(executeAt != null);
- if (toEpoch >= executeAt.epoch())
+ Invariants.checkState(executeAtIfKnown != null);
+ if (toEpoch >= executeAtIfKnown.epoch())
{
- confirm(Commands.apply(safeStore, safeCommand, txnId, route, progressKey, executeAt, partialDeps, partialTxn, writes, result));
+ confirm(Commands.apply(safeStore, safeCommand, txnId, route, progressKey, executeAtIfKnown, committedDeps, partialTxn, writes, result));
break;
}
case Committed:
case ReadyToExecute:
- confirm(Commands.commit(safeStore, safeCommand, txnId, route, progressKey, partialTxn, executeAt, partialDeps));
+ confirm(Commands.commit(safeStore, safeCommand, txnId, route, progressKey, partialTxn, executeAtIfKnown, committedDeps));
break;
case PreCommitted:
- Commands.precommit(safeStore, safeCommand, txnId, executeAt, route);
+ confirm(Commands.precommit(safeStore, safeCommand, txnId, executeAtIfKnown, route));
+ // TODO (desired): would it be clearer to yield a SaveStatus so we can have PreCommittedWithDefinition
if (!achieved.definition.isKnown())
break;
@@ -278,15 +313,119 @@ public class Propagate implements MapReduceConsume<SafeCommandStore, Void>, Epoc
break;
}
+ return updateDurability(safeStore, safeCommand);
+ }
- if (!durability.isDurable() || homeKey == null)
+ // if can only propagate Truncated, we might be stale; try to upgrade for this command store only, even partially if necessary
+ // note: this is invoked if the command is truncated for ANY local command store - we might
+ private Status.Known applyOrUpgradeTruncated(SafeCommandStore safeStore, SafeCommand safeCommand, Command command, Timestamp executeAtIfKnown)
+ {
+ Invariants.checkState(!maxKnowledgeSaveStatus.is(Status.Invalidated));
+
+ if (Infer.safeToCleanup(safeStore, command, route, executeAtIfKnown))
+ {
+ // don't create a new Erased record if we're already cleaned up
+ Commands.setErased(safeStore, safeCommand);
return null;
+ }
+
+ Ranges ranges = safeStore.ranges().allBetween(txnId.epoch(), (executeAtIfKnown == null ? txnId : executeAtIfKnown).epoch());
+ Participants<?> participants = route.participants(ranges, Minimal);
+ Invariants.checkState(!participants.isEmpty()); // we shouldn't be fetching data for transactions we only coordinate
+ boolean isTruncatedForLocalRanges = known.hasTruncated(participants);
- if (!safeStore.ranges().coordinates(txnId).contains(homeKey))
+ if (!isTruncatedForLocalRanges)
+ {
+ // we're truncated *somewhere* but not locally; whether we have the executeAt is immaterial to this calculus,
+ // as we're either ready to go or we're waiting on the coordinating shard to complete this transaction, so pick
+ // the maximum we can achieve and return that
+ return known.knownFor(participants);
+ }
+
+ // if our peers have truncated this command, then either:
+ // 1) we have already applied it locally; 2) the command doesn't apply locally; 3) we are stale; or 4) the command is invalidated
+ // we're now at least partially stale, but let's first see if we can progress this shard, or we can do so in part
+ if (executeAtIfKnown == null)
+ {
+ // TODO (desired): merge this with the executeAt known path where possible, as a lot of duplication
+ if (safeStore.commandStore().redundantBefore().status(txnId, txnId, participants).compareTo(LOCALLY_REDUNDANT) >= 0)
+ {
+ // invalidated
+ Commands.setErased(safeStore, safeCommand);
+ return null;
+ }
+
+ ranges = safeStore.commandStore().redundantBefore().everExpectToExecute(txnId, ranges);
+ if (!ranges.isEmpty())
+ {
+ // we don't even know the execution time, so we cannot possibly proceed besides erasing the command state and marking ourselves stale
+ // TODO (required): we could in principle be stale for future epochs we haven't witnessed yet. Ensure up to date epochs before finalising this application, or else fetch a maximum possible epoch
+ safeStore.commandStore().markShardStale(safeStore, txnId, participants.toRanges().slice(ranges, Minimal), false);
+ }
+ Commands.setErased(safeStore, safeCommand);
+ return null;
+ }
+
+ if (safeStore.commandStore().redundantBefore().status(txnId, executeAtIfKnown, participants).compareTo(LOCALLY_REDUNDANT) >= 0)
+ {
+ // invalidated
+ Commands.setErased(safeStore, safeCommand);
+ return null;
+ }
+
+ // compute the ranges we expect to execute - i.e. those we own, and are not stale or pre-bootstrap
+ ranges = safeStore.commandStore().redundantBefore().expectToExecute(txnId, executeAtIfKnown, ranges);
+ if (ranges.isEmpty())
+ {
+ // TODO (expected): we might prefer to adopt Redundant status, and permit ourselves to later accept the result of the execution and/or definition
+ Commands.setTruncatedApply(safeStore, safeCommand, route);
+ return null;
+ }
+
+ // if the command has been truncated globally, then we should expect to apply it
+ // if we cannot obtain enough information from a majority to do so then we have been left behind
+ Status.Known required = PreApplied.minKnown;
+ Status.Known requireExtra = required.subtract(command.known()); // the extra information we need to reach pre-applied
+ Ranges achieveRanges = known.knownFor(requireExtra, ranges); // the ranges for which we can successfully achieve this
+
+ if (participants.isEmpty())
+ {
+ // we only coordinate this transaction, so being unable to retrieve its state does not imply any staleness
+ // TODO (now): double check this doesn't stop us coordinating the transaction (it shouldn't, as doesn't imply durability)
+ Commands.setTruncatedApply(safeStore, safeCommand, route);
+ return null;
+ }
+
+ // any ranges we execute but cannot achieve the pre-applied status for have been left behind and are stale
+ Ranges staleRanges = ranges.subtract(achieveRanges);
+ Participants<?> staleParticipants = participants.slice(staleRanges, Minimal);
+ staleRanges = staleParticipants.toRanges();
+
+ if (staleRanges.isEmpty())
+ {
+ Invariants.checkState(achieveRanges.containsAll(participants));
+ return required;
+ }
+
+ safeStore.commandStore().markShardStale(safeStore, executeAtIfKnown, staleRanges, true);
+ if (!staleRanges.containsAll(participants))
+ return required;
+
+ // TODO (expected): we might prefer to adopt Redundant status, and permit ourselves to later accept the result of the execution and/or definition
+ Commands.setTruncatedApply(safeStore, safeCommand, route);
+ return null;
+ }
+
+ /*
+ * If there is new information about the command being durable and we are in the coordination shard in the coordination epoch then update the durability information and possibly cleanup
+ */
+ private Void updateDurability(SafeCommandStore safeStore, SafeCommand safeCommand)
+ {
+ // TODO (expected): Infer durability status from cleanup/truncation
+ if (!durability.isDurable() || homeKey == null)
return null;
- Timestamp executeAt = saveStatus.known.executeAt.hasDecidedExecuteAt() ? this.executeAt : null;
- Commands.setDurability(safeStore, safeCommand, durability, route, executeAt);
+ Commands.setDurability(safeStore, safeCommand, durability, route, committedExecuteAt);
return null;
}
@@ -306,11 +445,12 @@ public class Propagate implements MapReduceConsume<SafeCommandStore, Void>, Epoc
@Override
public MessageType type()
{
- switch (achieved.propagate())
+ // TODO (now): this logic doesn't work now we permit upgrading; need to pick the maximum *possible* status we might propagate
+ switch (achieved.propagatesStatus())
{
case Applied:
case PreApplied:
- if (toEpoch >= executeAt.epoch())
+ if (toEpoch >= committedExecuteAt.epoch())
return MessageType.PROPAGATE_APPLY_MSG;
case Committed:
case ReadyToExecute:
@@ -359,13 +499,14 @@ public class Propagate implements MapReduceConsume<SafeCommandStore, Void>, Epoc
public String toString()
{
return "Propagate{type:" + type() +
- ", txnId: " + txnId +
- ", saveStatus: " + saveStatus +
- ", deps: " + partialDeps +
- ", txn: " + partialTxn +
- ", executeAt: " + executeAt +
- ", writes:" + writes +
- ", result:" + result +
- '}';
+ ", txnId: " + txnId +
+ ", saveStatus: " + maxKnowledgeSaveStatus +
+ ", deps: " + committedDeps +
+ ", txn: " + partialTxn +
+ ", executeAt: " + committedExecuteAt +
+ ", writes:" + writes +
+ ", result:" + result +
+ '}';
}
+
}
diff --git a/accord-core/src/main/java/accord/messages/ReadData.java b/accord-core/src/main/java/accord/messages/ReadData.java
index 889b7067..b2c73948 100644
--- a/accord-core/src/main/java/accord/messages/ReadData.java
+++ b/accord-core/src/main/java/accord/messages/ReadData.java
@@ -182,20 +182,19 @@ public abstract class ReadData extends AbstractEpochRequest<ReadNack>
ack(unavailable);
}
- protected AsyncChain<Data> execute(SafeCommandStore safeStore, Timestamp executeAt, PartialTxn txn)
+ protected AsyncChain<Data> execute(SafeCommandStore safeStore, Timestamp executeAt, PartialTxn txn, Ranges unavailable)
{
- return txn.read(safeStore, executeAt);
+ return txn.read(safeStore, executeAt, unavailable);
}
void read(SafeCommandStore safeStore, Timestamp executeAt, PartialTxn txn)
{
CommandStore unsafeStore = safeStore.commandStore();
Ranges unavailable = safeStore.ranges().unsafeToReadAt(executeAt);
-
- execute(safeStore, executeAt, txn).begin((next, throwable) -> {
+ // TODO (required): do we need to check unavailable again on completion, or throughout execution?
+ execute(safeStore, executeAt, txn, unavailable).begin((next, throwable) -> {
if (throwable != null)
{
- // TODO (expected, exceptions): should send exception to client, and consistency handle/propagate locally
logger.trace("{}: read failed for {}: {}", txnId, unsafeStore, throwable);
node.reply(replyTo, replyContext, null, throwable);
cancel();
diff --git a/accord-core/src/main/java/accord/primitives/AbstractKeys.java b/accord-core/src/main/java/accord/primitives/AbstractKeys.java
index 8f3fcddf..d14acea4 100644
--- a/accord-core/src/main/java/accord/primitives/AbstractKeys.java
+++ b/accord-core/src/main/java/accord/primitives/AbstractKeys.java
@@ -28,6 +28,7 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import accord.api.RoutingKey;
+import accord.utils.ArrayBuffers.ObjectBuffers;
import accord.utils.IndexedFold;
import accord.utils.IndexedFoldToLong;
import accord.utils.Invariants;
@@ -183,11 +184,21 @@ public abstract class AbstractKeys<K extends RoutableKey> implements Iterable<K>
return SortedArrays.sliceWithMultipleMatches(keys, ranges.ranges, factory, (k, r) -> -r.compareTo(k), Range::compareTo);
}
- protected K[] subtract(Ranges ranges, IntFunction<K[]> factory)
+ protected K[] subtract(AbstractRanges ranges, IntFunction<K[]> factory)
{
return SortedArrays.subtractWithMultipleMatches(keys, ranges.ranges, factory, (k, r) -> -r.compareTo(k), Range::compareTo);
}
+ protected K[] intersect(AbstractKeys<K> that, ObjectBuffers<K> buffers)
+ {
+ return SortedArrays.linearIntersection(this.keys, that.keys, buffers);
+ }
+
+ protected K[] intersect(AbstractRanges ranges, ObjectBuffers<K> buffers)
+ {
+ return SortedArrays.intersectWithMultipleMatches(keys, keys.length, ranges.ranges, ranges.ranges.length, (k, r) -> -r.compareTo(k), buffers);
+ }
+
public boolean any(Ranges ranges, Predicate<? super K> predicate)
{
return 1 == foldl(ranges, (key, p2, prev, index) -> predicate.test(key) ? 1 : 0, 0, 0, 1);
diff --git a/accord-core/src/main/java/accord/primitives/AbstractRanges.java b/accord-core/src/main/java/accord/primitives/AbstractRanges.java
index 751f79b9..35ecc569 100644
--- a/accord-core/src/main/java/accord/primitives/AbstractRanges.java
+++ b/accord-core/src/main/java/accord/primitives/AbstractRanges.java
@@ -157,6 +157,30 @@ public abstract class AbstractRanges implements Iterable<Range>, Routables<Range
return indexOf(that) >= 0;
}
+ public boolean intersects(Routable key)
+ {
+ switch (key.domain())
+ {
+ default: throw new AssertionError();
+ case Range: return intersects((Range)key);
+ case Key: return contains((RoutableKey) key);
+ }
+ }
+
+ public Ranges subtract(Unseekables<?> keysOrRanges)
+ {
+ if (keysOrRanges.domain() == Routable.Domain.Key)
+ keysOrRanges = ((AbstractUnseekableKeys)keysOrRanges).toRanges();
+ return subtract((AbstractRanges) keysOrRanges);
+ }
+
+ public Ranges intersect(Unseekables<?> keysOrRanges)
+ {
+ if (keysOrRanges.domain() == Routable.Domain.Key)
+ keysOrRanges = ((AbstractUnseekableKeys)keysOrRanges).toRanges();
+ return sliceMinimal((AbstractRanges)keysOrRanges, this, (AbstractRanges)keysOrRanges, (i1, i2, rs) -> i2.ranges == rs && i2 instanceof Ranges ? (Ranges)i2 : Ranges.ofSortedAndDeoverlapped(rs));
+ }
+
// returns ri in low 32 bits, ki in top, or -1 if no match found
@Override
public final long findNextIntersection(int ri, AbstractKeys<?> keys, int ki)
@@ -193,6 +217,14 @@ public abstract class AbstractRanges implements Iterable<Range>, Routables<Range
* Subtracts the given set of ranges from this
*/
public Ranges subtract(Ranges that)
+ {
+ return subtract((AbstractRanges) that);
+ }
+
+ /**
+ * Subtracts the given set of ranges from this
+ */
+ private Ranges subtract(AbstractRanges that)
{
if (that.isEmpty())
return this instanceof Ranges ? (Ranges)this : ofSortedAndDeoverlappedUnchecked(ranges);
@@ -272,7 +304,7 @@ public abstract class AbstractRanges implements Iterable<Range>, Routables<Range
case Key:
{
AbstractKeys<?> that = (AbstractKeys<?>) keysOrRanges;
- Range[] result = SortedArrays.asymmetricLinearIntersectionWithOverlaps(input.ranges, input.ranges.length, that.keys, that.keys.length, Range::compareTo, cachedRanges());
+ Range[] result = SortedArrays.intersectWithMultipleMatches(input.ranges, input.ranges.length, that.keys, that.keys.length, Range::compareTo, cachedRanges());
return result == input.ranges ? input : constructor.construct(input, param, result);
}
}
@@ -296,7 +328,7 @@ public abstract class AbstractRanges implements Iterable<Range>, Routables<Range
static <C extends AbstractRanges, P, O> O sliceOverlapping(C covering, AbstractRanges input, P param, SliceConstructor<? super C, P, O> constructor)
{
- Range[] result = SortedArrays.asymmetricLinearIntersectionWithOverlaps(input.ranges, input.ranges.length, covering.ranges, covering.ranges.length, Range::compareIntersecting, cachedRanges());
+ Range[] result = SortedArrays.intersectWithMultipleMatches(input.ranges, input.ranges.length, covering.ranges, covering.ranges.length, Range::compareIntersecting, cachedRanges());
return constructor.construct(covering, param, result);
}
diff --git a/accord-core/src/main/java/accord/primitives/AbstractUnseekableKeys.java b/accord-core/src/main/java/accord/primitives/AbstractUnseekableKeys.java
index 80903cd3..0b1d1b31 100644
--- a/accord-core/src/main/java/accord/primitives/AbstractUnseekableKeys.java
+++ b/accord-core/src/main/java/accord/primitives/AbstractUnseekableKeys.java
@@ -19,6 +19,8 @@
package accord.primitives;
import accord.api.RoutingKey;
+import accord.utils.ArrayBuffers;
+import accord.utils.SortedArrays;
import java.util.Arrays;
@@ -38,9 +40,69 @@ implements Iterable<RoutingKey>, Unseekables<RoutingKey>, Participants<RoutingKe
}
@Override
- public Unseekables<RoutingKey> subtract(Ranges ranges)
+ public final AbstractUnseekableKeys intersect(Unseekables<?> keysOrRanges)
+ {
+ switch (keysOrRanges.domain())
+ {
+ default: throw new AssertionError("Unhandled domain: " + keysOrRanges.domain());
+ case Key:
+ {
+ AbstractUnseekableKeys that = (AbstractUnseekableKeys) keysOrRanges;
+ return weakWrap(intersect(that, ArrayBuffers.cachedRoutingKeys()), that);
+ }
+ case Range:
+ {
+ AbstractRanges that = (AbstractRanges) keysOrRanges;
+ return wrap(intersect(that, ArrayBuffers.cachedRoutingKeys()));
+ }
+ }
+ }
+
+ @Override
+ public final Participants<RoutingKey> subtract(Unseekables<?> keysOrRanges)
+ {
+ switch (keysOrRanges.domain())
+ {
+ default: throw new AssertionError("Unhandled domain: " + keysOrRanges.domain());
+ case Key:
+ {
+ AbstractUnseekableKeys that = (AbstractUnseekableKeys) keysOrRanges;
+ return weakWrap(SortedArrays.linearSubtract(this.keys, that.keys, RoutingKey[]::new), that);
+ }
+ case Range:
+ {
+ return subtract((AbstractRanges)keysOrRanges);
+ }
+ }
+ }
+
+ @Override
+ public Participants<RoutingKey> subtract(Ranges ranges)
+ {
+ return subtract((AbstractRanges) ranges);
+ }
+
+ private Participants<RoutingKey> subtract(AbstractRanges ranges)
{
RoutingKey[] output = subtract(ranges, RoutingKey[]::new);
return output == keys ? this : new RoutingKeys(output);
}
+
+ public Ranges toRanges()
+ {
+ Range[] ranges = new Range[keys.length];
+ for (int i = 0 ; i < keys.length ; ++i)
+ ranges[i] = keys[i].asRange();
+ return Ranges.ofSortedAndDeoverlapped(ranges);
+ }
+
+ private AbstractUnseekableKeys weakWrap(RoutingKey[] wrap, AbstractUnseekableKeys that)
+ {
+ return wrap == keys ? this : wrap == that.keys ? that : new RoutingKeys(wrap);
+ }
+
+ private AbstractUnseekableKeys wrap(RoutingKey[] wrap)
+ {
+ return wrap == keys ? this : new RoutingKeys(wrap);
+ }
}
diff --git a/accord-core/src/main/java/accord/primitives/Deps.java b/accord-core/src/main/java/accord/primitives/Deps.java
index 69476ed2..d8e134d0 100644
--- a/accord-core/src/main/java/accord/primitives/Deps.java
+++ b/accord-core/src/main/java/accord/primitives/Deps.java
@@ -109,6 +109,11 @@ public class Deps
return keyDeps.contains(txnId) || rangeDeps.contains(txnId);
}
+ public boolean intersects(TxnId txnId, Ranges ranges)
+ {
+ return keyDeps.intersects(txnId, ranges) || rangeDeps.intersects(txnId, ranges);
+ }
+
public Deps with(Deps that)
{
return new Deps(this.keyDeps.with(that.keyDeps), this.rangeDeps.with(that.rangeDeps));
diff --git a/accord-core/src/main/java/accord/primitives/EpochSupplier.java b/accord-core/src/main/java/accord/primitives/EpochSupplier.java
index 2d43e7a3..d6ad196e 100644
--- a/accord-core/src/main/java/accord/primitives/EpochSupplier.java
+++ b/accord-core/src/main/java/accord/primitives/EpochSupplier.java
@@ -21,4 +21,9 @@ package accord.primitives;
public interface EpochSupplier
{
long epoch();
+
+ static EpochSupplier constant(long epoch)
+ {
+ return () -> epoch;
+ }
}
diff --git a/accord-core/src/main/java/accord/primitives/KeyDeps.java b/accord-core/src/main/java/accord/primitives/KeyDeps.java
index 3b92b941..dae6729c 100644
--- a/accord-core/src/main/java/accord/primitives/KeyDeps.java
+++ b/accord-core/src/main/java/accord/primitives/KeyDeps.java
@@ -237,6 +237,30 @@ public class KeyDeps implements Iterable<Map.Entry<Key, TxnId>>
return Arrays.binarySearch(txnIds, txnId) >= 0;
}
+ public boolean intersects(TxnId txnId, Ranges ranges)
+ {
+ int txnIdx = Arrays.binarySearch(txnIds, txnId);
+ if (txnIdx < 0)
+ return false;
+
+ int[] txnIdsToKeys = txnIdsToKeys();
+
+ int start = txnIdx == 0 ? txnIds.length : txnIdsToKeys[txnIdx - 1];
+ int end = txnIdsToKeys[txnIdx];
+ if (start == end)
+ return false;
+
+ int li = start, ri = 0;
+ while (li < end && ri < ranges.size())
+ {
+ ri = ranges.findNext(ri, keys.get(txnIdsToKeys[li]), FAST);
+ if (ri >= 0) return true;
+ ri = -1 - ri;
+ ++li;
+ }
+ return false;
+ }
+
// return true iff we map any keys to any txnId
// if the mapping is empty we return false, whether or not we have any keys or txnId by themselves
public boolean isEmpty()
diff --git a/accord-core/src/main/java/accord/primitives/KeyRoute.java b/accord-core/src/main/java/accord/primitives/KeyRoute.java
index 4373f72c..6d413b81 100644
--- a/accord-core/src/main/java/accord/primitives/KeyRoute.java
+++ b/accord-core/src/main/java/accord/primitives/KeyRoute.java
@@ -96,8 +96,8 @@ public abstract class KeyRoute extends AbstractUnseekableKeys implements Route<R
public Participants<RoutingKey> participants(Ranges ranges)
{
RoutingKey[] keys = slice(ranges, RoutingKey[]::new);
- if (keys == this.keys && !isParticipatingHomeKey)
- return this;
+ if (isParticipatingHomeKey)
+ return keys == this.keys ? this : new RoutingKeys(keys);
int removePos = Arrays.binarySearch(keys, homeKey);
if (removePos < 0)
@@ -110,6 +110,18 @@ public abstract class KeyRoute extends AbstractUnseekableKeys implements Route<R
return new RoutingKeys(result);
}
+ @Override
+ public Participants<RoutingKey> participants(Ranges ranges, Slice slice)
+ {
+ return participants(ranges);
+ }
+
+ public Ranges toRanges()
+ {
+ Invariants.checkState(isParticipatingHomeKey);
+ return super.toRanges();
+ }
+
@Override
public RoutingKey homeKey()
{
diff --git a/accord-core/src/main/java/accord/primitives/Keys.java b/accord-core/src/main/java/accord/primitives/Keys.java
index ee358d66..74db63a4 100644
--- a/accord-core/src/main/java/accord/primitives/Keys.java
+++ b/accord-core/src/main/java/accord/primitives/Keys.java
@@ -84,6 +84,11 @@ public class Keys extends AbstractKeys<Key> implements Seekables<Key, Keys>
return wrap(slice(ranges, Key[]::new));
}
+ public final Keys intersect(Keys that)
+ {
+ return wrap(SortedArrays.linearIntersection(this.keys, that.keys, ArrayBuffers.cachedKeys()), that);
+ }
+
public Keys with(Key key)
{
int insertPos = Arrays.binarySearch(keys, key);
@@ -218,6 +223,11 @@ public class Keys extends AbstractKeys<Key> implements Seekables<Key, Keys>
return wrap == keys ? this : wrap == that.keys && that instanceof Keys ? (Keys)that : new Keys(wrap);
}
+ private AbstractKeys<Key> weakWrap(Key[] wrap, AbstractKeys<Key> that)
+ {
+ return wrap == keys ? this : wrap == that.keys ? that : new Keys(wrap);
+ }
+
private Keys wrap(Key[] wrap)
{
return wrap == keys ? this : new Keys(wrap);
diff --git a/accord-core/src/main/java/accord/primitives/PartialDeps.java b/accord-core/src/main/java/accord/primitives/PartialDeps.java
index f3770062..7a9b09f8 100644
--- a/accord-core/src/main/java/accord/primitives/PartialDeps.java
+++ b/accord-core/src/main/java/accord/primitives/PartialDeps.java
@@ -66,6 +66,13 @@ public class PartialDeps extends Deps
&& rangeDeps.txnIds().containsAll(that.rangeDeps.txnIds());
}
+ public Deps with(Deps that)
+ {
+ if (that instanceof PartialDeps)
+ return with((PartialDeps) that);
+ return super.with(that);
+ }
+
public PartialDeps with(PartialDeps that)
{
Invariants.checkArgument((this.rangeDeps == null) == (that.rangeDeps == null));
diff --git a/accord-core/src/main/java/accord/primitives/Participants.java b/accord-core/src/main/java/accord/primitives/Participants.java
index e7b6ed7b..1f84ae3b 100644
--- a/accord-core/src/main/java/accord/primitives/Participants.java
+++ b/accord-core/src/main/java/accord/primitives/Participants.java
@@ -26,11 +26,52 @@ package accord.primitives;
*/
public interface Participants<K extends Unseekable> extends Unseekables<K>
{
+ Participants<K> with(Participants<K> with);
+
+ @Override
+ Participants<K> intersect(Unseekables<?> with);
@Override
Participants<K> slice(Ranges ranges);
@Override
Participants<K> slice(Ranges ranges, Slice slice);
- Participants<K> with(Participants<K> with);
+ @Override
+ Participants<K> subtract(Ranges ranges);
+ Participants<K> subtract(Unseekables<?> without);
+
+ Ranges toRanges();
+
+ static boolean isParticipants(Unseekables<?> unseekables)
+ {
+ switch (unseekables.kind())
+ {
+ default: throw new AssertionError("Unhandled Unseekables.Kind: " + unseekables.kind());
+ case FullKeyRoute:
+ case FullRangeRoute:
+ case PartialKeyRoute:
+ case PartialRangeRoute:
+ return Route.castToRoute(unseekables).isParticipatingHomeKey();
+ case RoutingRanges:
+ case RoutingKeys:
+ return true;
+ }
+ }
+
+ static Participants<?> participants(Unseekables<?> unseekables)
+ {
+ switch (unseekables.kind())
+ {
+ default: throw new AssertionError("Unhandled Unseekables.Kind: " + unseekables.kind());
+ case FullKeyRoute:
+ case FullRangeRoute:
+ case PartialKeyRoute:
+ case PartialRangeRoute:
+ return Route.castToRoute(unseekables).participants();
+ case RoutingRanges:
+ return (Ranges) unseekables;
+ case RoutingKeys:
+ return (RoutingKeys) unseekables;
+ }
+ }
/**
* If both left and right are a Route, invoke {@link Route#union} on them. Otherwise invoke {@link #with}.
diff --git a/accord-core/src/main/java/accord/primitives/ProgressToken.java b/accord-core/src/main/java/accord/primitives/ProgressToken.java
index d80ac22b..d1dd50a7 100644
--- a/accord-core/src/main/java/accord/primitives/ProgressToken.java
+++ b/accord-core/src/main/java/accord/primitives/ProgressToken.java
@@ -31,10 +31,10 @@ import javax.annotation.Nonnull;
public class ProgressToken implements Comparable<ProgressToken>, Outcome
{
public static final ProgressToken NONE = new ProgressToken(Durability.NotDurable, Status.NotDefined, Ballot.ZERO, false);
- public static final ProgressToken INVALIDATED = new ProgressToken(Durability.DurableOrInvalidated, Status.Invalidated, Ballot.ZERO, false);
+ public static final ProgressToken INVALIDATED = new ProgressToken(Durability.MajorityOrInvalidated, Status.Invalidated, Ballot.ZERO, false);
public static final ProgressToken APPLIED = new ProgressToken(Durability.NotDurable, Status.PreApplied, Ballot.ZERO, false);
public static final ProgressToken DURABLE = new ProgressToken(Durability.Majority, Status.PreApplied, Ballot.ZERO, false);
- public static final ProgressToken TRUNCATED = new ProgressToken(Durability.DurableOrInvalidated, Status.Truncated, Ballot.ZERO, false);
+ public static final ProgressToken TRUNCATED = new ProgressToken(Durability.MajorityOrInvalidated, Status.Truncated, Ballot.ZERO, false);
public final Durability durability;
public final Status status;
diff --git a/accord-core/src/main/java/accord/primitives/Range.java b/accord-core/src/main/java/accord/primitives/Range.java
index 65d88bdb..fd091676 100644
--- a/accord-core/src/main/java/accord/primitives/Range.java
+++ b/accord-core/src/main/java/accord/primitives/Range.java
@@ -53,6 +53,20 @@ public abstract class Range implements Comparable<RoutableKey>, Unseekable, Seek
return 0;
}
+ @Override
+ public int compareStartTo(RoutableKey key)
+ {
+ int c = start().compareTo(key);
+ if (c == 0) c = 1;
+ return c;
+ }
+
+ @Override
+ public int compareEndTo(RoutableKey key)
+ {
+ return end().compareTo(key);
+ }
+
@Override
public boolean startInclusive()
{
@@ -101,6 +115,20 @@ public abstract class Range implements Comparable<RoutableKey>, Unseekable, Seek
return false;
}
+ @Override
+ public int compareStartTo(RoutableKey key)
+ {
+ return start().compareTo(key);
+ }
+
+ @Override
+ public int compareEndTo(RoutableKey key)
+ {
+ int c = end().compareTo(key);
+ if (c == 0) c = -1;
+ return c;
+ }
+
@Override
public Range newRange(RoutingKey start, RoutingKey end)
{
@@ -155,6 +183,22 @@ public abstract class Range implements Comparable<RoutableKey>, Unseekable, Seek
}
return 0;
}
+
+ @Override
+ public int compareStartTo(RoutableKey key)
+ {
+ int c = start().compareTo(key);
+ if (c == 0 && !startInclusive) c = 1;
+ return c;
+ }
+
+ @Override
+ public int compareEndTo(RoutableKey key)
+ {
+ int c = end().compareTo(key);
+ if (c == 0 && !endInclusive) c = -1;
+ return c;
+ }
};
}
@@ -223,6 +267,22 @@ public abstract class Range implements Comparable<RoutableKey>, Unseekable, Seek
@Override
public abstract int compareTo(RoutableKey key);
+ /**
+ * Returns a negative integer, zero, or a positive integer as the provided key is greater than, equal to,
+ * or less than the start of this range. This comparison is informed by the inclusivity of the start, i.e.
+ * if the raw keys are equal but the start is exclusive, the start is considered to sort after the provided key,
+ * so that it falls outside of the range.
+ */
+ public abstract int compareStartTo(RoutableKey key);
+
+ /**
+ * Returns a negative integer, zero, or a positive integer as the provided key is greater than, contained by,
+ * or less than the end of this range. This comparison is informed by the inclusivity of the end, i.e.
+ * if the raw keys are equal but the end is exclusive, the end is considered to sort before the provided key,
+ * so that it falls outside of the range.
+ */
+ public abstract int compareEndTo(RoutableKey key);
+
public boolean contains(RoutableKey key)
{
return compareTo(key) == 0;
diff --git a/accord-core/src/main/java/accord/primitives/RangeDeps.java b/accord-core/src/main/java/accord/primitives/RangeDeps.java
index 1deb644c..cbff7aa3 100644
--- a/accord-core/src/main/java/accord/primitives/RangeDeps.java
+++ b/accord-core/src/main/java/accord/primitives/RangeDeps.java
@@ -28,6 +28,7 @@ import net.nicoulaj.compilecommand.annotations.Inline;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.util.*;
+import java.util.function.BiConsumer;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.Predicate;
@@ -35,12 +36,15 @@ import java.util.function.Predicate;
import static accord.utils.ArrayBuffers.*;
import static accord.utils.RelationMultiMap.*;
import static accord.utils.SortedArrays.Search.CEIL;
+import static accord.utils.SortedArrays.Search.FAST;
/**
* <p>Maintains a lazily-constructed, bidirectional map between Range and TxnId.
* <p>Ranges are stored sorted by start then end, and indexed by a secondary {@link SearchableRangeList} structure.
* <p>The relationship between Range and TxnId is maintained via {@code int[]} utilising {@link RelationMultiMap}
* functionality.
+ *
+ * TODO (expected): de-overlap ranges per txnId if possible cheaply, or else reduce use of partial ranges where possible
*/
public class RangeDeps implements Iterable<Map.Entry<Range, TxnId>>
{
@@ -123,98 +127,103 @@ public class RangeDeps implements Iterable<Map.Entry<Range, TxnId>>
}
@Inline
- public <P1, P2, P3> int forEach(RoutableKey key, IndexedTriConsumer<P1, P2, P3> forEachScanOrCheckpoint, IndexedRangeTriConsumer<P1, P2, P3> forEachRange, P1 p1, P2 p2, P3 p3, int minIndex)
+ public <P1, P2, P3, P4> int forEach(RoutableKey key, IndexedQuadConsumer<P1, P2, P3, P4> forEachScanOrCheckpoint, IndexedRangeQuadConsumer<P1, P2, P3, P4> forEachRange, P1 p1, P2 p2, P3 p3, P4 p4, int minIndex)
{
- return ensureSearchable().forEach(key, forEachScanOrCheckpoint, forEachRange, p1, p2, p3, minIndex);
+ return ensureSearchable().forEach(key, forEachScanOrCheckpoint, forEachRange, p1, p2, p3, p4, minIndex);
}
- private int forEach(RoutableKey key, Consumer<TxnId> forEach, int minIndex, @Nullable BitSet visited)
+ private <P1> int forEach(RoutableKey key, BiConsumer<P1, TxnId> forEach, P1 p1, int minIndex, @Nullable BitSet visited)
{
return forEach(key, RangeDeps::visitTxnIdsForRangeIndex, RangeDeps::visitTxnIdsForRangeIndex,
- this, forEach, visited, minIndex);
+ this, forEach, p1, visited, minIndex);
}
private <P> int forEach(RoutableKey key, IndexedConsumer<P> forEach, P param, int minIndex)
+ {
+ return forEach(key, IndexedConsumer::accept, forEach, param, minIndex);
+ }
+
+ private <P1, P2> int forEach(RoutableKey key, IndexedBiConsumer<P1, P2> forEach, P1 p1, P2 p2, int minIndex)
{
return forEach(key, RangeDeps::visitTxnIdxsForRangeIndex, RangeDeps::visitTxnIdxsForRangeIndex,
- this, forEach, param, minIndex);
+ this, forEach, p1, p2, minIndex);
}
@Inline
- public <P1, P2, P3> int forEach(Range range, IndexedTriConsumer<P1, P2, P3> forEachScanOrCheckpoint, IndexedRangeTriConsumer<P1, P2, P3> forEachRange, P1 p1, P2 p2, P3 p3, int minIndex)
+ public <P1, P2, P3, P4> int forEach(Range range, IndexedQuadConsumer<P1, P2, P3, P4> forEachScanOrCheckpoint, IndexedRangeQuadConsumer<P1, P2, P3, P4> forEachRange, P1 p1, P2 p2, P3 p3, P4 p4, int minIndex)
{
- return ensureSearchable().forEach(range, forEachScanOrCheckpoint, forEachRange, p1, p2, p3, minIndex);
+ return ensureSearchable().forEach(range, forEachScanOrCheckpoint, forEachRange, p1, p2, p3, p4, minIndex);
}
@Inline
- public <P1, P2, P3> int forEach(RoutingKey start, RoutingKey end, IndexedTriConsumer<P1, P2, P3> forEachScanOrCheckpoint, IndexedRangeTriConsumer<P1, P2, P3> forEachRange, P1 p1, P2 p2, P3 p3, int minIndex)
+ public <P1, P2, P3, P4> int forEach(RoutingKey start, RoutingKey end, IndexedQuadConsumer<P1, P2, P3, P4> forEachScanOrCheckpoint, IndexedRangeQuadConsumer<P1, P2, P3, P4> forEachRange, P1 p1, P2 p2, P3 p3, P4 p4, int minIndex)
{
- return ensureSearchable().forEach(start, end, forEachScanOrCheckpoint, forEachRange, p1, p2, p3, minIndex);
+ return ensureSearchable().forEach(start, end, forEachScanOrCheckpoint, forEachRange, p1, p2, p3, p4, minIndex);
}
- private <P1, P2, P3> void forEach(Ranges ranges, IndexedTriConsumer<P1, P2, P3> forEachScanOrCheckpoint, IndexedRangeTriConsumer<P1, P2, P3> forEachRange, P1 p1, P2 p2, P3 p3)
+ private <P1, P2, P3, P4> void forEach(Ranges ranges, IndexedQuadConsumer<P1, P2, P3, P4> forEachScanOrCheckpoint, IndexedRangeQuadConsumer<P1, P2, P3, P4> forEachRange, P1 p1, P2 p2, P3 p3, P4 p4)
{
int minIndex = 0;
for (int i = 0; i < ranges.size() ; ++i)
- minIndex = forEach(ranges.get(i), forEachScanOrCheckpoint, forEachRange, p1, p2, p3, minIndex);
+ minIndex = forEach(ranges.get(i), forEachScanOrCheckpoint, forEachRange, p1, p2, p3, p4, minIndex);
}
- private int forEach(Range range, Consumer<TxnId> forEach, int minIndex, @Nullable BitSet visited)
+ private <P1> int forEach(Range range, BiConsumer<P1, TxnId> forEach, P1 p1, int minIndex, @Nullable BitSet visited)
{
return forEach(range, RangeDeps::visitTxnIdsForRangeIndex, RangeDeps::visitTxnIdsForRangeIndex,
- this, forEach, visited, minIndex);
+ this, forEach, p1, visited, minIndex);
}
- public <P> int forEach(Range range, IndexedConsumer<P> forEach, P param, int minIndex)
+ public <P1, P2> int forEach(Range range, IndexedBiConsumer<P1, P2> forEach, P1 p1, P2 p2, int minIndex)
{
return forEach(range, RangeDeps::visitTxnIdxsForRangeIndex, RangeDeps::visitTxnIdxsForRangeIndex,
- this, forEach, param, minIndex);
+ this, forEach, p1, p2, minIndex);
}
- public <P> int forEach(RoutingKey start, RoutingKey end, IndexedConsumer<P> forEach, P param, int minIndex)
+ public <P1, P2> int forEach(RoutingKey start, RoutingKey end, IndexedBiConsumer<P1, P2> forEach, P1 p1, P2 p2, int minIndex)
{
return forEach(start, end, RangeDeps::visitTxnIdxsForRangeIndex, RangeDeps::visitTxnIdxsForRangeIndex,
- this, forEach, param, minIndex);
+ this, forEach, p1, p2, minIndex);
}
- private void visitTxnIdsForRangeIndex(Consumer<TxnId> forEach, @Nullable BitSet visited, int rangeIndex)
+ private <P1> void visitTxnIdsForRangeIndex(BiConsumer<P1, TxnId> forEach, P1 p1, @Nullable BitSet visited, int rangeIndex)
{
for (int i = startOffset(ranges, rangesToTxnIds, rangeIndex), end = endOffset(rangesToTxnIds, rangeIndex) ; i < end ; ++i)
- visitTxnId(rangesToTxnIds[i], forEach, visited);
+ visitTxnId(rangesToTxnIds[i], forEach, p1, visited);
}
- private void visitTxnIdsForRangeIndex(Consumer<TxnId> forEach, @Nullable BitSet visited, int start, int end)
+ private <P1> void visitTxnIdsForRangeIndex(BiConsumer<P1, TxnId> forEach, P1 p1, @Nullable BitSet visited, int start, int end)
{
if (end == 0)
return;
for (int i = startOffset(ranges, rangesToTxnIds, start) ; i < endOffset(rangesToTxnIds, end - 1) ; ++i)
- visitTxnId(rangesToTxnIds[i], forEach, visited);
+ visitTxnId(rangesToTxnIds[i], forEach, p1, visited);
}
// TODO (low priority, efficiency): ideally we would accept something like a BitHashSet or IntegerTrie
// as O(N) space needed for BitSet here (but with a very low constant multiplier)
- private void visitTxnId(int txnIdx, Consumer<TxnId> forEach, @Nullable BitSet visited)
+ private <P1> void visitTxnId(int txnIdx, BiConsumer<P1, TxnId> forEach, P1 p1, @Nullable BitSet visited)
{
if (visited == null || !visited.get(txnIdx))
{
if (visited != null)
visited.set(txnIdx);
- forEach.accept(txnIds[txnIdx]);
+ forEach.accept(p1, txnIds[txnIdx]);
}
}
- private <P> void visitTxnIdxsForRangeIndex(IndexedConsumer<P> forEach, P param, int rangeIndex)
+ private <P1, P2> void visitTxnIdxsForRangeIndex(IndexedBiConsumer<P1, P2> forEach, P1 p1, P2 p2, int rangeIndex)
{
for (int i = startOffset(ranges, rangesToTxnIds, rangeIndex), end = endOffset(rangesToTxnIds, rangeIndex) ; i < end ; ++i)
- forEach.accept(param, rangesToTxnIds[i]);
+ forEach.accept(p1, p2, rangesToTxnIds[i]);
}
- private <P> void visitTxnIdxsForRangeIndex(IndexedConsumer<P> forEach, P param, int start, int end)
+ private <P1, P2> void visitTxnIdxsForRangeIndex(IndexedBiConsumer<P1, P2> forEach, P1 p1, P2 p2, int start, int end)
{
if (end == 0)
return;
for (int i = startOffset(ranges, rangesToTxnIds, start) ; i < endOffset(rangesToTxnIds, end - 1) ; ++i)
- forEach.accept(param, rangesToTxnIds[i]);
+ forEach.accept(p1, p2, rangesToTxnIds[i]);
}
/**
@@ -222,7 +231,15 @@ public class RangeDeps implements Iterable<Map.Entry<Range, TxnId>>
*/
public void forEachUniqueTxnId(RoutableKey key, Consumer<TxnId> forEach)
{
- forEach(key, forEach, 0, new BitSet());
+ forEachUniqueTxnId(key, forEach, Consumer::accept);
+ }
+
+ /**
+ * Each matching TxnId will be provided precisely once
+ */
+ public <P1> void forEachUniqueTxnId(RoutableKey key, P1 p1, BiConsumer<P1, TxnId> forEach)
+ {
+ forEach(key, forEach, p1, 0, new BitSet());
}
/**
@@ -236,27 +253,35 @@ public class RangeDeps implements Iterable<Map.Entry<Range, TxnId>>
/**
* The same TxnId may be provided as a parameter multiple times
*/
- public <P> void forEach(Range range, IndexedConsumer<P> forEach, P param)
+ public <P1, P2> void forEach(Range range, P1 p1, P2 p2, IndexedBiConsumer<P1, P2> forEach)
{
- forEach(range, forEach, param, 0);
+ forEach(range, forEach, p1, p2, 0);
}
/**
* The same TxnId may be provided as a parameter multiple times
*/
- public void forEach(AbstractRanges ranges, Consumer<TxnId> forEach)
+ public <P1> void forEach(AbstractRanges ranges, P1 p1, BiConsumer<P1, TxnId> forEach)
{
int minIndex = 0;
for (int i = 0; i < ranges.size() ; ++i)
- minIndex = forEach(ranges.get(i), forEach, minIndex, null);
+ minIndex = forEach(ranges.get(i), forEach, p1, minIndex, null);
+ }
+
+ /**
+ * The same TxnId may be provided as a parameter multiple times
+ */
+ public <P1> void forEach(Unseekables<?> unseekables, P1 p1, IndexedConsumer<P1> forEach)
+ {
+ forEach(unseekables, forEach, p1, IndexedConsumer::accept);
}
/**
* The same TxnId may be provided as a parameter multiple times
*/
- public <P> void forEach(Unseekables<?> unseekables, P param, IndexedConsumer<P> forEach)
+ public <P1, P2> void forEach(Unseekables<?> unseekables, P1 p1, P2 p2, IndexedBiConsumer<P1, P2> forEach)
{
- forEach(unseekables, null, 0, unseekables.size(), param, forEach);
+ forEach(unseekables, null, 0, unseekables.size(), p1, p2, forEach);
}
/**
@@ -264,16 +289,16 @@ public class RangeDeps implements Iterable<Map.Entry<Range, TxnId>>
* @param slice is only useful in case unseekables is a Ranges, in which case we only visit the intersection of the slice and the ranges we walk
* for keys it is expected that the caller has already sliced in this manner
*/
- public <P> void forEach(Unseekables<?> unseekables, @Nullable Range slice, int from, int to, P param, IndexedConsumer<P> forEach)
+ public <P1, P2> void forEach(Unseekables<?> unseekables, @Nullable Range slice, int from, int to, P1 p1, P2 p2, IndexedBiConsumer<P1, P2> forEach)
{
switch (unseekables.domain())
{
default: throw new AssertionError("Unknown domain: " + unseekables.domain());
case Key:
- forEach((AbstractKeys<?>) unseekables, from, to, param, forEach);
+ forEach((AbstractKeys<?>) unseekables, from, to, p1, p2, forEach);
break;
case Range:
- forEach((AbstractRanges) unseekables, slice, from, to, param, forEach);
+ forEach((AbstractRanges) unseekables, slice, from, to, p1, p2, forEach);
break;
}
}
@@ -281,33 +306,33 @@ public class RangeDeps implements Iterable<Map.Entry<Range, TxnId>>
/**
* The same TxnId may be provided as a parameter multiple times
*/
- public <P> void forEach(AbstractKeys<?> keys, P param, IndexedConsumer<P> forEach)
+ public <P1, P2> void forEach(AbstractKeys<?> keys, P1 p1, P2 p2, IndexedBiConsumer<P1, P2> forEach)
{
- forEach(keys, 0, keys.size(), param, forEach);
+ forEach(keys, 0, keys.size(), p1, p2, forEach);
}
/**
* The same TxnId may be provided as a parameter multiple times
*/
- public <P> void forEach(AbstractRanges ranges, P param, IndexedConsumer<P> forEach)
+ public <P1, P2> void forEach(AbstractRanges ranges, P1 p1, P2 p2, IndexedBiConsumer<P1, P2> forEach)
{
- forEach(ranges, null, 0, ranges.size(), param, forEach);
+ forEach(ranges, null, 0, ranges.size(), p1, p2, forEach);
}
/**
* The same TxnId may be provided as a parameter multiple times
*/
- public <P> void forEach(AbstractKeys<?> keys, int from, int to, P param, IndexedConsumer<P> forEach)
+ public <P1, P2> void forEach(AbstractKeys<?> keys, int from, int to, P1 p1, P2 p2, IndexedBiConsumer<P1, P2> forEach)
{
int minIndex = 0;
for (int i = from ; i < to ; ++i)
- minIndex = forEach(keys.get(i), forEach, param, minIndex);
+ minIndex = forEach(keys.get(i), forEach, p1, p2, minIndex);
}
/**
* The same TxnId may be provided as a parameter multiple times
*/
- public <P> void forEach(AbstractRanges ranges, @Nullable Range slice, int from, int to, P param, IndexedConsumer<P> forEach)
+ public <P1, P2> void forEach(AbstractRanges ranges, @Nullable Range slice, int from, int to, P1 p1, P2 p2, IndexedBiConsumer<P1, P2> forEach)
{
int minIndex = 0;
for (int i = from; i < to ; ++i)
@@ -322,7 +347,7 @@ public class RangeDeps implements Iterable<Map.Entry<Range, TxnId>>
if (end.compareTo(start) <= 0)
continue;
}
- minIndex = forEach(start, end, forEach, param, minIndex);
+ minIndex = forEach(start, end, forEach, p1, p2, minIndex);
}
}
@@ -331,7 +356,7 @@ public class RangeDeps implements Iterable<Map.Entry<Range, TxnId>>
*/
public void forEachUniqueTxnId(Range range, Consumer<TxnId> forEach)
{
- forEach(range, forEach, 0, new BitSet());
+ forEach(range, Consumer::accept, forEach, 0, new BitSet());
}
/**
@@ -341,10 +366,21 @@ public class RangeDeps implements Iterable<Map.Entry<Range, TxnId>>
* @param forEach function to call on each unique {@link TxnId}
*/
public void forEachUniqueTxnId(AbstractRanges ranges, Consumer<TxnId> forEach)
+ {
+ forEachUniqueTxnId(ranges, forEach, Consumer::accept);
+ }
+
+ /**
+ * Each matching TxnId will be provided precisely once
+ *
+ * @param ranges to match on
+ * @param forEach function to call on each unique {@link TxnId}
+ */
+ public <P1> void forEachUniqueTxnId(AbstractRanges ranges, P1 p1, BiConsumer<P1, TxnId> forEach)
{
int minIndex = 0;
for (int i = 0; i < ranges.size() ; ++i)
- minIndex = forEach(ranges.get(i), forEach, minIndex, new BitSet());
+ minIndex = forEach(ranges.get(i), forEach, p1, minIndex, new BitSet());
}
// return true iff we map any ranges to any txnId
@@ -361,14 +397,19 @@ public class RangeDeps implements Iterable<Map.Entry<Range, TxnId>>
public Ranges ranges(TxnId txnId)
{
- int txnIdIndex = Arrays.binarySearch(txnIds, txnId);
- if (txnIdIndex < 0)
- throw new IllegalArgumentException("Key not found");
+ int txnIdx = Arrays.binarySearch(txnIds, txnId);
+ if (txnIdx < 0)
+ return Ranges.EMPTY;
+ return ranges(txnIdx);
+ }
+
+ public Ranges ranges(int txnIdx)
+ {
ensureTxnIdToRange();
- int start = txnIdIndex == 0 ? txnIds.length : txnIdsToRanges[txnIdIndex - 1];
- int end = txnIdsToRanges[txnIdIndex];
+ int start = txnIdx == 0 ? txnIds.length : txnIdsToRanges[txnIdx - 1];
+ int end = txnIdsToRanges[txnIdx];
if (start == end)
return Ranges.EMPTY;
@@ -391,6 +432,70 @@ public class RangeDeps implements Iterable<Map.Entry<Range, TxnId>>
return Ranges.ofSorted(result);
}
+ public boolean intersects(TxnId txnId, Ranges ranges)
+ {
+ int txnIdx = Arrays.binarySearch(txnIds, txnId);
+ if (txnIdx < 0)
+ return false;
+
+ return intersects(txnIdx, ranges);
+ }
+
+ public boolean intersects(int txnIdx, Ranges intersects)
+ {
+ ensureTxnIdToRange();
+
+ int start = txnIdx == 0 ? txnIds.length : txnIdsToRanges[txnIdx - 1];
+ int end = txnIdsToRanges[txnIdx];
+ if (start == end)
+ return false;
+
+ int li = start, ri = 0;
+ while (li < end && ri < intersects.size())
+ {
+ ri = intersects.findNext(ri, ranges[txnIdsToRanges[li]], FAST);
+ if (ri >= 0) return true;
+ ri = -1 - ri;
+ ++li;
+ }
+ return false;
+ }
+
+ public boolean intersects(TxnId txnId, RoutableKey key)
+ {
+ int txnIdx = Arrays.binarySearch(txnIds, txnId);
+ if (txnIdx < 0)
+ throw new IllegalArgumentException("Key not found");
+
+ return intersects(txnIdx, key);
+ }
+
+ public boolean intersects(int txnIdx, RoutableKey key)
+ {
+ ensureTxnIdToRange();
+
+ int start = txnIdx == 0 ? txnIds.length : txnIdsToRanges[txnIdx - 1];
+ int end = txnIdsToRanges[txnIdx];
+ for (int i = start ; i < end ; ++i)
+ {
+ int c = ranges[i].compareTo(key);
+ if (c == 0) return true;
+ if (c > 0) return false;
+ }
+ return false;
+ }
+
+ public <P1, V> V foldEachRange(int txnIdx, P1 p1, V accumulate, TriFunction<P1, Range, V, V> fold)
+ {
+ ensureTxnIdToRange();
+
+ int start = txnIdx == 0 ? txnIds.length : txnIdsToRanges[txnIdx - 1];
+ int end = txnIdsToRanges[txnIdx];
+ for (int i = start; i < end ; ++i)
+ accumulate = fold.apply(p1, ranges[txnIdsToRanges[i++]], accumulate);
+ return accumulate;
+ }
+
void ensureTxnIdToRange()
{
if (txnIdsToRanges != null)
@@ -406,7 +511,7 @@ public class RangeDeps implements Iterable<Map.Entry<Range, TxnId>>
try (RangeAndMapCollector collector = new RangeAndMapCollector(ensureSearchable().maxScanAndCheckpointMatches))
{
- forEach(select, collector, collector, ranges, rangesToTxnIds, null);
+ forEach(select, collector, collector, ranges, rangesToTxnIds, null, null);
if (collector.rangesCount == 0)
return new RangeDeps(NO_RANGES, NO_TXNIDS, NO_INTS);
@@ -577,8 +682,8 @@ public class RangeDeps implements Iterable<Map.Entry<Range, TxnId>>
}
static class RangeCollector implements
- IndexedRangeTriConsumer<Range[], int[], Object>,
- IndexedTriConsumer<Range[], int[], Object>,
+ IndexedRangeQuadConsumer<Range[], int[], Object, Object>,
+ IndexedQuadConsumer<Range[], int[], Object, Object>,
AutoCloseable
{
int[] oooBuffer;
@@ -592,13 +697,13 @@ public class RangeDeps implements Iterable<Map.Entry<Range, TxnId>>
}
@Override
- public void accept(Range[] o, int[] o2, Object o3, int index)
+ public void accept(Range[] o, int[] o2, Object o3, Object o4, int index)
{
oooBuffer[oooCount++] = index;
}
@Override
- public void accept(Range[] ranges, int[] rangesToTxnIds, Object o3, int fromIndex, int toIndex)
+ public void accept(Range[] ranges, int[] rangesToTxnIds, Object o3, Object o4, int fromIndex, int toIndex)
{
if (oooCount > 0)
{
diff --git a/accord-core/src/main/java/accord/primitives/RangeRoute.java b/accord-core/src/main/java/accord/primitives/RangeRoute.java
index b3c905b2..39c41c56 100644
--- a/accord-core/src/main/java/accord/primitives/RangeRoute.java
+++ b/accord-core/src/main/java/accord/primitives/RangeRoute.java
@@ -118,13 +118,18 @@ public abstract class RangeRoute extends AbstractRanges implements Route<Range>,
// TODO (desired): efficiency (lots of unnecessary allocations)
// TODO (expected): this should return a PartialRangeRoute, but we need to remove Route.covering()
- return ranges().subtract(Ranges.of(homeKey().asRange()));
+ return toRanges().subtract(Ranges.of(homeKey().asRange()));
}
@Override
public Participants<Range> participants(Ranges slice)
{
- Range[] ranges = slice(slice, Overlapping, this, null, (i1, i2, rs) -> rs);
+ return participants(slice, Overlapping);
+ }
+
+ public Participants<Range> participants(Ranges slice, Slice kind)
+ {
+ Range[] ranges = slice(slice, kind, this, null, (i1, i2, rs) -> rs);
if (ranges == this.ranges && isParticipatingHomeKey)
return this;
@@ -137,7 +142,7 @@ public abstract class RangeRoute extends AbstractRanges implements Route<Range>,
return result.subtract(Ranges.of(homeKey().asRange()));
}
- public Ranges ranges()
+ public Ranges toRanges()
{
return Ranges.ofSortedAndDeoverlapped(ranges);
}
diff --git a/accord-core/src/main/java/accord/primitives/Ranges.java b/accord-core/src/main/java/accord/primitives/Ranges.java
index 87e66010..5cc07c80 100644
--- a/accord-core/src/main/java/accord/primitives/Ranges.java
+++ b/accord-core/src/main/java/accord/primitives/Ranges.java
@@ -144,6 +144,12 @@ public class Ranges extends AbstractRanges implements Iterable<Range>, Seekables
return this;
}
+ @Override
+ public Ranges toRanges()
+ {
+ return this;
+ }
+
@Override
public FullRangeRoute toRoute(RoutingKey homeKey)
{
diff --git a/accord-core/src/main/java/accord/primitives/Routable.java b/accord-core/src/main/java/accord/primitives/Routable.java
index a4d5270f..1072df68 100644
--- a/accord-core/src/main/java/accord/primitives/Routable.java
+++ b/accord-core/src/main/java/accord/primitives/Routable.java
@@ -46,6 +46,11 @@ public interface Routable
{
return VALUES[ordinal];
}
+
+ public char shortName()
+ {
+ return this == Key ? 'K' : 'R';
+ }
}
Domain domain();
diff --git a/accord-core/src/main/java/accord/primitives/Routables.java b/accord-core/src/main/java/accord/primitives/Routables.java
index b4460b21..64806423 100644
--- a/accord-core/src/main/java/accord/primitives/Routables.java
+++ b/accord-core/src/main/java/accord/primitives/Routables.java
@@ -58,6 +58,7 @@ public interface Routables<K extends Routable> extends Iterable<K>
case Range: return intersects((AbstractRanges) routables);
}
}
+
boolean contains(RoutableKey key);
boolean containsAll(Routables<?> keysOrRanges);
diff --git a/accord-core/src/main/java/accord/primitives/Route.java b/accord-core/src/main/java/accord/primitives/Route.java
index 70fdb745..775c2b3b 100644
--- a/accord-core/src/main/java/accord/primitives/Route.java
+++ b/accord-core/src/main/java/accord/primitives/Route.java
@@ -27,7 +27,7 @@ public interface Route<K extends Unseekable> extends Unseekables<K>
RoutingKey homeKey();
/**
- * @return true iff homeKey() is not involved in the transaction, only in its coordination (i.e. !txn.keys().contains(homeKey())
+ * @return true iff homeKey() is involved in the transaction, not only in its coordination (i.e. txn.keys().contains(homeKey())
*/
boolean isParticipatingHomeKey();
RoutingKey someParticipatingKey();
@@ -68,6 +68,11 @@ public interface Route<K extends Unseekable> extends Unseekables<K>
*/
Participants<K> participants(Ranges ranges);
+ /**
+ * Return the unseekables excluding any coordination-only home key, that intersect the provided ranges
+ */
+ Participants<K> participants(Ranges ranges, Slice slice);
+
default boolean hasParticipants()
{
return size() > (isParticipatingHomeKey() || !contains(homeKey()) ? 0 : 1);
@@ -87,7 +92,30 @@ public interface Route<K extends Unseekable> extends Unseekables<K>
switch (unseekables.domain())
{
- default: throw new AssertionError();
+ default: throw new ClassCastException(unseekables + " is not a FullRoute");
+ case Key: return (FullKeyRoute) unseekables;
+ case Range: return (FullRangeRoute) unseekables;
+ }
+ }
+
+ // this method exists solely to circumvent JDK bug with testing and casting interfaces
+ static FullRoute<?> castToNonNullFullRoute(@Nullable Unseekables<?> unseekables)
+ {
+ FullRoute<?> route = tryCastToFullRoute(unseekables);
+ if (route == null)
+ throw new ClassCastException(unseekables + " is not a FullRoute");
+ return route;
+ }
+
+ // this method exists solely to circumvent JDK bug with testing and casting interfaces
+ static FullRoute<?> tryCastToFullRoute(@Nullable Unseekables<?> unseekables)
+ {
+ if (unseekables == null)
+ return null;
+
+ switch (unseekables.domain())
+ {
+ default: return null;
case Key: return (FullKeyRoute) unseekables;
case Range: return (FullRangeRoute) unseekables;
}
diff --git a/accord-core/src/main/java/accord/primitives/Txn.java b/accord-core/src/main/java/accord/primitives/Txn.java
index 5a792918..85cb0cbe 100644
--- a/accord-core/src/main/java/accord/primitives/Txn.java
+++ b/accord-core/src/main/java/accord/primitives/Txn.java
@@ -32,6 +32,14 @@ import accord.api.Update;
import accord.local.SafeCommandStore;
import accord.utils.async.AsyncChain;
import accord.utils.async.AsyncChains;
+import java.util.function.Predicate;
+
+import static accord.primitives.Txn.Kind.Kinds.Any;
+import static accord.primitives.Txn.Kind.Kinds.AnyGlobal;
+import static accord.primitives.Txn.Kind.Kinds.RorWs;
+import static accord.primitives.Txn.Kind.Kinds.SyncPoints;
+import static accord.primitives.Txn.Kind.Kinds.Ws;
+import static accord.primitives.Txn.Kind.Kinds.WsOrSyncPoint;
public interface Txn
{
@@ -93,6 +101,26 @@ public interface Txn
LocalOnly
;
+ public enum Kinds implements Predicate<Kind>
+ {
+ Ws, RorWs, WsOrSyncPoint, SyncPoints, AnyGlobal, Any;
+
+ @Override
+ public boolean test(Kind kind)
+ {
+ switch (this)
+ {
+ default: throw new AssertionError();
+ case Any: return true;
+ case AnyGlobal: return !kind.isLocal();
+ case WsOrSyncPoint: return kind == Write || kind == Kind.SyncPoint || kind == ExclusiveSyncPoint;
+ case SyncPoints: return kind == Kind.SyncPoint || kind == ExclusiveSyncPoint;
+ case Ws: return kind == Write;
+ case RorWs: return kind == Read || kind == Write;
+ }
+ }
+ }
+
// in future: BlindWrite, Interactive?
private static final Kind[] VALUES = Kind.values();
@@ -138,6 +166,43 @@ public interface Txn
{
return VALUES[ordinal];
}
+
+ public Kinds witnesses()
+ {
+ switch (this)
+ {
+ default: throw new AssertionError();
+ case Read:
+ case NoOp:
+ return Ws;
+ case Write:
+ return RorWs;
+ case SyncPoint:
+ case ExclusiveSyncPoint:
+ return AnyGlobal;
+ }
+ }
+
+ public Kinds witnessedBy()
+ {
+ switch (this)
+ {
+ default: throw new AssertionError();
+ case Read:
+ return WsOrSyncPoint;
+ case Write:
+ return AnyGlobal;
+ case SyncPoint:
+ case ExclusiveSyncPoint:
+ case NoOp:
+ return SyncPoints;
+ }
+ }
+
+ public char shortName()
+ {
+ return toString().charAt(0);
+ }
}
class InMemory implements Txn
@@ -267,10 +332,10 @@ public interface Txn
return new Writes(txnId, executeAt, update.keys(), update.apply(executeAt, data));
}
- default AsyncChain<Data> read(SafeCommandStore safeStore, Timestamp executeAt)
+ default AsyncChain<Data> read(SafeCommandStore safeStore, Timestamp executeAt, Ranges unavailable)
{
- Ranges ranges = safeStore.ranges().safeToReadAt(executeAt);
- List<AsyncChain<Data>> chains = Routables.foldlMinimal(read().keys(), ranges, (key, accumulate, index) -> {
+ Ranges ranges = safeStore.ranges().allAt(executeAt).subtract(unavailable);
+ List<AsyncChain<Data>> chains = Routables.foldlMinimal(keys(), ranges, (key, accumulate, index) -> {
AsyncChain<Data> result = read().read(key, safeStore, executeAt, safeStore.dataStore());
accumulate.add(result);
return accumulate;
diff --git a/accord-core/src/main/java/accord/primitives/TxnId.java b/accord-core/src/main/java/accord/primitives/TxnId.java
index dc2a0986..7e153970 100644
--- a/accord-core/src/main/java/accord/primitives/TxnId.java
+++ b/accord-core/src/main/java/accord/primitives/TxnId.java
@@ -18,12 +18,16 @@
package accord.primitives;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
import accord.local.Node.Id;
import accord.primitives.Routable.Domain;
import accord.primitives.Txn.Kind;
import static accord.primitives.Txn.Kind.Read;
import static accord.primitives.Txn.Kind.Write;
+import static accord.utils.Invariants.illegalArgument;
public class TxnId extends Timestamp
{
@@ -111,6 +115,12 @@ public class TxnId extends Timestamp
return merge(this, that, TxnId::fromBits);
}
+ @Override
+ public String toString()
+ {
+ return "[" + epoch() + ',' + hlc() + ',' + flags() + '(' + domain().shortName() + rw().shortName() + ')' + ',' + node + ']';
+ }
+
private static int flags(Kind rw, Domain domain)
{
return flags(rw) | flags(domain);
@@ -155,4 +165,13 @@ public class TxnId extends Timestamp
{
return new TxnId(epochMsb(epoch), 0, Id.NONE);
}
+
+ private static final Pattern PARSE = Pattern.compile("\\[(?<epoch>[0-9]+),(?<hlc>[0-9]+),(?<flags>[0-9]+),(?<node>[0-9]+)]");
+ public static TxnId parse(String txnIdString)
+ {
+ Matcher m = PARSE.matcher(txnIdString);
+ if (!m.matches())
+ throw illegalArgument("Invalid TxnId string: " + txnIdString);
+ return fromValues(Long.parseLong(m.group("epoch")), Long.parseLong(m.group("hlc")), Integer.parseInt(m.group("flags")), new Id(Integer.parseInt(m.group("node"))));
+ }
}
diff --git a/accord-core/src/main/java/accord/primitives/Unseekables.java b/accord-core/src/main/java/accord/primitives/Unseekables.java
index e373a4d9..51b80e7d 100644
--- a/accord-core/src/main/java/accord/primitives/Unseekables.java
+++ b/accord-core/src/main/java/accord/primitives/Unseekables.java
@@ -47,6 +47,8 @@ public interface Unseekables<K extends Unseekable> extends Iterable<K>, Routable
Unseekables<K> slice(Ranges ranges, Slice slice);
Unseekables<K> subtract(Ranges ranges);
+ Unseekables<K> subtract(Unseekables<?> subtract);
+ Unseekables<K> intersect(Unseekables<?> intersect);
/**
* Return an object containing any {@code K} present in either of the original collections.
diff --git a/accord-core/src/main/java/accord/utils/Invariants.java b/accord-core/src/main/java/accord/utils/Invariants.java
index cd5ba6bd..361c29a8 100644
--- a/accord-core/src/main/java/accord/utils/Invariants.java
+++ b/accord-core/src/main/java/accord/utils/Invariants.java
@@ -39,7 +39,7 @@ public class Invariants
return DEBUG;
}
- private static void illegalState(String msg)
+ public static void illegalState(String msg)
{
throw new IllegalStateException(msg);
}
@@ -49,7 +49,7 @@ public class Invariants
illegalState(null);
}
- private static void illegalArgument(String msg)
+ public static RuntimeException illegalArgument(String msg)
{
throw new IllegalArgumentException(msg);
}
diff --git a/accord-core/src/main/java/accord/utils/RandomSource.java b/accord-core/src/main/java/accord/utils/RandomSource.java
index 3d4861e5..8528acf8 100644
--- a/accord-core/src/main/java/accord/utils/RandomSource.java
+++ b/accord-core/src/main/java/accord/utils/RandomSource.java
@@ -18,16 +18,15 @@
package accord.utils;
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.List;
-import java.util.NavigableSet;
import java.util.Random;
-import java.util.Set;
-import java.util.stream.DoubleStream;
-import java.util.stream.IntStream;
-import java.util.stream.LongStream;
+import java.util.function.BooleanSupplier;
+import java.util.function.IntSupplier;
+import java.util.function.LongSupplier;
+import java.util.function.Supplier;
+import accord.utils.random.Picker;
+
+// TODO (expected): merge with C* RandomSource
public interface RandomSource
{
static RandomSource wrap(Random random)
@@ -38,14 +37,40 @@ public interface RandomSource
void nextBytes(byte[] bytes);
boolean nextBoolean();
+ default BooleanSupplier uniformBools() { return this::nextBoolean; }
+ default BooleanSupplier biasedUniformBools(float chance) { return () -> decide(chance); }
+ default Supplier<BooleanSupplier> biasedUniformBoolsSupplier(float minChance)
+ {
+ return () -> {
+ float chance = minChance + (1 - minChance)*nextFloat();
+ return () -> decide(chance);
+ };
+ }
- int nextInt();
+ /**
+ * Returns true with a probability of {@code chance}. This is logically the same as
+ * <pre>{@code nextFloat() < chance}</pre>
+ *
+ * @param chance cumulative probability in range [0..1]
+ */
+ default boolean decide(float chance)
+ {
+ return nextFloat() < chance;
+ }
- default int nextInt(int maxExclusive)
+ /**
+ * Returns true with a probability of {@code chance}. This is logically the same as
+ * <pre>{@code nextDouble() < chance}</pre>
+ *
+ * @param chance cumulative probability in range [0..1]
+ */
+ default boolean decide(double chance)
{
- return nextInt(0, maxExclusive);
+ return nextDouble() < chance;
}
+ int nextInt();
+ default int nextInt(int maxExclusive) { return nextInt(0, maxExclusive); }
default int nextInt(int minInclusive, int maxExclusive)
{
// this is diff behavior than ThreadLocalRandom, which returns nextInt
@@ -74,29 +99,47 @@ public interface RandomSource
}
return result;
}
-
- default IntStream ints()
+ default int nextBiasedInt(int minInclusive, int median, int maxExclusive)
{
- return IntStream.generate(this::nextInt);
+ checkBiasedUniform(minInclusive, median, maxExclusive);
+
+ int range = Math.max(maxExclusive - median, median - minInclusive) * 2;
+ int next = nextInt(range) - range/2;
+ next += median;
+ return next >= median ? next < maxExclusive ? next : nextInt(median, maxExclusive)
+ : next >= minInclusive ? next : minInclusive == median ? median : nextInt(minInclusive, median);
}
- default IntStream ints(int maxExclusive)
+ default IntSupplier uniformInts(int minInclusive, int maxExclusive) { return () -> nextInt(minInclusive, maxExclusive); }
+ default IntSupplier biasedUniformInts(int minInclusive, int median, int maxExclusive)
{
- return IntStream.generate(() -> nextInt(maxExclusive));
+ checkBiasedUniform(minInclusive, median, maxExclusive);
+ return () -> nextBiasedInt(minInclusive, median, maxExclusive);
}
-
- default IntStream ints(int minInclusive, int maxExclusive)
+ default Supplier<IntSupplier> biasedUniformIntsSupplier(int absoluteMinInclusive, int absoluteMaxExclusive, int minMedian, int maxMedian, int minRange, int maxRange)
{
- return IntStream.generate(() -> nextInt(minInclusive, maxExclusive));
+ return biasedUniformIntsSupplier(absoluteMinInclusive, absoluteMaxExclusive, minMedian, (minMedian+maxMedian)/2, maxMedian, minRange, (minRange+maxRange)/2, maxRange);
}
-
- long nextLong();
-
- default long nextLong(long maxExclusive)
+ default Supplier<IntSupplier> biasedUniformIntsSupplier(int absoluteMinInclusive, int absoluteMaxExclusive, int minMedian, int medianMedian, int maxMedian, int minRange, int medianRange, int maxRange)
{
- return nextLong(0, maxExclusive);
+ checkBiasedUniform(minMedian, medianMedian, maxMedian);
+ checkBiasedUniform(minRange, medianRange, maxRange);
+ if (minMedian < absoluteMinInclusive)
+ throw new IllegalArgumentException(String.format("absoluteMin (%s) should be less than or equal to minMedian (%s)", absoluteMinInclusive, minMedian));
+ if (maxMedian > absoluteMaxExclusive)
+ throw new IllegalArgumentException(String.format("absoluteMax (%s) should be greater than or equal to maxMedian (%s)", absoluteMaxExclusive, maxMedian));
+ if (minRange < 1)
+ throw new IllegalArgumentException(String.format("minRange (%s) should be greater than or equal to 1", minRange));
+ return () -> {
+ int median = nextBiasedInt(minMedian, medianMedian, maxMedian);
+ int minInclusive = Math.max(absoluteMinInclusive, median - nextBiasedInt(minRange, medianRange, maxRange)/2);
+ int maxExclusive = Math.min(absoluteMaxExclusive, median + (nextBiasedInt(minRange, medianRange, maxRange)+1)/2);
+ return biasedUniformInts(minInclusive, median, maxExclusive);
+ };
}
+ long nextLong();
+ default long nextLong(long maxExclusive) { return nextLong(0, maxExclusive); }
default long nextLong(long minInclusive, long maxExclusive)
{
// this is diff behavior than ThreadLocalRandom, which returns nextLong
@@ -125,31 +168,57 @@ public interface RandomSource
}
return result;
}
-
- default LongStream longs()
+ default long nextBiasedLong(long minInclusive, long median, long maxExclusive)
{
- return LongStream.generate(this::nextLong);
+ checkBiasedUniform(minInclusive, median, maxExclusive);
+
+ long range = Math.max(maxExclusive - median, median - minInclusive) * 2;
+ long next = nextLong(range) - range/2;
+ next += median;
+ return next >= median ? next < maxExclusive ? next : nextLong(median, maxExclusive)
+ : next >= minInclusive ? next : minInclusive == median ? median : nextLong(minInclusive, median);
}
- default LongStream longs(long maxExclusive)
+ default LongSupplier uniformLongs(long minInclusive, long maxExclusive) { return () -> nextLong(minInclusive, maxExclusive); }
+ default LongSupplier biasedUniformLongs(long minInclusive, long median, long maxExclusive)
+ {
+ checkBiasedUniform(minInclusive, median, maxExclusive);
+ return () -> nextBiasedLong(minInclusive, median, maxExclusive);
+ }
+ default Supplier<LongSupplier> biasedUniformLongsSupplier(long absoluteMinInclusive, long absoluteMaxExclusive, long minMedian, long maxMedian, long minRange, long maxRange)
+ {
+ return biasedUniformLongsSupplier(absoluteMinInclusive, absoluteMaxExclusive, minMedian, (minMedian+maxMedian)/2, maxRange, minRange, (minRange+maxRange)/2, maxRange);
+ }
+ default Supplier<LongSupplier> biasedUniformLongsSupplier(long absoluteMinInclusive, long absoluteMaxExclusive, long minMedian, long medianMedian, long maxMedian, long minRange, long medianRange, long maxRange)
{
- return LongStream.generate(() -> nextLong(maxExclusive));
+ checkBiasedUniform(minMedian, medianMedian, maxMedian);
+ checkBiasedUniform(minRange, medianRange, maxRange);
+ if (minMedian < absoluteMinInclusive)
+ throw new IllegalArgumentException(String.format("absoluteMin (%s) should be less than or equal to minMedian (%s)", absoluteMinInclusive, minMedian));
+ if (maxMedian > absoluteMaxExclusive)
+ throw new IllegalArgumentException(String.format("absoluteMax (%s) should be greater than or equal to maxMedian (%s)", absoluteMaxExclusive, maxMedian));
+ if (minRange < 1)
+ throw new IllegalArgumentException(String.format("minRange (%s) should be greater than or equal to 1", minRange));
+ return () -> {
+ long median = nextBiasedLong(minMedian, medianMedian, maxMedian);
+ long minInclusive = Math.max(absoluteMinInclusive, median - nextBiasedLong(minRange, medianRange, maxRange)/2);
+ long maxExclusive = Math.min(absoluteMaxExclusive, median + (1+nextBiasedLong(minRange, medianRange, maxRange))/2);
+ return biasedUniformLongs(minInclusive, median, maxExclusive);
+ };
}
- default LongStream longs(long minInclusive, long maxExclusive)
+ static void checkBiasedUniform(long minInclusive, long median, long maxExclusive)
{
- return LongStream.generate(() -> nextLong(minInclusive, maxExclusive));
+ if (minInclusive > median)
+ throw new IllegalArgumentException(String.format("Min (%s) should be equal to or less than median (%d).", minInclusive, median));
+ if (median >= maxExclusive)
+ throw new IllegalArgumentException(String.format("Median (%s) should be less than max (%d).", median, maxExclusive));
}
float nextFloat();
double nextDouble();
-
- default double nextDouble(double maxExclusive)
- {
- return nextDouble(0, maxExclusive);
- }
-
+ default double nextDouble(double maxExclusive) { return nextDouble(0, maxExclusive); }
default double nextDouble(double minInclusive, double maxExclusive)
{
if (minInclusive >= maxExclusive)
@@ -162,136 +231,15 @@ public interface RandomSource
return result;
}
- default DoubleStream doubles()
- {
- return DoubleStream.generate(this::nextDouble);
- }
-
- default DoubleStream doubles(double maxExclusive)
- {
- return DoubleStream.generate(() -> nextDouble(maxExclusive));
- }
-
- default DoubleStream doubles(double minInclusive, double maxExclusive)
- {
- return DoubleStream.generate(() -> nextDouble(minInclusive, maxExclusive));
- }
-
double nextGaussian();
- default int pickInt(int first, int second, int... rest)
- {
- int offset = nextInt(0, rest.length + 2);
- switch (offset)
- {
- case 0: return first;
- case 1: return second;
- default: return rest[offset - 2];
- }
- }
-
- default int pickInt(int[] array)
- {
- return pickInt(array, 0, array.length);
- }
-
- default int pickInt(int[] array, int offset, int length)
- {
- Invariants.checkIndexInBounds(array.length, offset, length);
- if (length == 1)
- return array[offset];
- return array[nextInt(offset, offset + length)];
- }
-
- default long pickLong(long first, long second, long... rest)
- {
- int offset = nextInt(0, rest.length + 2);
- switch (offset)
- {
- case 0: return first;
- case 1: return second;
- default: return rest[offset - 2];
- }
- }
-
- default long pickLong(long[] array)
- {
- return pickLong(array, 0, array.length);
- }
-
- default long pickLong(long[] array, int offset, int length)
- {
- Invariants.checkIndexInBounds(array.length, offset, length);
- if (length == 1)
- return array[offset];
- return array[nextInt(offset, offset + length)];
- }
-
- default <T extends Comparable<T>> T pick(Set<T> set)
- {
- List<T> values = new ArrayList<>(set);
- // Non-ordered sets may have different iteration order on different environments, which would make a seed produce different histories!
- // To avoid such a problem, make sure to apply a deterministic function (sort).
- if (!(set instanceof NavigableSet))
- values.sort(Comparator.naturalOrder());
- return pick(values);
- }
-
- default <T> T pick(T first, T second, T... rest)
- {
- int offset = nextInt(0, rest.length + 2);
- switch (offset)
- {
- case 0: return first;
- case 1: return second;
- default: return rest[offset - 2];
- }
- }
-
- default <T> T pick(T[] array)
- {
- return array[nextInt(array.length)];
- }
-
- default <T> T pick(List<T> values)
- {
- return pick(values, 0, values.size());
- }
-
- default <T> T pick(List<T> values, int offset, int length)
- {
- Invariants.checkIndexInBounds(values.size(), offset, length);
- if (length == 1)
- return values.get(offset);
- return values.get(nextInt(offset, offset + length));
- }
+ default <T> Supplier<T> randomWeightedPicker(T[] objects) { return Picker.WeightedObjectPicker.randomWeighted(this, objects); }
+ default <T> Supplier<T> randomWeightedPicker(T[] objects, float[] bias) { return Picker.WeightedObjectPicker.randomWeighted(this, objects, bias); }
+ default <T> Supplier<T> weightedPicker(T[] objects, float[] proportionalWeights) { return Picker.WeightedObjectPicker.weighted(this, objects, proportionalWeights); }
void setSeed(long seed);
-
RandomSource fork();
- /**
- * Returns true with a probability of {@code chance}. This logic is logically the same as
- * <pre>{@code nextFloat() < chance}</pre>
- *
- * @param chance cumulative probability in range [0..1]
- */
- default boolean decide(float chance)
- {
- return nextFloat() < chance;
- }
-
- /**
- * Returns true with a probability of {@code chance}. This logic is logically the same as
- * <pre>{@code nextDouble() < chance}</pre>
- *
- * @param chance cumulative probability in range [0..1]
- */
- default boolean decide(double chance)
- {
- return nextDouble() < chance;
- }
-
default long reset()
{
long seed = nextLong();
diff --git a/accord-core/src/main/java/accord/utils/ReducingIntervalMap.java b/accord-core/src/main/java/accord/utils/ReducingIntervalMap.java
index c4c0fab4..9c3184c6 100644
--- a/accord-core/src/main/java/accord/utils/ReducingIntervalMap.java
+++ b/accord-core/src/main/java/accord/utils/ReducingIntervalMap.java
@@ -30,11 +30,14 @@ import javax.annotation.Nullable;
import com.google.common.annotations.VisibleForTesting;
/**
- * Mostly copied/adapted from Cassandra's PaxosRepairHistory class
- *
* Represents a map of ranges where precisely one value is bound to each point in the continuum of ranges,
* and a simple function is sufficient to merge values inserted to overlapping ranges.
*
+ * Copied/adapted from Cassandra's PaxosRepairHistory class, however has a major distinction: applies only to the
+ * covered ranges, i.e. the first start bound is the lower bound, and the last start bound is the upper bound,
+ * and everything else is considered unknown. This is in contrast to the C* version where every logical range has
+ * some associated information, with the first and last entries applying to everything either side of the start/end bound.
+ *
* A simple sorted array of bounds is sufficient to represent the state and perform efficient lookups.
*
* TODO (desired): use a mutable b-tree instead
@@ -47,8 +50,8 @@ public class ReducingIntervalMap<K extends Comparable<? super K>, V>
// for simplicity at construction, we permit this to be overridden by the first insertion
final boolean inclusiveEnds;
// starts is 1 longer than values, so that starts[0] == start of values[0]
- final K[] starts;
- final V[] values;
+ protected final K[] starts;
+ protected final V[] values;
public ReducingIntervalMap()
{
@@ -83,6 +86,35 @@ public class ReducingIntervalMap<K extends Comparable<? super K>, V>
return result;
}
+ public <V2> V2 foldl(BiFunction<V, V2, V2> reduce, V2 accumulator, Predicate<V2> terminate)
+ {
+ for (V value : values)
+ {
+ if (value != null)
+ {
+ accumulator = reduce.apply(value, accumulator);
+ if (terminate.test(accumulator))
+ break;
+ }
+ }
+ return accumulator;
+ }
+
+ public <V2> V2 foldlWithBounds(QuadFunction<V, V2, K, K, V2> fold, V2 accumulator, Predicate<V2> terminate)
+ {
+ for (int i = 0 ; i < values.length ; ++i)
+ {
+ V value = values[i];
+ if (value != null)
+ {
+ accumulator = fold.apply(value, accumulator, starts[i], starts[i+1]);
+ if (terminate.test(accumulator))
+ break;
+ }
+ }
+ return accumulator;
+ }
+
public String toString()
{
return toString(v -> true);
@@ -329,6 +361,10 @@ public class ReducingIntervalMap<K extends Comparable<? super K>, V>
return a;
}
+ /**
+ * null is a valid value to represent no knowledge, and is the *expected* final value, representing
+ * the bound of our knowledge (any higher key will find no associated information)
+ */
public void append(K start, @Nullable V value, BiFunction<V, V, V> reduce)
{
int tailIdx = starts.size() - 1;
diff --git a/accord-core/src/main/java/accord/utils/ReducingRangeMap.java b/accord-core/src/main/java/accord/utils/ReducingRangeMap.java
index a762fdc2..190a99e1 100644
--- a/accord-core/src/main/java/accord/utils/ReducingRangeMap.java
+++ b/accord-core/src/main/java/accord/utils/ReducingRangeMap.java
@@ -28,6 +28,11 @@ import static accord.utils.SortedArrays.exponentialSearch;
public class ReducingRangeMap<V> extends ReducingIntervalMap<RoutingKey, V>
{
+ public interface ReduceFunction<V, V2, P1, P2>
+ {
+ V2 apply(V v, V2 v2, P1 p1, P2 p2, int startMatchingInput, int endMatchingInput, int matchingStartBound);
+ }
+
public static class SerializerSupport
{
public static <V> ReducingRangeMap<V> create(boolean inclusiveEnds, RoutingKey[] ends, V[] values)
@@ -41,9 +46,9 @@ public class ReducingRangeMap<V> extends ReducingIntervalMap<RoutingKey, V>
super();
}
- protected ReducingRangeMap(boolean inclusiveEnds, RoutingKey[] ends, V[] values)
+ protected ReducingRangeMap(boolean inclusiveEnds, RoutingKey[] starts, V[] values)
{
- super(inclusiveEnds, ends, values);
+ super(inclusiveEnds, starts, values);
}
public V foldl(Routables<?> routables, BiFunction<V, V, V> fold, V accumulator)
@@ -56,6 +61,16 @@ public class ReducingRangeMap<V> extends ReducingIntervalMap<RoutingKey, V>
return foldl(routables, (a, b, f, ignore) -> f.apply(a, b), accumulator, fold, null, terminate);
}
+ public <V2> V2 foldlWithBounds(Routables<?> routables, QuadFunction<V, V2, RoutingKey, RoutingKey, V2> fold, V2 accumulator, Predicate<V2> terminate)
+ {
+ return foldl(routables, (a, b, f, self, i, j, k) -> f.apply(a, b, self.starts[k], self.starts[k+1]), accumulator, fold, this, terminate);
+ }
+
+ public <R extends Routable, V2> V2 foldlWithInputAndBounds(Routables<R> routables, IndexedRangeQuadFunction<V, V2, RoutingKey, RoutingKey, V2> fold, V2 accumulator, Predicate<V2> terminate)
+ {
+ return foldl(routables, (a, b, f, self, i, j, k) -> f.apply(a, b, self.starts[k], self.starts[k+1], i, j), accumulator, fold, this, terminate);
+ }
+
public <V2, P1> V2 foldl(Routables<?> routables, TriFunction<V, V2, P1, V2> fold, V2 accumulator, P1 p1, Predicate<V2> terminate)
{
return foldl(routables, (a, b, f, p) -> f.apply(a, b, p), accumulator, fold, p1, terminate);
@@ -63,10 +78,35 @@ public class ReducingRangeMap<V> extends ReducingIntervalMap<RoutingKey, V>
public <V2, P1, P2> V2 foldl(Routables<?> routables, QuadFunction<V, V2, P1, P2, V2> fold, V2 accumulator, P1 p1, P2 p2, Predicate<V2> terminate)
{
- return foldl(routables, (v, v2, param1, param2, i, j) -> fold.apply(v, v2, param1, param2), accumulator, p1, p2, terminate);
+ return foldl(routables, (v, v2, param1, param2, i, j, k) -> fold.apply(v, v2, param1, param2), accumulator, p1, p2, terminate);
}
public <V2, P1, P2> V2 foldl(Routables<?> routables, IndexedRangeQuadFunction<V, V2, P1, P2, V2> fold, V2 accumulator, P1 p1, P2 p2, Predicate<V2> terminate)
+ {
+ return foldl(routables, (v, v2, param1, param2, i, j, k) -> fold.apply(v, v2, param1, param2, i, j), accumulator, p1, p2, terminate);
+ }
+
+ public V foldlWithDefault(Routables<?> routables, BiFunction<V, V, V> fold, V defaultValue, V accumulator)
+ {
+ return foldlWithDefault(routables, (a, b, f, ignore) -> f.apply(a, b), defaultValue, accumulator, fold, null, ignore -> false);
+ }
+
+ public <V2> V2 foldlWithDefault(Routables<?> routables, BiFunction<V, V2, V2> fold, V defaultValue, V2 accumulator, Predicate<V2> terminate)
+ {
+ return foldlWithDefault(routables, (a, b, f, ignore) -> f.apply(a, b), defaultValue, accumulator, fold, null, terminate);
+ }
+
+ public <V2, P1> V2 foldlWithDefault(Routables<?> routables, TriFunction<V, V2, P1, V2> fold, V defaultValue, V2 accumulator, P1 p1, Predicate<V2> terminate)
+ {
+ return foldlWithDefault(routables, (a, b, f, p) -> f.apply(a, b, p), defaultValue, accumulator, fold, p1, terminate);
+ }
+
+ public <V2, P1, P2> V2 foldlWithDefault(Routables<?> routables, QuadFunction<V, V2, P1, P2, V2> fold, V defaultValue, V2 accumulator, P1 p1, P2 p2, Predicate<V2> terminate)
+ {
+ return foldlWithDefault(routables, (v, v2, param1, param2, i, j) -> fold.apply(v, v2, param1, param2), defaultValue, accumulator, p1, p2, terminate);
+ }
+
+ private <V2, P1, P2> V2 foldl(Routables<?> routables, ReduceFunction<V, V2, P1, P2> fold, V2 accumulator, P1 p1, P2 p2, Predicate<V2> terminate)
{
switch (routables.domain())
{
@@ -76,7 +116,7 @@ public class ReducingRangeMap<V> extends ReducingIntervalMap<RoutingKey, V>
}
}
- public <V2, P1, P2> V2 foldl(AbstractKeys<?> keys, IndexedRangeQuadFunction<V, V2, P1, P2, V2> fold, V2 accumulator, P1 p1, P2 p2, Predicate<V2> terminate)
+ private <V2, P1, P2> V2 foldl(AbstractKeys<?> keys, ReduceFunction<V, V2, P1, P2> fold, V2 accumulator, P1 p1, P2 p2, Predicate<V2> terminate)
{
if (values.length == 0)
return accumulator;
@@ -100,7 +140,7 @@ public class ReducingRangeMap<V> extends ReducingIntervalMap<RoutingKey, V>
if (j != nextj && values[i] != null)
{
- accumulator = fold.apply(values[i], accumulator, p1, p2, j, nextj);
+ accumulator = fold.apply(values[i], accumulator, p1, p2, j, nextj, i);
if (terminate.test(accumulator))
return accumulator;
}
@@ -110,7 +150,7 @@ public class ReducingRangeMap<V> extends ReducingIntervalMap<RoutingKey, V>
return accumulator;
}
- public <V2, P1, P2> V2 foldl(AbstractRanges ranges, IndexedRangeQuadFunction<V, V2, P1, P2, V2> fold, V2 accumulator, P1 p1, P2 p2, Predicate<V2> terminate)
+ private <V2, P1, P2> V2 foldl(AbstractRanges ranges, ReduceFunction<V, V2, P1, P2> fold, V2 accumulator, P1 p1, P2 p2, Predicate<V2> terminate)
{
if (values.length == 0)
return accumulator;
@@ -144,7 +184,58 @@ public class ReducingRangeMap<V> extends ReducingIntervalMap<RoutingKey, V>
if (toj > j && values[i] != null)
{
- accumulator = fold.apply(values[i], accumulator, p1, p2, j, toj);
+ accumulator = fold.apply(values[i], accumulator, p1, p2, j, toj, i);
+ if (terminate.test(accumulator))
+ return accumulator;
+ }
+ ++i;
+ j = nextj;
+ }
+ return accumulator;
+ }
+
+ public <V2, P1, P2> V2 foldlWithDefault(Routables<?> routables, IndexedRangeQuadFunction<V, V2, P1, P2, V2> fold, V defaultValue, V2 accumulator, P1 p1, P2 p2, Predicate<V2> terminate)
+ {
+ switch (routables.domain())
+ {
+ default: throw new AssertionError("Unknown domain: " + routables.domain());
+ case Key: return foldlWithDefault((AbstractKeys<?>) routables, fold, defaultValue, accumulator, p1, p2, terminate);
+ case Range: return foldlWithDefault((AbstractRanges) routables, fold, defaultValue, accumulator, p1, p2, terminate);
+ }
+ }
+
+ private <V2, P1, P2> V2 foldlWithDefault(AbstractKeys<?> keys, IndexedRangeQuadFunction<V, V2, P1, P2, V2> fold, V defaultValue, V2 accumulator, P1 p1, P2 p2, Predicate<V2> terminate)
+ {
+ if (values.length == 0 || keys.isEmpty())
+ return fold.apply(defaultValue, accumulator, p1, p2, 0, keys.size());
+
+ int i = 0, j = keys.findNext(0, starts[0], FAST);
+ if (j < 0) j = -1 - j;
+ else if (inclusiveEnds) ++j;
+
+ if (j > 0)
+ accumulator = fold.apply(defaultValue, accumulator, p1, p2, 0, j);
+
+ while (j < keys.size())
+ {
+ i = exponentialSearch(starts, i, starts.length, keys.get(j));
+ if (i < 0) i = -2 - i;
+ else if (inclusiveEnds) --i;
+
+ if (i >= values.length)
+ return fold.apply(defaultValue, accumulator, p1, p2, j, keys.size());
+
+ int nextj = keys.findNext(j, starts[i + 1], FAST);
+ if (nextj < 0) nextj = -1 -nextj;
+ else if (inclusiveEnds) ++nextj;
+
+ if (j != nextj)
+ {
+ V value = values[i];
+ if (value == null)
+ value = defaultValue;
+
+ accumulator = fold.apply(value, accumulator, p1, p2, j, nextj);
if (terminate.test(accumulator))
return accumulator;
}
@@ -154,6 +245,58 @@ public class ReducingRangeMap<V> extends ReducingIntervalMap<RoutingKey, V>
return accumulator;
}
+ private <V2, P1, P2> V2 foldlWithDefault(AbstractRanges ranges, IndexedRangeQuadFunction<V, V2, P1, P2, V2> fold, V defaultValue, V2 accumulator, P1 p1, P2 p2, Predicate<V2> terminate)
+ {
+ if (values.length == 0 || ranges.isEmpty())
+ return fold.apply(defaultValue, accumulator, p1, p2, 0, ranges.size());
+
+ // TODO (desired): first searches should be binarySearch
+ int j = ranges.findNext(0, starts[0], FAST);
+ if (j < 0) j = -1 - j;
+ else if (inclusiveEnds && ranges.get(j).end().equals(starts[0])) ++j;
+
+ if (j > 0 || starts[0].compareTo(ranges.get(0).start()) > 0)
+ accumulator = fold.apply(defaultValue, accumulator, p1, p2, 0, j);
+
+ int i = 0;
+ while (j < ranges.size())
+ {
+ Range range = ranges.get(j);
+ RoutingKey start = range.start();
+ int nexti = exponentialSearch(starts, i, starts.length, start);
+ if (nexti < 0) i = Math.max(i, -2 - nexti);
+ else if (nexti > i && !inclusiveStarts()) i = nexti - 1;
+ else i = nexti;
+
+ if (i >= values.length)
+ return fold.apply(defaultValue, accumulator, p1, p2, j, ranges.size());
+
+ int toj, nextj = ranges.findNext(j, starts[i + 1], FAST);
+ if (nextj < 0) toj = nextj = -1 -nextj;
+ else
+ {
+ toj = nextj + 1;
+ if (inclusiveEnds && ranges.get(nextj).end().equals(starts[i + 1]))
+ ++nextj;
+ }
+
+ if (toj > j)
+ {
+ V value = values[i];
+ if (value == null)
+ value = defaultValue;
+
+ accumulator = fold.apply(value, accumulator, p1, p2, j, toj);
+ if (terminate.test(accumulator))
+ return accumulator;
+ }
+ ++i;
+ j = nextj;
+ }
+
+ return accumulator;
+ }
+
public static <V> ReducingRangeMap<V> create(Ranges ranges, V value)
{
if (value == null)
diff --git a/accord-core/src/main/java/accord/utils/SearchableRangeList.java b/accord-core/src/main/java/accord/utils/SearchableRangeList.java
index 06027bb7..22aeb3a1 100644
--- a/accord-core/src/main/java/accord/utils/SearchableRangeList.java
+++ b/accord-core/src/main/java/accord/utils/SearchableRangeList.java
@@ -145,12 +145,12 @@ public class SearchableRangeList
}
@Inline
- public <P1, P2, P3> int forEach(Range range, IndexedTriConsumer<P1, P2, P3> forEachScanOrCheckpoint, IndexedRangeTriConsumer<P1, P2, P3> forEachRange, P1 p1, P2 p2, P3 p3, int minIndex)
+ public <P1, P2, P3, P4> int forEach(Range range, IndexedQuadConsumer<P1, P2, P3, P4> forEachScanOrCheckpoint, IndexedRangeQuadConsumer<P1, P2, P3, P4> forEachRange, P1 p1, P2 p2, P3 p3, P4 p4, int minIndex)
{
- return forEach(range.start(), range.end(), forEachScanOrCheckpoint, forEachRange, p1, p2, p3, minIndex);
+ return forEach(range.start(), range.end(), forEachScanOrCheckpoint, forEachRange, p1, p2, p3, p4, minIndex);
}
- public <P1, P2, P3> int forEach(RoutingKey startKey, RoutingKey endKey, IndexedTriConsumer<P1, P2, P3> forEachScanOrCheckpoint, IndexedRangeTriConsumer<P1, P2, P3> forEachRange, P1 p1, P2 p2, P3 p3, int minIndex)
+ public <P1, P2, P3, P4> int forEach(RoutingKey startKey, RoutingKey endKey, IndexedQuadConsumer<P1, P2, P3, P4> forEachScanOrCheckpoint, IndexedRangeQuadConsumer<P1, P2, P3, P4> forEachRange, P1 p1, P2 p2, P3 p3, P4 p4, int minIndex)
{
if (ranges.length == 0 || minIndex == ranges.length)
return minIndex;
@@ -175,20 +175,21 @@ public class SearchableRangeList
++start;
}
- return forEach(start, end, floor, startKey, forEachScanOrCheckpoint, forEachRange, p1, p2, p3, minIndex);
+ // Since endInclusive() != startInclusive(), so no need to adjust start/end comparisons
+ return forEach(start, end, floor, startKey, 0, forEachScanOrCheckpoint, forEachRange, p1, p2, p3, p4, minIndex);
}
@Inline
- public <P1, P2, P3> int forEach(RoutableKey key, IndexedTriConsumer<P1, P2, P3> forEachScanOrCheckpoint, IndexedRangeTriConsumer<P1, P2, P3> forEachRange, P1 p1, P2 p2, P3 p3, int minIndex)
+ public <P1, P2, P3, P4> int forEach(RoutableKey key, IndexedQuadConsumer<P1, P2, P3, P4> forEachScanOrCheckpoint, IndexedRangeQuadConsumer<P1, P2, P3, P4> forEachRange, P1 p1, P2 p2, P3 p3, P4 p4, int minIndex)
{
if (ranges.length == 0 || minIndex == ranges.length)
return minIndex;
- int end = SortedArrays.binarySearch(ranges, minIndex, ranges.length, key, (a, b) -> a.compareTo(b.start()), FLOOR);
+ int end = SortedArrays.binarySearch(ranges, minIndex, ranges.length, key, (a, b) -> -b.compareStartTo(a), FLOOR);
if (end < 0) end = -1 - end;
if (end <= minIndex) return minIndex;
- int floor = SortedArrays.binarySearch(ranges, minIndex, ranges.length, key, (a, b) -> a.compareTo(b.start()), CEIL);
+ int floor = SortedArrays.binarySearch(ranges, minIndex, ranges.length, key, (a, b) -> -b.compareStartTo(a), CEIL);
int start = floor;
if (floor < 0)
{
@@ -200,18 +201,18 @@ public class SearchableRangeList
start = floor = -2 - floor;
if (start < 0)
start = floor = 0;
- else if (ranges[start].end().compareTo(key) <= 0)
+ else if (ranges[start].compareEndTo(key) < 0)
++start;
}
- // Key cannot equal RoutingKey, so can ignore inclusivity/exclusivity of end bound
- return forEach(start, end, floor, key, forEachScanOrCheckpoint, forEachRange, p1, p2, p3, minIndex);
+ int bound = ranges[0].endInclusive() ? -1 : 0;
+ return forEach(start, end, floor, key, bound, forEachScanOrCheckpoint, forEachRange, p1, p2, p3, p4, minIndex);
}
@Inline
- private <P1, P2, P3> int forEach(int start, int end, int floor, RoutableKey startBound,
- IndexedTriConsumer<P1, P2, P3> forEachScanOrCheckpoint, IndexedRangeTriConsumer<P1, P2, P3> forEachRange,
- P1 p1, P2 p2, P3 p3, int minIndex)
+ private <P1, P2, P3, P4> int forEach(int start, int end, int floor, RoutableKey startBound, int cmpStartBoundWithEnd,
+ IndexedQuadConsumer<P1, P2, P3, P4> forEachScanOrCheckpoint, IndexedRangeQuadConsumer<P1, P2, P3, P4> forEachRange,
+ P1 p1, P2 p2, P3 p3, P4 p4, int minIndex)
{
if (start < minIndex) start = minIndex;
@@ -267,11 +268,11 @@ public class SearchableRangeList
if (ri < 0)
continue;
- if (ranges[ri].end().compareTo(startBound) <= 0)
+ if (ranges[ri].end().compareTo(startBound) <= cmpStartBoundWithEnd)
break;
if (ri >= minIndex && ri < minScanIndex)
- forEachScanOrCheckpoint.accept(p1, p2, p3, ri);
+ forEachScanOrCheckpoint.accept(p1, p2, p3, p4, ri);
}
}
else
@@ -279,21 +280,21 @@ public class SearchableRangeList
// if startBound is key, we cannot be equal to it;
// if startBound is a Range start, we also cannot be equal to it due to the requirement that
// endInclusive() != startInclusive(), so equality really means inequality
- if (ranges[ri].end().compareTo(startBound) <= 0)
+ if (ranges[ri].end().compareTo(startBound) <= cmpStartBoundWithEnd)
break;
if (ri >= minIndex && ri < minScanIndex)
- forEachScanOrCheckpoint.accept(p1, p2, p3, ri);
+ forEachScanOrCheckpoint.accept(p1, p2, p3, p4, ri);
}
}
for (int i = minScanIndex; i < floor ; ++i)
{
- if (ranges[i].end().compareTo(startBound) > 0)
- forEachScanOrCheckpoint.accept(p1, p2, p3, i);
+ if (ranges[i].end().compareTo(startBound) > cmpStartBoundWithEnd)
+ forEachScanOrCheckpoint.accept(p1, p2, p3, p4, i);
}
- forEachRange.accept(p1, p2, p3, start, end);
+ forEachRange.accept(p1, p2, p3, p4, start, end);
return end;
}
diff --git a/accord-core/src/main/java/accord/utils/SimpleBitSet.java b/accord-core/src/main/java/accord/utils/SimpleBitSet.java
index bff0b5c1..133a1f53 100644
--- a/accord-core/src/main/java/accord/utils/SimpleBitSet.java
+++ b/accord-core/src/main/java/accord/utils/SimpleBitSet.java
@@ -401,4 +401,10 @@ public class SimpleBitSet
{
return Arrays.equals(this.bits, other.bits);
}
+
+ public void clear()
+ {
+ count = 0;
+ Arrays.fill(bits, 0L);
+ }
}
diff --git a/accord-core/src/main/java/accord/utils/SortedArrays.java b/accord-core/src/main/java/accord/utils/SortedArrays.java
index 7602484c..255bc89e 100644
--- a/accord-core/src/main/java/accord/utils/SortedArrays.java
+++ b/accord-core/src/main/java/accord/utils/SortedArrays.java
@@ -403,15 +403,15 @@ public class SortedArrays
/**
* A linear intersection where we only want results from the left inputs, and the right inputs may either be a different type or otherwise only used for filtering
*/
- public static <T1, T2> T1[] asymmetricLinearIntersectionWithOverlaps(T1[] left, int leftLength, T2[] right, int rightLength, AsymmetricComparator<? super T1, ? super T2> comparator, ObjectBuffers<T1> buffers)
+ public static <T1, T2> T1[] intersectWithMultipleMatches(T1[] left, int leftLength, T2[] right, int rightLength, AsymmetricComparator<? super T1, ? super T2> comparator, ObjectBuffers<T1> buffers)
{
- return (T1[])internalLinearIntersectionWithOverlaps(true, left, leftLength, right, rightLength, comparator, buffers, null);
+ return (T1[]) internalLinearIntersectionWithMultipleMatches(true, left, leftLength, right, rightLength, comparator, buffers, null);
}
/**
* A linear intersection where we only want results from the left inputs, and the right inputs may either be a different type or otherwise only used for filtering
*/
- private static <T1, T2> Object[] internalLinearIntersectionWithOverlaps(boolean preferLeft, T1[] left, int leftLength, T2[] right, int rightLength, AsymmetricComparator<? super T1, ? super T2> comparator, ObjectBuffers<T1> leftBuffers, @Nullable ObjectBuffers<T2> rightBuffers)
+ private static <T1, T2> Object[] internalLinearIntersectionWithMultipleMatches(boolean preferLeft, T1[] left, int leftLength, T2[] right, int rightLength, AsymmetricComparator<? super T1, ? super T2> comparator, ObjectBuffers<T1> leftBuffers, @Nullable ObjectBuffers<T2> rightBuffers)
{
int leftIdx = 0;
int rightIdx = 0;
@@ -587,6 +587,8 @@ public class SortedArrays
/**
* Given two sorted arrays, return the elements present only in the first, preferentially returning the first array
* itself if possible
+ *
+ * TODO (expected): use cachedBuffers
*/
@SuppressWarnings("unused") // was used until recently, might be used again?
public static <T extends Comparable<? super T>> T[] linearSubtract(T[] left, T[] right, IntFunction<T[]> allocate)
@@ -657,6 +659,8 @@ public class SortedArrays
* Given two sorted arrays {@code slice} and {@code select}, where each array's contents is unique and non-overlapping
* with itself, but may match multiple entries in the other array, return a new array containing the elements of {@code slice}
* that match elements of {@code select} as per the provided comparators.
+ *
+ * TODO (expected): use buffers rather than factory
*/
public static <A, R> A[] sliceWithMultipleMatches(A[] input, R[] select, IntFunction<A[]> factory, AsymmetricComparator<A, R> cmp1, AsymmetricComparator<R, A> cmp2)
{
@@ -721,6 +725,8 @@ public class SortedArrays
* Given two sorted arrays {@code slice} and {@code select}, where each array's contents is unique and non-overlapping
* with itself, but may match multiple entries in the other array, return a new array containing the elements of {@code slice}
* that match elements of {@code select} as per the provided comparators.
+ *
+ * TODO (expected): use buffers rather than factory
*/
public static <A, R> A[] subtractWithMultipleMatches(A[] input, R[] subtract, IntFunction<A[]> factory, AsymmetricComparator<A, R> cmp1, AsymmetricComparator<R, A> cmp2)
{
diff --git a/accord-core/src/main/java/accord/utils/random/Picker.java b/accord-core/src/main/java/accord/utils/random/Picker.java
new file mode 100644
index 00000000..f6f07d75
--- /dev/null
+++ b/accord-core/src/main/java/accord/utils/random/Picker.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package accord.utils.random;
+
+import java.util.Arrays;
+import java.util.function.Supplier;
+
+import accord.utils.Invariants;
+import accord.utils.RandomSource;
+
+public class Picker
+{
+ static abstract class Weighted
+ {
+ final RandomSource random;
+ final float[] weights;
+
+ public Weighted(RandomSource random, int length)
+ {
+ this(random, randomWeights(random, length));
+ }
+
+ public Weighted(RandomSource random, float[] weights)
+ {
+ this.random = random;
+ this.weights = weights;
+ }
+
+ static float[] randomWeights(RandomSource random, int length)
+ {
+ float[] weights = new float[length - 1];
+ float sum = 0;
+ for (int i = 0 ; i < weights.length ; ++i)
+ weights[i] = sum += random.nextFloat();
+ sum += random.nextFloat();
+ for (int i = 0 ; i < weights.length ; ++i)
+ weights[i] /= sum;
+ return weights;
+ }
+
+ static float[] randomWeights(RandomSource random, float[] bias)
+ {
+ float[] weights = new float[bias.length - 1];
+ float sum = 0;
+ for (int i = 0 ; i < weights.length ; ++i)
+ weights[i] = sum += random.nextFloat() * bias[i];
+ sum += random.nextFloat() * bias[weights.length];
+ for (int i = 0 ; i < weights.length ; ++i)
+ weights[i] /= sum;
+ return weights;
+ }
+
+ static float[] normaliseWeights(float[] input)
+ {
+ float[] output = new float[input.length - 1];
+ float sum = 0;
+ for (int i = 0 ; i < output.length ; ++i)
+ output[i] = sum += input[i];
+ sum += input[output.length];
+ for (int i = 0 ; i < output.length ; ++i)
+ output[i] /= sum;
+ return output;
+ }
+
+ int pickIndex()
+ {
+ int i = Arrays.binarySearch(weights, random.nextFloat());
+ if (i < 0) i = -1 - i;
+ return i;
+ }
+ }
+
+ public static class WeightedObjectPicker<T> extends Weighted implements Supplier<T>
+ {
+ final T[] values;
+
+ private WeightedObjectPicker(RandomSource random, T[] values, float[] weights)
+ {
+ super(random, values.length);
+ this.values = values;
+ }
+
+ @Override
+ public T get()
+ {
+ return values[pickIndex()];
+ }
+
+ public static <T> WeightedObjectPicker<T> randomWeighted(RandomSource random, T[] values)
+ {
+ return new WeightedObjectPicker<>(random, values, randomWeights(random, values.length));
+ }
+
+ public static <T> WeightedObjectPicker<T> randomWeighted(RandomSource random, T[] values, float[] bias)
+ {
+ Invariants.checkArgument(values.length == bias.length);
+ return new WeightedObjectPicker<>(random, values, randomWeights(random, bias));
+ }
+
+ public static <T> WeightedObjectPicker<T> weighted(RandomSource random, T[] values, float[] proportionalWeights)
+ {
+ Invariants.checkArgument(values.length == proportionalWeights.length);
+ return new WeightedObjectPicker<>(random, values, normaliseWeights(proportionalWeights));
+ }
+ }
+}
diff --git a/accord-core/src/main/java/accord/local/CommandListener.java b/accord-core/src/main/java/accord/utils/random/Suppliers.java
similarity index 51%
rename from accord-core/src/main/java/accord/local/CommandListener.java
rename to accord-core/src/main/java/accord/utils/random/Suppliers.java
index cb7a782a..e353308c 100644
--- a/accord-core/src/main/java/accord/local/CommandListener.java
+++ b/accord-core/src/main/java/accord/utils/random/Suppliers.java
@@ -16,24 +16,40 @@
* limitations under the License.
*/
-package accord.local;
+package accord.utils.random;
-import accord.primitives.TxnId;
+import java.util.function.BooleanSupplier;
+import java.util.function.IntSupplier;
+import java.util.function.LongSupplier;
+import java.util.function.Supplier;
-public interface CommandListener
+public class Suppliers
{
- void onChange(SafeCommandStore safeStore, SafeCommand safeCommand);
+ public interface Bools extends BooleanSupplier
+ {
+ boolean get();
+ void cycle();
+ default boolean getAsBoolean() { return get(); }
+ }
+
+ public interface Ints extends IntSupplier
+ {
+ int get();
+ void cycle();
+ default int getAsInt() { return get(); }
+ }
- /**
- * Scope needed to run onChange
- */
- PreLoadContext listenerPreLoadContext(TxnId caller);
+ public interface Longs extends LongSupplier
+ {
+ long get();
+ void cycle();
+ default long getAsLong() { return get(); }
+ }
- /**
- * Indicates the command to listener relationship doesn't need to persist across restarts
- */
- default boolean isTransient()
+ public interface Objects<T> extends Supplier<T>
{
- return false;
+ T get();
+ void cycle();
}
+
}
diff --git a/accord-core/src/test/java/accord/burn/BurnTest.java b/accord-core/src/test/java/accord/burn/BurnTest.java
index 721cf030..8e45b464 100644
--- a/accord-core/src/test/java/accord/burn/BurnTest.java
+++ b/accord-core/src/test/java/accord/burn/BurnTest.java
@@ -34,6 +34,7 @@ import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BiConsumer;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.LongSupplier;
@@ -74,6 +75,7 @@ import accord.messages.MessageType;
import accord.primitives.Keys;
import accord.primitives.Range;
import accord.primitives.Ranges;
+import accord.primitives.Timestamp;
import accord.primitives.Txn;
import accord.utils.DefaultRandom;
import accord.utils.RandomSource;
@@ -116,8 +118,6 @@ public class BurnTest
ListQuery query = new ListQuery(client, count);
ListRequest request = new ListRequest(new Txn.InMemory(ranges, read, query, null), listener);
packets.add(new Packet(client, node, count, request));
-
-
}
else
{
@@ -200,24 +200,25 @@ public class BurnTest
RandomDelayQueue delayQueue = new Factory(random).get();
PropagatingPendingQueue queue = new PropagatingPendingQueue(failures, delayQueue);
RandomSource retryRandom = random.fork();
- ListAgent agent = new ListAgent(1000L, failures::add, retry -> {
+ Function<BiConsumer<Timestamp, Ranges>, ListAgent> agentSupplier = onStale -> new ListAgent(1000L, failures::add, retry -> {
long delay = retryRandom.nextInt(1, 15);
queue.add((PendingRunnable)retry::run, delay, TimeUnit.SECONDS);
- });
+ }, onStale);
Supplier<LongSupplier> nowSupplier = () -> {
RandomSource forked = random.fork();
+ // TODO (now): meta-randomise scale of clock drift
return FrequentLargeRange.builder(forked)
.ratio(1, 5)
.small(50, 5000, TimeUnit.MICROSECONDS)
.large(1, 10, TimeUnit.MILLISECONDS)
.build()
- .mapAsLong(j -> Math.max(0, queue.nowInMillis() + j))
+ .mapAsLong(j -> Math.max(0, queue.nowInMillis() + TimeUnit.NANOSECONDS.toMillis(j)))
.asLongSupplier(forked);
};
Verifier verifier = createVerifier(keyCount);
- SimulatedDelayedExecutorService globalExecutor = new SimulatedDelayedExecutorService(queue, agent);
+ SimulatedDelayedExecutorService globalExecutor = new SimulatedDelayedExecutorService(queue, null);
Function<CommandStore, AsyncExecutor> executor = ignore -> globalExecutor;
@@ -311,9 +312,10 @@ public class BurnTest
Map<MessageType, Stats> messageStatsMap;
try
{
- messageStatsMap = Cluster.run(toArray(nodes, Id[]::new), listener, () -> queue, queue::checkFailures,
- responseSink, globalExecutor,
- random::fork, nowSupplier,
+ messageStatsMap = Cluster.run(toArray(nodes, Id[]::new), listener, () -> queue,
+ (id, onStale) -> globalExecutor.withAgent(agentSupplier.apply(onStale)),
+ queue::checkFailures,
+ responseSink, random::fork, nowSupplier,
topologyFactory, initialRequests::poll,
onSubmitted::set
);
diff --git a/accord-core/src/test/java/accord/burn/TopologyUpdates.java b/accord-core/src/test/java/accord/burn/TopologyUpdates.java
index ae3d3f0a..caaa7099 100644
--- a/accord-core/src/test/java/accord/burn/TopologyUpdates.java
+++ b/accord-core/src/test/java/accord/burn/TopologyUpdates.java
@@ -28,15 +28,16 @@ import accord.utils.MessageTask;
import org.agrona.collections.Long2ObjectHashMap;
import java.util.*;
+import java.util.function.Function;
public class TopologyUpdates
{
private final Long2ObjectHashMap<Map<Node.Id, Ranges>> pendingTopologies = new Long2ObjectHashMap<>();
- private final AgentExecutor executor;
- public TopologyUpdates(AgentExecutor executor)
+ Function<Node.Id, AgentExecutor> executors;
+ public TopologyUpdates(Function<Node.Id, AgentExecutor> executors)
{
- this.executor = executor;
+ this.executors = executors;
}
public synchronized MessageTask notify(Node originator, Topology prev, Topology update)
@@ -50,7 +51,7 @@ public class TopologyUpdates
nodeToNewRanges.put(node, newRanges);
}
pendingTopologies.put(update.epoch(), nodeToNewRanges);
- return MessageTask.begin(originator, nodes, executor, "TopologyNotify:" + update.epoch(), (node, from, onDone) -> {
+ return MessageTask.begin(originator, nodes, executors.apply(originator.id()), "TopologyNotify:" + update.epoch(), (node, from, onDone) -> {
long nodeEpoch = node.epoch();
if (nodeEpoch + 1 < update.epoch())
onDone.accept(false);
@@ -71,7 +72,7 @@ public class TopologyUpdates
if (pending.isEmpty())
pendingTopologies.remove(epoch);
- MessageTask.begin(originator, cluster, executor, "SyncComplete:" + epoch, (node, from, onDone) -> {
+ MessageTask.begin(originator, cluster, executors.apply(originator.id()), "SyncComplete:" + epoch, (node, from, onDone) -> {
node.onRemoteSyncComplete(originator.id(), epoch);
onDone.accept(true);
});
@@ -79,8 +80,8 @@ public class TopologyUpdates
public synchronized void epochClosed(Node originator, Collection<Node.Id> cluster, Ranges ranges, long epoch)
{
- executor.execute(() -> {
- MessageTask.begin(originator, cluster, executor, "EpochClosed:" + epoch, (node, from, onDone) -> {
+ executors.apply(originator.id()).execute(() -> {
+ MessageTask.begin(originator, cluster, executors.apply(originator.id()), "EpochClosed:" + epoch, (node, from, onDone) -> {
node.onEpochClosed(ranges, epoch);
onDone.accept(true);
});
@@ -89,8 +90,8 @@ public class TopologyUpdates
public synchronized void epochRedundant(Node originator, Collection<Node.Id> cluster, Ranges ranges, long epoch)
{
- executor.execute(() -> {
- MessageTask.begin(originator, cluster, executor, "EpochComplete:" + epoch, (node, from, onDone) -> {
+ executors.apply(originator.id()).execute(() -> {
+ MessageTask.begin(originator, cluster, executors.apply(originator.id()), "EpochComplete:" + epoch, (node, from, onDone) -> {
node.onEpochRedundant(ranges, epoch);
onDone.accept(true);
});
diff --git a/accord-core/src/test/java/accord/coordinate/tracking/TrackerReconciler.java b/accord-core/src/test/java/accord/coordinate/tracking/TrackerReconciler.java
index 0c9bdea8..7fca1f82 100644
--- a/accord-core/src/test/java/accord/coordinate/tracking/TrackerReconciler.java
+++ b/accord-core/src/test/java/accord/coordinate/tracking/TrackerReconciler.java
@@ -123,7 +123,7 @@ public abstract class TrackerReconciler<ST extends ShardTracker, T extends Abstr
Deque<Topology> topologies = new ArrayDeque<>();
topologies.add(topology);
- TopologyUpdates topologyUpdates = new TopologyUpdates(executor);
+ TopologyUpdates topologyUpdates = new TopologyUpdates(null);
TopologyRandomizer configRandomizer = new TopologyRandomizer(() -> random, topology, topologyUpdates, null);
while (--count > 0)
{
diff --git a/accord-core/src/test/java/accord/impl/TestAgent.java b/accord-core/src/test/java/accord/impl/TestAgent.java
index d3e4832b..0624c4ea 100644
--- a/accord-core/src/test/java/accord/impl/TestAgent.java
+++ b/accord-core/src/test/java/accord/impl/TestAgent.java
@@ -95,6 +95,12 @@ public class TestAgent implements Agent
retry.run();
}
+ @Override
+ public void onStale(Timestamp staleSince, Ranges ranges)
+ {
+
+ }
+
@Override
public void onUncaughtException(Throwable t)
{
@@ -119,7 +125,8 @@ public class TestAgent implements Agent
}
@Override
- public void onLocalBarrier(@Nonnull Seekables<?, ?> keysOrRanges, @Nonnull Timestamp executeAt) {
+ public void onLocalBarrier(@Nonnull Seekables<?, ?> keysOrRanges, @Nonnull Timestamp executeAt)
+ {
completedLocalBarriers.computeIfAbsent(executeAt, ignored -> new AtomicInteger()).incrementAndGet();
}
}
diff --git a/accord-core/src/test/java/accord/impl/basic/Cluster.java b/accord-core/src/test/java/accord/impl/basic/Cluster.java
index 8595f7a6..7455a413 100644
--- a/accord-core/src/test/java/accord/impl/basic/Cluster.java
+++ b/accord-core/src/test/java/accord/impl/basic/Cluster.java
@@ -22,15 +22,18 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.function.BiConsumer;
+import java.util.function.BiFunction;
+import java.util.function.BooleanSupplier;
import java.util.function.Consumer;
import java.util.function.Function;
+import java.util.function.IntSupplier;
import java.util.function.LongSupplier;
import java.util.function.Supplier;
@@ -46,6 +49,7 @@ import accord.burn.BurnTestConfigurationService;
import accord.burn.TopologyUpdates;
import accord.coordinate.TxnExecute;
import accord.coordinate.TxnPersist;
+import accord.burn.random.FrequentLargeRange;
import accord.impl.CoordinateDurabilityScheduling;
import accord.impl.IntHashKey;
import accord.impl.SimpleProgressLog;
@@ -65,12 +69,22 @@ import accord.messages.Message;
import accord.messages.Reply;
import accord.messages.Request;
import accord.messages.SafeCallback;
+import accord.primitives.Ranges;
+import accord.primitives.Timestamp;
import accord.topology.Topology;
import accord.topology.TopologyRandomizer;
import accord.utils.RandomSource;
import accord.utils.async.AsyncChains;
import accord.utils.async.AsyncResult;
+import static accord.impl.basic.Cluster.OverrideLinksKind.NONE;
+import static accord.impl.basic.Cluster.OverrideLinksKind.RANDOM_BIDIRECTIONAL;
+import static accord.impl.basic.NodeSink.Action.DELIVER;
+import static accord.impl.basic.NodeSink.Action.DROP;
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.singletonMap;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
import static java.util.concurrent.TimeUnit.SECONDS;
import static java.util.stream.Collectors.toList;
@@ -86,9 +100,34 @@ public class Cluster implements Scheduler
public String toString() { return Integer.toString(count); }
}
- Map<MessageType, Stats> statsMap = new HashMap<>();
+ public static class LinkConfig
+ {
+ final Function<List<Id>, BiFunction<Id, Id, Link>> overrideLinks;
+ final BiFunction<Id, Id, Link> defaultLinks;
+
+ public LinkConfig(Function<List<Id>, BiFunction<Id, Id, Link>> overrideLinks, BiFunction<Id, Id, Link> defaultLinks)
+ {
+ this.overrideLinks = overrideLinks;
+ this.defaultLinks = defaultLinks;
+ }
+ }
- final RandomSource randomSource;
+ static class Link
+ {
+ final Supplier<NodeSink.Action> action;
+ final LongSupplier latencyMicros;
+
+ Link(Supplier<NodeSink.Action> action, LongSupplier latencyMicros)
+ {
+ this.action = action;
+ this.latencyMicros = latencyMicros;
+ }
+ }
+
+ final Map<MessageType, Stats> statsMap = new HashMap<>();
+
+ final RandomSource random;
+ final LinkConfig linkConfig;
final Function<Id, Node> lookup;
final PendingQueue pending;
final Runnable checkFailures;
@@ -98,17 +137,18 @@ public class Cluster implements Scheduler
final MessageListener messageListener;
int clock;
int recurring;
- Set<Id> partitionSet;
+ BiFunction<Id, Id, Link> links;
- public Cluster(RandomSource randomSource, MessageListener messageListener, Supplier<PendingQueue> queueSupplier, Runnable checkFailures, Function<Id, Node> lookup, Consumer<Packet> responseSink)
+ public Cluster(RandomSource random, MessageListener messageListener, Supplier<PendingQueue> queueSupplier, Runnable checkFailures, Function<Id, Node> lookup, IntSupplier rf, Consumer<Packet> responseSink)
{
- this.randomSource = randomSource;
+ this.random = random;
this.messageListener = messageListener;
this.pending = queueSupplier.get();
this.checkFailures = checkFailures;
this.lookup = lookup;
this.responseSink = responseSink;
- this.partitionSet = new HashSet<>();
+ this.linkConfig = defaultLinkConfig(random, rf);
+ this.links = linkConfig.defaultLinks;
}
NodeSink create(Id self, RandomSource random)
@@ -218,47 +258,75 @@ public class Cluster implements Scheduler
run.run();
}
- public static Map<MessageType, Stats> run(Id[] nodes, MessageListener messageListener, Supplier<PendingQueue> queueSupplier, Runnable checkFailures, Consumer<Packet> responseSink, AgentExecutor executor, Supplier<RandomSource> randomSupplier, Supplier<LongSupplier> nowSupplierSupplier, TopologyFactory topologyFactory, Supplier<Packet> in, Consumer<Runnable> noMoreWorkSignal)
+ public static Map<MessageType, Stats> run(Id[] nodes, MessageListener messageListener, Supplier<PendingQueue> queueSupplier,
+ BiFunction<Id, BiConsumer<Timestamp, Ranges>, AgentExecutor> nodeExecutorSupplier,
+ Runnable checkFailures, Consumer<Packet> responseSink,
+ Supplier<RandomSource> randomSupplier, Supplier<LongSupplier> nowSupplierSupplier,
+ TopologyFactory topologyFactory, Supplier<Packet> in, Consumer<Runnable> noMoreWorkSignal)
{
Topology topology = topologyFactory.toTopology(nodes);
- Map<Id, Node> lookup = new LinkedHashMap<>();
+ Map<Id, Node> nodeMap = new LinkedHashMap<>();
+ Map<Id, AgentExecutor> executorMap = new LinkedHashMap<>();
try
{
- Cluster sinks = new Cluster(randomSupplier.get(), messageListener, queueSupplier, checkFailures, lookup::get, responseSink);
- TopologyUpdates topologyUpdates = new TopologyUpdates(executor);
- TopologyRandomizer configRandomizer = new TopologyRandomizer(randomSupplier, topology, topologyUpdates, lookup::get);
+ RandomSource random = randomSupplier.get();
+ Cluster sinks = new Cluster(randomSupplier.get(), messageListener, queueSupplier, checkFailures, nodeMap::get, () -> topologyFactory.rf, responseSink);
+ TopologyUpdates topologyUpdates = new TopologyUpdates(executorMap::get);
+ TopologyRandomizer configRandomizer = new TopologyRandomizer(randomSupplier, topology, topologyUpdates, nodeMap::get);
List<CoordinateDurabilityScheduling> durabilityScheduling = new ArrayList<>();
for (Id id : nodes)
{
MessageSink messageSink = sinks.create(id, randomSupplier.get());
LongSupplier nowSupplier = nowSupplierSupplier.get();
- BurnTestConfigurationService configService = new BurnTestConfigurationService(id, executor, randomSupplier, topology, lookup::get, topologyUpdates);
LocalConfig localConfig = new MutableLocalConfig();
+ BiConsumer<Timestamp, Ranges> onStale = (sinceAtLeast, ranges) -> configRandomizer.onStale(id, sinceAtLeast, ranges);
+ AgentExecutor nodeExecutor = nodeExecutorSupplier.apply(id, onStale);
+ executorMap.put(id, nodeExecutor);
+ BurnTestConfigurationService configService = new BurnTestConfigurationService(id, nodeExecutor, randomSupplier, topology, nodeMap::get, topologyUpdates);
+ BooleanSupplier isLoadedCheck = random.biasedUniformBools(0.5f);
Node node = new Node(id, messageSink, LocalMessage::process, configService, nowSupplier, NodeTimeService.unixWrapper(TimeUnit.MILLISECONDS, nowSupplier),
() -> new ListStore(id), new ShardDistributor.EvenSplit<>(8, ignore -> new IntHashKey.Splitter()),
- executor.agent(),
+ nodeExecutor.agent(),
randomSupplier.get(), sinks, SizeOfIntersectionSorter.SUPPLIER,
- SimpleProgressLog::new, DelayedCommandStores.factory(sinks.pending), TxnExecute.FACTORY, TxnPersist.FACTORY, Apply.FACTORY,
+ SimpleProgressLog::new, DelayedCommandStores.factory(sinks.pending, isLoadedCheck), TxnExecute.FACTORY, TxnPersist.FACTORY, Apply.FACTORY,
localConfig);
- lookup.put(id, node);
CoordinateDurabilityScheduling durability = new CoordinateDurabilityScheduling(node);
// TODO (desired): randomise
durability.setFrequency(60, SECONDS);
durability.setGlobalCycleTime(180, SECONDS);
durabilityScheduling.add(durability);
+ nodeMap.put(id, node);
+ durabilityScheduling.add(new CoordinateDurabilityScheduling(node));
+ }
+
+ Runnable updateDurabilityRate;
+ {
+ IntSupplier frequencySeconds = random.biasedUniformIntsSupplier( 1, 120, 10, 30, 10, 60).get();
+ IntSupplier shardCycleTimeSeconds = random.biasedUniformIntsSupplier(5, 60, 10, 30, 1, 30).get();
+ IntSupplier globalCycleTimeSeconds = random.biasedUniformIntsSupplier( 1, 90, 10, 30, 10, 60).get();
+ updateDurabilityRate = () -> {
+ int f = frequencySeconds.getAsInt();
+ int s = shardCycleTimeSeconds.getAsInt();
+ int g = globalCycleTimeSeconds.getAsInt();
+ durabilityScheduling.forEach(d -> {
+ d.setFrequency(f, SECONDS);
+ d.setShardCycleTime(s, SECONDS);
+ d.setGlobalCycleTime(g, SECONDS);
+ });
+ };
}
+ updateDurabilityRate.run();
// startup
- AsyncResult<?> startup = AsyncChains.reduce(lookup.values().stream().map(Node::unsafeStart).collect(toList()), (a, b) -> null).beginAsResult();
+ AsyncResult<?> startup = AsyncChains.reduce(nodeMap.values().stream().map(Node::unsafeStart).collect(toList()), (a, b) -> null).beginAsResult();
while (sinks.processPending());
Assertions.assertTrue(startup.isDone());
List<Id> nodesList = new ArrayList<>(Arrays.asList(nodes));
- RandomSource shuffleRandom = randomSupplier.get();
Scheduled chaos = sinks.recurring(() -> {
- Collections.shuffle(nodesList, shuffleRandom.asJdkRandom());
- int partitionSize = shuffleRandom.nextInt((topologyFactory.rf+1)/2);
- sinks.partitionSet = new LinkedHashSet<>(nodesList.subList(0, partitionSize));
+ sinks.links = sinks.linkConfig.overrideLinks.apply(nodesList);
+ if (random.decide(0.1f))
+ updateDurabilityRate.run();
}, 5L, SECONDS);
Scheduled reconfigure = sinks.recurring(configRandomizer::maybeUpdateTopology, 1, SECONDS);
@@ -278,7 +346,7 @@ public class Cluster implements Scheduler
chaos.cancel();
reconfigure.cancel();
durabilityScheduling.forEach(CoordinateDurabilityScheduling::stop);
- sinks.partitionSet = Collections.emptySet();
+ sinks.links = sinks.linkConfig.defaultLinks;
// give progress log et al a chance to finish
// TODO (desired, testing): would be nice to make this more certain than an arbitrary number of additional rounds
@@ -300,7 +368,157 @@ public class Cluster implements Scheduler
}
finally
{
- lookup.values().forEach(Node::shutdown);
+ nodeMap.values().forEach(Node::shutdown);
}
}
+
+ private static BiFunction<Id, Id, Link> partition(List<Id> nodes, RandomSource random, int rf, BiFunction<Id, Id, Link> up)
+ {
+ Collections.shuffle(nodes, random.asJdkRandom());
+ int partitionSize = random.nextInt((rf+1)/2);
+ Set<Id> partition = new LinkedHashSet<>(nodes.subList(0, partitionSize));
+ BiFunction<Id, Id, Link> down = (from, to) -> new Link(() -> DROP, up.apply(from, to).latencyMicros);
+ return (from, to) -> (partition.contains(from) == partition.contains(to) ? up : down).apply(from, to);
+ }
+
+ /**
+ * pair every node with one other node in one direction with a network behaviour override
+ */
+ private static BiFunction<Id, Id, Link> pairedUnidirectionalOverrides(Function<Link, Link> linkOverride, List<Id> nodes, RandomSource random, BiFunction<Id, Id, Link> fallback)
+ {
+ Map<Id, Map<Id, Link>> map = new HashMap<>();
+ Collections.shuffle(nodes, random.asJdkRandom());
+ for (int i = 0 ; i + 1 < nodes.size() ; i += 2)
+ {
+ Id from = nodes.get(i);
+ Id to = nodes.get(i + 1);
+ Link link = linkOverride.apply(fallback.apply(from, to));
+ map.put(from, singletonMap(to, link));
+ }
+ return (from, to) -> nonNullOrGet(map.getOrDefault(from, emptyMap()).get(to), from, to, fallback);
+ }
+
+ private static BiFunction<Id, Id, Link> randomOverrides(boolean bidirectional, Function<Link, Link> linkOverride, int count, List<Id> nodes, RandomSource random, BiFunction<Id, Id, Link> fallback)
+ {
+ Map<Id, Map<Id, Link>> map = new HashMap<>();
+ while (count > 0)
+ {
+ Id from = nodes.get(random.nextInt(nodes.size()));
+ Id to = nodes.get(random.nextInt(nodes.size()));
+ Link fwd = linkOverride.apply(fallback.apply(from, to));
+ if (null == map.computeIfAbsent(from, ignore -> new HashMap<>()).putIfAbsent(to, fwd))
+ {
+ if (bidirectional)
+ {
+ Link rev = linkOverride.apply(fallback.apply(to, from));
+ map.computeIfAbsent(to, ignore -> new HashMap<>()).put(from, rev);
+ }
+ --count;
+ }
+ }
+ return (from, to) -> nonNullOrGet(map.getOrDefault(from, emptyMap()).get(to), from, to, fallback);
+ }
+
+ private static Link nonNullOrGet(Link ifNotNull, Id from, Id to, BiFunction<Id, Id, Link> function)
+ {
+ if (ifNotNull != null)
+ return ifNotNull;
+ return function.apply(from, to);
+ }
+
+ private static Link healthy(LongSupplier latency)
+ {
+ return new Link(() -> DELIVER, latency);
+ }
+
+ private static Link down(LongSupplier latency)
+ {
+ return new Link(() -> DROP, latency);
+ }
+
+ private LongSupplier defaultRandomWalkLatencyMicros(RandomSource random)
+ {
+ LongSupplier range = FrequentLargeRange.builder(random)
+ .ratio(1, 5)
+ .small(500, TimeUnit.MICROSECONDS, 5, MILLISECONDS)
+ .large(50, MILLISECONDS, 5, SECONDS)
+ .build().asLongSupplier(random);
+
+ return () -> NANOSECONDS.toMicros(range.getAsLong());
+ }
+
+ enum OverrideLinkKind { LATENCY, ACTION, BOTH }
+
+ private Supplier<Function<Link, Link>> linkOverrideSupplier(RandomSource random)
+ {
+ Supplier<OverrideLinkKind> nextKind = random.randomWeightedPicker(OverrideLinkKind.values());
+ Supplier<LongSupplier> latencySupplier = random.biasedUniformLongsSupplier(
+ MILLISECONDS.toMicros(1L), SECONDS.toMicros(2L),
+ MILLISECONDS.toMicros(1L), MILLISECONDS.toMicros(300L), SECONDS.toMicros(1L),
+ MILLISECONDS.toMicros(1L), MILLISECONDS.toMicros(300L), SECONDS.toMicros(1L)
+ );
+ NodeSink.Action[] actions = NodeSink.Action.values();
+ Supplier<Supplier<NodeSink.Action>> actionSupplier = () -> random.randomWeightedPicker(actions);
+ return () -> {
+ OverrideLinkKind kind = nextKind.get();
+ switch (kind)
+ {
+ default: throw new AssertionError("Unhandled: " + kind);
+ case BOTH: return ignore -> new Link(actionSupplier.get(), latencySupplier.get());
+ case ACTION: return override -> new Link(actionSupplier.get(), override.latencyMicros);
+ case LATENCY: return override -> new Link(override.action, latencySupplier.get());
+ }
+ };
+ }
+
+ enum OverrideLinksKind { NONE, PAIRED_UNIDIRECTIONAL, RANDOM_UNIDIRECTIONAL, RANDOM_BIDIRECTIONAL }
+
+ private Function<List<Id>, BiFunction<Id, Id, Link>> overrideLinks(RandomSource random, IntSupplier rf, BiFunction<Id, Id, Link> defaultLinks)
+ {
+ Supplier<Function<Link, Link>> linkOverrideSupplier = linkOverrideSupplier(random);
+ BooleanSupplier partitionChance = random.biasedUniformBools(random.nextFloat());
+ Supplier<OverrideLinksKind> nextKind = random.randomWeightedPicker(OverrideLinksKind.values());
+ return nodesList -> {
+ BiFunction<Id, Id, Link> links = defaultLinks;
+ if (partitionChance.getAsBoolean()) // 50% chance of a whole network partition
+ links = partition(nodesList, random, rf.getAsInt(), links);
+
+ OverrideLinksKind kind = nextKind.get();
+ if (kind == NONE)
+ return links;
+
+ Function<Link, Link> linkOverride = linkOverrideSupplier.get();
+ switch (kind)
+ {
+ default: throw new AssertionError("Unhandled: " + kind);
+ case PAIRED_UNIDIRECTIONAL:
+ return pairedUnidirectionalOverrides(linkOverride, nodesList, random, defaultLinks);
+ case RANDOM_BIDIRECTIONAL:
+ case RANDOM_UNIDIRECTIONAL:
+ boolean bidirectional = kind == RANDOM_BIDIRECTIONAL;
+ int count = random.nextInt(bidirectional || random.nextBoolean() ? nodesList.size() : (nodesList.size() * nodesList.size())/2);
+ return randomOverrides(bidirectional, linkOverride, count, nodesList, random, defaultLinks);
... 750 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org