You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by be...@apache.org on 2024/11/26 18:04:57 UTC
(cassandra-accord) branch trunk updated: Fix unit tests and improve burn test stability
This is an automated email from the ASF dual-hosted git repository.
benedict pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git
The following commit(s) were added to refs/heads/trunk by this push:
new 61f4ebd2 Fix unit tests and improve burn test stability
61f4ebd2 is described below
commit 61f4ebd21ff34b0807081d9df442236d8c334b52
Author: Benedict Elliott Smith <be...@apache.org>
AuthorDate: Tue Nov 12 16:20:13 2024 +0000
Fix unit tests and improve burn test stability
patch by Benedict; reviewed by Alex Petrov for CASSANDRA-20113
---
.../src/main/java/accord/api/ProgressLog.java | 6 +-
.../accord/coordinate/CoordinateShardDurable.java | 2 +-
.../accord/coordinate/CoordinateSyncPoint.java | 10 +-
.../accord/coordinate/CoordinationAdapter.java | 18 +-
.../java/accord/coordinate/ExecuteSyncPoint.java | 12 +-
.../java/accord/coordinate/FetchMaxConflict.java | 2 +-
.../main/java/accord/coordinate/FindSomeRoute.java | 2 +-
.../src/main/java/accord/coordinate/Infer.java | 4 +-
.../main/java/accord/coordinate/MaybeRecover.java | 11 +-
.../src/main/java/accord/coordinate/Persist.java | 15 +-
.../java/accord/coordinate/PersistSyncPoint.java | 4 +-
.../main/java/accord/coordinate/PersistTxn.java | 5 +-
.../src/main/java/accord/coordinate/Recover.java | 7 +-
.../java/accord/coordinate/RecoverWithRoute.java | 11 +-
.../java/accord/coordinate/SynchronousAwait.java | 11 +-
.../coordinate/tracking/FastPathTracker.java | 118 +++---
.../java/accord/impl/AbstractSafeCommandStore.java | 209 +++++-----
.../java/accord/impl/DefaultLocalListeners.java | 2 +-
.../java/accord/impl/DefaultRemoteListeners.java | 5 +
.../java/accord/impl/DurabilityScheduling.java | 2 +-
.../java/accord/impl/InMemoryCommandStore.java | 110 +++---
.../java/accord/impl/progresslog/BaseTxnState.java | 15 +-
.../impl/progresslog/DefaultProgressLog.java | 18 +-
.../impl/progresslog/DefaultProgressLogs.java | 2 +-
.../java/accord/impl/progresslog/HomeState.java | 8 +-
.../java/accord/impl/progresslog/TxnState.java | 5 +
.../java/accord/impl/progresslog/WaitingState.java | 4 +-
.../src/main/java/accord/local/Bootstrap.java | 4 +-
.../src/main/java/accord/local/Cleanup.java | 170 +++++----
.../src/main/java/accord/local/CommandStore.java | 5 +-
.../src/main/java/accord/local/Commands.java | 60 ++-
.../src/main/java/accord/local/PreLoadContext.java | 7 +
.../main/java/accord/local/RedundantBefore.java | 98 ++++-
.../main/java/accord/local/RedundantStatus.java | 83 +++-
.../main/java/accord/local/SafeCommandStore.java | 28 +-
.../main/java/accord/local/StoreParticipants.java | 14 +-
.../main/java/accord/local/cfk/CommandsForKey.java | 194 ++++++----
.../src/main/java/accord/local/cfk/NotifySink.java | 4 +-
.../main/java/accord/local/cfk/PostProcess.java | 102 +++--
.../src/main/java/accord/local/cfk/Pruning.java | 26 +-
.../src/main/java/accord/local/cfk/Updating.java | 114 +++---
.../src/main/java/accord/local/cfk/Utils.java | 8 +-
.../src/main/java/accord/messages/Apply.java | 30 +-
.../accord/messages/ApplyThenWaitUntilApplied.java | 8 +-
.../src/main/java/accord/messages/Await.java | 37 +-
.../main/java/accord/messages/BeginRecovery.java | 16 +-
.../src/main/java/accord/messages/CheckStatus.java | 13 +-
.../src/main/java/accord/messages/PreAccept.java | 3 +
.../src/main/java/accord/messages/Propagate.java | 36 +-
.../src/main/java/accord/messages/ReadData.java | 21 +-
.../java/accord/messages/WaitUntilApplied.java | 15 +-
.../java/accord/primitives/AbstractRanges.java | 42 +--
.../src/main/java/accord/primitives/KeyRoute.java | 31 ++
.../src/main/java/accord/primitives/Known.java | 6 +-
.../main/java/accord/primitives/LatestDeps.java | 1 -
.../src/main/java/accord/primitives/RangeDeps.java | 2 +-
.../main/java/accord/primitives/RangeRoute.java | 19 +
.../src/main/java/accord/primitives/Ranges.java | 22 ++
.../src/main/java/accord/primitives/Route.java | 2 +
.../src/main/java/accord/primitives/Status.java | 5 +
.../src/main/java/accord/primitives/Timestamp.java | 2 +-
.../src/main/java/accord/primitives/Txn.java | 17 +-
.../src/main/java/accord/primitives/TxnId.java | 5 +
.../src/main/java/accord/topology/Topologies.java | 40 +-
.../src/main/java/accord/topology/Topology.java | 9 +-
.../main/java/accord/topology/TopologyManager.java | 26 +-
accord-core/src/main/java/accord/utils/Base62.java | 57 +++
.../java/accord/utils/ThreadPoolScheduler.java | 2 +-
.../src/main/java/accord/utils/Timestamped.java | 5 +
accord-core/src/test/java/accord/KeysTest.java | 3 +-
.../src/test/java/accord/burn/BurnTest.java | 54 +--
.../accord/burn/BurnTestConfigurationService.java | 5 +-
.../accord/coordinate/PreAcceptTrackerTest.java | 36 +-
.../tracking/FastPathTrackerReconciler.java | 5 +-
.../coordinate/tracking/ReadTrackerReconciler.java | 1 +
.../coordinate/tracking/ReadTrackerTest.java | 1 +
.../coordinate/tracking/TrackerReconciler.java | 2 +-
.../test/java/accord/impl/RemoteListenersTest.java | 2 +-
.../src/test/java/accord/impl/basic/Cluster.java | 420 ++++++++++++++++-----
.../accord/impl/basic/DelayedCommandStores.java | 116 +++---
.../src/test/java/accord/impl/basic/Journal.java | 174 ++++++---
.../src/test/java/accord/impl/basic/NodeSink.java | 66 ++--
.../src/test/java/accord/impl/basic/Packet.java | 21 +-
.../src/test/java/accord/impl/basic/Pending.java | 22 ++
.../java/accord/impl/basic/PendingRunnable.java | 18 +
.../java/accord/impl/basic/RandomDelayQueue.java | 8 +-
.../impl/basic/RecurringPendingRunnable.java | 16 +-
.../accord/impl/basic/TaskExecutorService.java | 6 +
.../src/test/java/accord/impl/list/ListAgent.java | 20 +-
.../accord/impl/list/ListFetchCoordinator.java | 2 +-
.../test/java/accord/impl/list/ListRequest.java | 10 +-
.../src/test/java/accord/impl/list/ListResult.java | 2 +-
.../src/test/java/accord/impl/list/ListStore.java | 92 +++--
.../src/test/java/accord/impl/list/ListWrite.java | 4 +-
.../test/java/accord/impl/mock/MockCluster.java | 6 +-
.../src/test/java/accord/local/CommandsTest.java | 7 +-
.../java/accord/local/ImmutableCommandTest.java | 26 +-
.../java/accord/local/cfk/CommandsForKeyTest.java | 4 +-
.../test/java/accord/messages/PreAcceptTest.java | 2 +-
.../test/java/accord/messages/ReadDataTest.java | 5 +-
.../java/accord/primitives/AbstractRangesTest.java | 6 +
.../test/java/accord/primitives/RangeDepsTest.java | 4 +-
.../java/accord/topology/TopologyManagerTest.java | 8 +-
.../java/accord/topology/TopologyRandomizer.java | 27 +-
.../test/java/accord/topology/TopologyTest.java | 8 +-
.../src/test/java/accord/utils/AccordGens.java | 1 -
.../test/java/accord/utils/LogGroupTimersTest.java | 3 +-
.../test/java/accord/utils/SortedArraysTest.java | 45 ++-
.../verify/StrictSerializabilityVerifier.java | 2 +-
109 files changed, 2167 insertions(+), 1100 deletions(-)
diff --git a/accord-core/src/main/java/accord/api/ProgressLog.java b/accord-core/src/main/java/accord/api/ProgressLog.java
index 3f1263cb..8ce8e4f7 100644
--- a/accord-core/src/main/java/accord/api/ProgressLog.java
+++ b/accord-core/src/main/java/accord/api/ProgressLog.java
@@ -25,6 +25,7 @@ import accord.local.CommandStore;
import accord.local.Node;
import accord.local.SafeCommand;
import accord.local.SafeCommandStore;
+import accord.local.StoreParticipants;
import accord.primitives.SaveStatus;
import accord.primitives.Status.Durability;
import accord.primitives.Participants;
@@ -177,8 +178,9 @@ public interface ProgressLog
* @param blockedBy is the transaction id that is blocking progress
* @param blockedOnRoute the route (if any) we are blocked on execution for
* @param blockedOnParticipants the participating keys on which we are blocked for execution
+ * @param participants
*/
- void waiting(BlockedUntil blockedUntil, SafeCommandStore safeStore, SafeCommand blockedBy, @Nullable Route<?> blockedOnRoute, @Nullable Participants<?> blockedOnParticipants);
+ void waiting(BlockedUntil blockedUntil, SafeCommandStore safeStore, SafeCommand blockedBy, @Nullable Route<?> blockedOnRoute, @Nullable Participants<?> blockedOnParticipants, @Nullable StoreParticipants participants);
/**
* We have finished processing this transaction; ensure its state is cleared
@@ -196,7 +198,7 @@ public interface ProgressLog
{
@Override public void update(SafeCommandStore safeStore, TxnId txnId, Command before, Command after) {}
@Override public void remoteCallback(SafeCommandStore safeStore, SafeCommand safeCommand, SaveStatus remoteStatus, int callbackId, Node.Id from) {}
- @Override public void waiting(BlockedUntil blockedUntil, SafeCommandStore safeStore, SafeCommand blockedBy, Route<?> blockedOnRoute, Participants<?> blockedOnParticipants) {}
+ @Override public void waiting(BlockedUntil blockedUntil, SafeCommandStore safeStore, SafeCommand blockedBy, Route<?> blockedOnRoute, Participants<?> blockedOnParticipants, StoreParticipants participants) {}
@Override public void clear(TxnId txnId) {}
@Override public void clear() {}
}
diff --git a/accord-core/src/main/java/accord/coordinate/CoordinateShardDurable.java b/accord-core/src/main/java/accord/coordinate/CoordinateShardDurable.java
index 63a0c3eb..e6892529 100644
--- a/accord-core/src/main/java/accord/coordinate/CoordinateShardDurable.java
+++ b/accord-core/src/main/java/accord/coordinate/CoordinateShardDurable.java
@@ -40,7 +40,7 @@ public class CoordinateShardDurable extends ExecuteExclusiveSyncPoint implements
addCallback((success, fail) -> {
if (fail == null)
{
- node.configService().reportEpochRedundant(syncPoint.route.toRanges(), syncPoint.syncId.epoch());
+ node.configService().reportEpochRedundant(syncPoint.route.toRanges(), syncPoint.syncId.epoch() - 1);
node.send(tracker.nodes(), new SetShardDurable(syncPoint));
}
});
diff --git a/accord-core/src/main/java/accord/coordinate/CoordinateSyncPoint.java b/accord-core/src/main/java/accord/coordinate/CoordinateSyncPoint.java
index 6f271cec..fc734b44 100644
--- a/accord-core/src/main/java/accord/coordinate/CoordinateSyncPoint.java
+++ b/accord-core/src/main/java/accord/coordinate/CoordinateSyncPoint.java
@@ -44,6 +44,7 @@ import accord.utils.async.AsyncResults;
import static accord.coordinate.ExecutePath.FAST;
import static accord.coordinate.Propose.Invalidate.proposeAndCommitInvalidate;
+import static accord.messages.Apply.executes;
import static accord.primitives.Timestamp.mergeMax;
import static accord.primitives.Txn.Kind.ExclusiveSyncPoint;
import static accord.utils.Functions.foldl;
@@ -172,11 +173,18 @@ public class CoordinateSyncPoint<U extends Unseekable> extends CoordinatePreAcce
}
public static void sendApply(Node node, Node.Id to, SyncPoint<?> syncPoint)
+ {
+ // TODO (required): consider, document and add invariants checking if this topologies is correct in all cases
+ // (notably ExclusiveSyncPoints should execute in earlier epochs for durability, but not for fetching )
+ Topologies executes = executes(node, syncPoint.route, syncPoint.syncId);
+ sendApply(node, to, syncPoint, executes);
+ }
+ public static void sendApply(Node node, Node.Id to, SyncPoint<?> syncPoint, Topologies executes)
{
TxnId txnId = syncPoint.syncId;
Timestamp executeAt = txnId;
Txn txn = node.agent().emptySystemTxn(txnId.kind(), txnId.domain());
Deps deps = syncPoint.waitFor;
- Apply.sendMaximal(node, to, txnId, syncPoint.route(), txn, executeAt, deps, txn.execute(txnId, executeAt, null), txn.result(txnId, executeAt, null));
+ Apply.sendMaximal(node, to, executes, txnId, syncPoint.route(), txn, executeAt, deps, txn.execute(txnId, executeAt, null), txn.result(txnId, executeAt, null), syncPoint.route());
}
}
diff --git a/accord-core/src/main/java/accord/coordinate/CoordinationAdapter.java b/accord-core/src/main/java/accord/coordinate/CoordinationAdapter.java
index 3c52d0a2..66d61cba 100644
--- a/accord-core/src/main/java/accord/coordinate/CoordinationAdapter.java
+++ b/accord-core/src/main/java/accord/coordinate/CoordinationAdapter.java
@@ -34,6 +34,7 @@ import accord.primitives.Ballot;
import accord.primitives.Deps;
import accord.primitives.FullRoute;
import accord.primitives.Participants;
+import accord.primitives.Route;
import accord.primitives.SyncPoint;
import accord.primitives.Timestamp;
import accord.primitives.Txn;
@@ -59,7 +60,7 @@ public interface CoordinationAdapter<R>
void propose(Node node, @Nullable Topologies preaccept, FullRoute<?> route, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer<? super R, Throwable> callback);
void stabilise(Node node, @Nullable Topologies any, FullRoute<?> route, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer<? super R, Throwable> callback);
void execute(Node node, @Nullable Topologies any, FullRoute<?> route, ExecutePath path, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer<? super R, Throwable> callback);
- void persist(Node node, @Nullable Topologies any, FullRoute<?> route, Participants<?> sendTo, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, BiConsumer<? super R, Throwable> callback);
+ void persist(Node node, @Nullable Topologies any, FullRoute<?> route, Route<?> sendTo, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, BiConsumer<? super R, Throwable> callback);
default void persist(Node node, @Nullable Topologies any, FullRoute<?> route, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, BiConsumer<? super R, Throwable> callback)
{
persist(node, any, route, route, txnId, txn, executeAt, deps, writes, result, callback);
@@ -159,13 +160,12 @@ public interface CoordinationAdapter<R>
public static final StandardTxnAdapter INSTANCE = new StandardTxnAdapter();
@Override
- public void persist(Node node, Topologies any, FullRoute<?> route, Participants<?> sendTo, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, BiConsumer<? super Result, Throwable> callback)
+ public void persist(Node node, Topologies any, FullRoute<?> route, Route<?> sendTo, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, BiConsumer<? super Result, Throwable> callback)
{
- // TODO (required): we aren't using sendTo
Topologies all = execution(node, any, sendTo, txnId, executeAt);
if (callback != null) callback.accept(result, null);
- new PersistTxn(node, all, txnId, route, txn, executeAt, deps, writes, result)
+ new PersistTxn(node, all, txnId, sendTo, txn, executeAt, deps, writes, result, route)
.start(Apply.FACTORY, Minimal, all, writes, result);
}
}
@@ -174,12 +174,12 @@ public interface CoordinationAdapter<R>
{
public static final RecoveryTxnAdapter INSTANCE = new RecoveryTxnAdapter();
@Override
- public void persist(Node node, Topologies any, FullRoute<?> route, Participants<?> participants, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, BiConsumer<? super Result, Throwable> callback)
+ public void persist(Node node, Topologies any, FullRoute<?> route, Route<?> sendTo, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, BiConsumer<? super Result, Throwable> callback)
{
- Topologies all = execution(node, any, participants, txnId, executeAt);
+ Topologies all = execution(node, any, sendTo, txnId, executeAt);
if (callback != null) callback.accept(result, null);
- new PersistTxn(node, all, txnId, route, txn, executeAt, deps, writes, result)
+ new PersistTxn(node, all, txnId, sendTo, txn, executeAt, deps, writes, result, route)
.start(Apply.FACTORY, Maximal, all, writes, result);
}
}
@@ -224,7 +224,7 @@ public interface CoordinationAdapter<R>
}
@Override
- public void persist(Node node, Topologies any, FullRoute<?> route, Participants<?> participants, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, BiConsumer<? super SyncPoint<U>, Throwable> callback)
+ public void persist(Node node, Topologies any, FullRoute<?> route, Route<?> participants, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, BiConsumer<? super SyncPoint<U>, Throwable> callback)
{
Topologies all = forExecution(node, route, txnId, executeAt, deps);
@@ -335,7 +335,7 @@ public interface CoordinationAdapter<R>
}
@Override
- public void persist(Node node, Topologies any, FullRoute<?> route, Participants<?> participants, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, BiConsumer<? super SyncPoint<U>, Throwable> callback)
+ public void persist(Node node, Topologies any, FullRoute<?> route, Route<?> participants, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, BiConsumer<? super SyncPoint<U>, Throwable> callback)
{
throw new UnsupportedOperationException();
}
diff --git a/accord-core/src/main/java/accord/coordinate/ExecuteSyncPoint.java b/accord-core/src/main/java/accord/coordinate/ExecuteSyncPoint.java
index aa98e713..dc010351 100644
--- a/accord-core/src/main/java/accord/coordinate/ExecuteSyncPoint.java
+++ b/accord-core/src/main/java/accord/coordinate/ExecuteSyncPoint.java
@@ -18,13 +18,15 @@
package accord.coordinate;
+import java.util.LinkedHashMap;
+import java.util.Map;
import java.util.function.Function;
import accord.api.Result;
import accord.coordinate.CoordinationAdapter.Adapters;
-import accord.coordinate.tracking.SimpleTracker;
import accord.coordinate.tracking.QuorumTracker;
import accord.coordinate.tracking.RequestStatus;
+import accord.coordinate.tracking.SimpleTracker;
import accord.local.Node;
import accord.messages.ApplyThenWaitUntilApplied;
import accord.messages.Callback;
@@ -134,6 +136,7 @@ public abstract class ExecuteSyncPoint<U extends Unseekable> extends SettableRes
final Function<Topologies, SimpleTracker<?>> trackerSupplier;
final SimpleTracker<?> tracker;
private Throwable failures = null;
+ final Map<Node.Id, Object> debug = Invariants.debug() ? new LinkedHashMap<>() : null;
ExecuteSyncPoint(Node node, SyncPoint<U> syncPoint, SimpleTracker<?> tracker)
{
@@ -162,6 +165,8 @@ public abstract class ExecuteSyncPoint<U extends Unseekable> extends SettableRes
public synchronized void onSuccess(Node.Id from, ReadReply reply)
{
if (isDone()) return;
+ if (debug != null)
+ debug.put(from, reply);
if (!reply.isOk())
{
@@ -170,7 +175,7 @@ public abstract class ExecuteSyncPoint<U extends Unseekable> extends SettableRes
default: throw new AssertionError("Unhandled: " + reply);
case Insufficient:
- CoordinateSyncPoint.sendApply(node, from, syncPoint);
+ CoordinateSyncPoint.sendApply(node, from, syncPoint, tracker.topologies());
return;
case Redundant:
@@ -199,6 +204,9 @@ public abstract class ExecuteSyncPoint<U extends Unseekable> extends SettableRes
public synchronized void onFailure(Node.Id from, Throwable failure)
{
if (isDone()) return;
+ if (debug != null)
+ debug.put(from, failure);
+
failures = FailureAccumulator.append(failures, failure);
if (tracker.recordFailure(from) == RequestStatus.Failed)
tryFailure(FailureAccumulator.createFailure(failures, syncPoint.syncId, syncPoint.route.homeKey()));
diff --git a/accord-core/src/main/java/accord/coordinate/FetchMaxConflict.java b/accord-core/src/main/java/accord/coordinate/FetchMaxConflict.java
index 9fe651e1..b2038f2a 100644
--- a/accord-core/src/main/java/accord/coordinate/FetchMaxConflict.java
+++ b/accord-core/src/main/java/accord/coordinate/FetchMaxConflict.java
@@ -39,7 +39,7 @@ import static accord.coordinate.tracking.RequestStatus.Success;
* Perform initial rounds of PreAccept and Accept until we have reached agreement about when we should execute.
* If we are preempted by a recovery coordinator, we abort and let them complete (and notify us about the execution result)
*
- * TODO (desired, testing): dedicated burn test to validate outcomes
+ * TODO (expected): we don't need this anymore, we can just use the local MaxConflicts collection when creating a snapshot for streaming
*/
public class FetchMaxConflict extends AbstractCoordinatePreAccept<Timestamp, GetMaxConflictOk>
{
diff --git a/accord-core/src/main/java/accord/coordinate/FindSomeRoute.java b/accord-core/src/main/java/accord/coordinate/FindSomeRoute.java
index 7f9c7eda..9e8d5f14 100644
--- a/accord-core/src/main/java/accord/coordinate/FindSomeRoute.java
+++ b/accord-core/src/main/java/accord/coordinate/FindSomeRoute.java
@@ -80,6 +80,6 @@ public class FindSomeRoute extends CheckShards<Participants<?>>
{
if (failure != null) callback.accept(null, failure);
else if (merged == null) callback.accept(new Result(null, Nothing, success.withQuorum), null);
- else callback.accept(new Result(merged.route, merged.finish(this.route, success.withQuorum).knownFor(txnId, this.route, this.route), success.withQuorum), null);
+ else callback.accept(new Result(merged.route, merged.finish(this.route, this.route, success.withQuorum).knownFor(txnId, this.route, this.route), success.withQuorum), null);
}
}
diff --git a/accord-core/src/main/java/accord/coordinate/Infer.java b/accord-core/src/main/java/accord/coordinate/Infer.java
index 787e45a7..81c935f5 100644
--- a/accord-core/src/main/java/accord/coordinate/Infer.java
+++ b/accord-core/src/main/java/accord/coordinate/Infer.java
@@ -41,7 +41,7 @@ import static accord.primitives.Status.PreCommitted;
import static accord.primitives.Route.castToRoute;
import static accord.primitives.Route.isRoute;
-// TODO (required): dedicated randomised testing of all inferences
+// TODO (testing): dedicated randomised testing of all inferences
public class Infer
{
public enum InvalidIf
@@ -73,7 +73,7 @@ public class Infer
public boolean inferInvalidWithQuorum(Known known)
{
- return this == IfUncommitted && !known.isDecided();
+ return this == IfUncommitted && (!known.isDecided() || !known.definition.isOrWasKnown());
}
}
diff --git a/accord-core/src/main/java/accord/coordinate/MaybeRecover.java b/accord-core/src/main/java/accord/coordinate/MaybeRecover.java
index 34de22f1..2c7e1fe3 100644
--- a/accord-core/src/main/java/accord/coordinate/MaybeRecover.java
+++ b/accord-core/src/main/java/accord/coordinate/MaybeRecover.java
@@ -20,6 +20,7 @@ package accord.coordinate;
import java.util.function.BiConsumer;
+import accord.messages.Commit;
import accord.messages.InformDurable;
import accord.primitives.*;
import accord.utils.Invariants;
@@ -57,7 +58,8 @@ public class MaybeRecover extends CheckShards<Route<?>>
@Override
protected boolean isSufficient(CheckStatusOk ok)
{
- return hasMadeProgress(ok) || ok.durability.isDurableOrInvalidated();
+ // We don't accept a single truncated response - must have a quorum so we can make inferences about invalidation
+ return !ok.isTruncatedResponse() && (hasMadeProgress(ok) || ok.durability.isDurableOrInvalidated());
}
public boolean hasMadeProgress(CheckStatusOk ok)
@@ -70,6 +72,8 @@ public class MaybeRecover extends CheckShards<Route<?>>
@Override
protected void onDone(Success success, Throwable fail)
{
+ // TODO (desired): we don't need a full quorum to proceed, just a quorum that intersects a full quorum (i.e. when rf=2, we need only 1 reply)
+ // this can be helpful in mitigating flakiness and helping forward progress for large transactions spanning many shards
if (fail != null)
{
callback.accept(null, fail);
@@ -77,7 +81,7 @@ public class MaybeRecover extends CheckShards<Route<?>>
else
{
Invariants.checkState(merged != null);
- CheckStatusOk full = merged.finish(this.route, success.withQuorum);
+ CheckStatusOk full = merged.finish(this.route, this.route, success.withQuorum);
Known known = full.maxKnown();
Route<?> someRoute = full.route;
@@ -120,12 +124,13 @@ public class MaybeRecover extends CheckShards<Route<?>>
case WasApply:
case Erased:
+ // TODO (required): if we're home replica, don't want to cancel coordination without either invalidating or applying unless we're stale
// TODO (required): on Erased, should we maybe mark stale, or leave to FetchData?
callback.accept(full.toProgressToken(), null);
break;
case Invalidated:
- // TODO (required): revisit lowEpoch here
+ Commit.Invalidate.commitInvalidate(node, txnId, Route.merge(full.route, (Route)route), txnId.epoch());
locallyInvalidateAndCallback(node, txnId, txnId, txnId, someRoute, full.toProgressToken(), callback);
break;
}
diff --git a/accord-core/src/main/java/accord/coordinate/Persist.java b/accord-core/src/main/java/accord/coordinate/Persist.java
index 042cb228..1082f631 100644
--- a/accord-core/src/main/java/accord/coordinate/Persist.java
+++ b/accord-core/src/main/java/accord/coordinate/Persist.java
@@ -31,6 +31,7 @@ import accord.messages.Callback;
import accord.messages.InformDurable;
import accord.primitives.Deps;
import accord.primitives.FullRoute;
+import accord.primitives.Route;
import accord.primitives.Timestamp;
import accord.primitives.Txn;
import accord.primitives.TxnId;
@@ -46,27 +47,29 @@ public abstract class Persist implements Callback<ApplyReply>
{
protected final Node node;
protected final TxnId txnId;
- protected final FullRoute<?> route;
+ protected final Route<?> sendTo;
protected final Txn txn;
protected final Timestamp executeAt;
protected final Deps stableDeps;
protected final Writes writes;
protected final Result result;
+ protected final FullRoute<?> route;
protected final Topologies topologies;
protected final QuorumTracker tracker;
protected final Set<Id> persistedOn;
boolean isDone;
- protected Persist(Node node, Topologies all, TxnId txnId, FullRoute<?> route, Txn txn, Timestamp executeAt, Deps stableDeps, Writes writes, Result result)
+ protected Persist(Node node, Topologies all, TxnId txnId, Route<?> sendTo, Txn txn, Timestamp executeAt, Deps stableDeps, Writes writes, Result result, FullRoute<?> route)
{
this.node = node;
this.txnId = txnId;
- this.route = route;
+ this.sendTo = sendTo;
this.txn = txn;
this.executeAt = executeAt;
this.stableDeps = stableDeps;
this.writes = writes;
this.result = result;
+ this.route = route;
this.topologies = all;
this.tracker = new QuorumTracker(all);
this.persistedOn = new HashSet<>();
@@ -81,7 +84,7 @@ public abstract class Persist implements Callback<ApplyReply>
case Redundant:
case Applied:
persistedOn.add(from);
- if (tracker.recordSuccess(from) == Success)
+ if (sendTo == route && tracker.recordSuccess(from) == Success)
{
if (!isDone)
{
@@ -96,7 +99,7 @@ public abstract class Persist implements Callback<ApplyReply>
}
break;
case Insufficient:
- Apply.sendMaximal(node, from, txnId, route, txn, executeAt, stableDeps, writes, result);
+ Apply.sendMaximal(node, from, txnId, route, txn, executeAt, stableDeps, writes, result, route);
}
}
@@ -124,7 +127,7 @@ public abstract class Persist implements Callback<ApplyReply>
}
else
{
- node.send(contact, to -> factory.create(kind, to, all, txnId, route, txn, executeAt, stableDeps, writes, result), this);
+ node.send(contact, to -> factory.create(kind, to, all, txnId, sendTo, txn, executeAt, stableDeps, writes, result, route), this);
}
}
}
diff --git a/accord-core/src/main/java/accord/coordinate/PersistSyncPoint.java b/accord-core/src/main/java/accord/coordinate/PersistSyncPoint.java
index f0be6f96..2eec31b6 100644
--- a/accord-core/src/main/java/accord/coordinate/PersistSyncPoint.java
+++ b/accord-core/src/main/java/accord/coordinate/PersistSyncPoint.java
@@ -34,7 +34,7 @@ public class PersistSyncPoint extends Persist
{
public PersistSyncPoint(Node node, Topologies topologies, TxnId txnId, FullRoute<?> route, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result)
{
- super(node, topologies, txnId, route, txn, executeAt, deps, writes, result);
+ super(node, topologies, txnId, route, txn, executeAt, deps, writes, result, route);
}
@Override
@@ -50,7 +50,7 @@ public class PersistSyncPoint extends Persist
{
for (Node.Id to : contact)
{
- Apply apply = factory.create(kind, to, all, txnId, route, txn, executeAt, stableDeps, writes, result);
+ Apply apply = factory.create(kind, to, all, txnId, route, txn, executeAt, stableDeps, writes, result, route);
if (apply == null)
tracker.recordSuccess(to);
else
diff --git a/accord-core/src/main/java/accord/coordinate/PersistTxn.java b/accord-core/src/main/java/accord/coordinate/PersistTxn.java
index b2db464c..797e8363 100644
--- a/accord-core/src/main/java/accord/coordinate/PersistTxn.java
+++ b/accord-core/src/main/java/accord/coordinate/PersistTxn.java
@@ -22,6 +22,7 @@ import accord.api.Result;
import accord.local.Node;
import accord.primitives.Deps;
import accord.primitives.FullRoute;
+import accord.primitives.Route;
import accord.primitives.Timestamp;
import accord.primitives.Txn;
import accord.primitives.TxnId;
@@ -30,8 +31,8 @@ import accord.topology.Topologies;
public class PersistTxn extends Persist
{
- public PersistTxn(Node node, Topologies topologies, TxnId txnId, FullRoute<?> route, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result)
+ public PersistTxn(Node node, Topologies topologies, TxnId txnId, Route<?> sendTo, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, FullRoute<?> route)
{
- super(node, topologies, txnId, route, txn, executeAt, deps, writes, result);
+ super(node, topologies, txnId, sendTo, txn, executeAt, deps, writes, result, route);
}
}
diff --git a/accord-core/src/main/java/accord/coordinate/Recover.java b/accord-core/src/main/java/accord/coordinate/Recover.java
index dcbf9417..568cdf7a 100644
--- a/accord-core/src/main/java/accord/coordinate/Recover.java
+++ b/accord-core/src/main/java/accord/coordinate/Recover.java
@@ -70,6 +70,8 @@ import static accord.utils.Invariants.illegalState;
// TODO (expected): do not wait for a quorum if we find enough information to execute
public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throwable>
{
+ // TODO (required): make sure we contact and wait (at least until timeout)
+ // for an ACK from any replica that witnessed a txn as something to wait for
AsyncResult<Object> awaitCommits(Node node, Deps waitOn)
{
AtomicInteger remaining = new AtomicInteger(waitOn.txnIdCount());
@@ -78,8 +80,9 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
{
TxnId txnId = waitOn.txnId(i);
Participants<?> participants = waitOn.participants(txnId);
- Topology topology = node.topology().globalForEpoch(txnId.epoch()).forSelection(participants);
- SynchronousAwait.awaitQuorum(node, topology, txnId, route.homeKey(), BlockedUntil.HasCommittedDeps, participants).addCallback((success, failure) -> {
+ long sinceEpoch = Math.min(txnId.epoch(), Recover.this.txnId.epoch());
+ Topologies topologies = this.tracker.topologies().select(participants, sinceEpoch);
+ SynchronousAwait.awaitQuorum(node, topologies, txnId, route.homeKey(), BlockedUntil.HasCommittedDeps, participants).addCallback((success, failure) -> {
if (result.isDone())
return;
if (failure == null && remaining.decrementAndGet() == 0)
diff --git a/accord-core/src/main/java/accord/coordinate/RecoverWithRoute.java b/accord-core/src/main/java/accord/coordinate/RecoverWithRoute.java
index c8fae845..f969a7e1 100644
--- a/accord-core/src/main/java/accord/coordinate/RecoverWithRoute.java
+++ b/accord-core/src/main/java/accord/coordinate/RecoverWithRoute.java
@@ -23,8 +23,6 @@ import javax.annotation.Nullable;
import accord.local.Node;
import accord.local.Node.Id;
-import accord.primitives.Status;
-import accord.primitives.Known;
import accord.messages.CheckStatus;
import accord.messages.CheckStatus.CheckStatusOk;
import accord.messages.CheckStatus.CheckStatusOkFull;
@@ -33,16 +31,16 @@ import accord.messages.Propagate;
import accord.primitives.Ballot;
import accord.primitives.Deps;
import accord.primitives.FullRoute;
-import accord.primitives.Participants;
+import accord.primitives.Known;
import accord.primitives.Ranges;
import accord.primitives.Route;
+import accord.primitives.Status;
import accord.primitives.Txn;
import accord.primitives.TxnId;
import accord.topology.Topologies;
import accord.utils.Invariants;
import static accord.coordinate.CoordinationAdapter.Factory.Step.InitiateRecovery;
-import static accord.primitives.Status.Durability.Majority;
import static accord.primitives.Known.KnownDeps.DepsKnown;
import static accord.primitives.Known.KnownExecuteAt.ExecuteAtKnown;
import static accord.primitives.Known.Outcome.Apply;
@@ -50,6 +48,7 @@ import static accord.primitives.ProgressToken.APPLIED;
import static accord.primitives.ProgressToken.INVALIDATED;
import static accord.primitives.ProgressToken.TRUNCATED_DURABLE_OR_INVALIDATED;
import static accord.primitives.Route.castToFullRoute;
+import static accord.primitives.Status.Durability.Majority;
import static accord.utils.Invariants.illegalState;
public class RecoverWithRoute extends CheckShards<FullRoute<?>>
@@ -141,7 +140,7 @@ public class RecoverWithRoute extends CheckShards<FullRoute<?>>
return;
}
- CheckStatusOkFull full = ((CheckStatusOkFull) this.merged).finish(route, success.withQuorum);
+ CheckStatusOkFull full = ((CheckStatusOkFull) this.merged).finish(route, route, success.withQuorum);
Known known = full.knownFor(txnId, route, route);
// TODO (required): audit this logic, and centralise with e.g. FetchData inferences
@@ -177,7 +176,7 @@ public class RecoverWithRoute extends CheckShards<FullRoute<?>>
{
// we might have only part of the full transaction, and a shard may have truncated;
// in this case we want to skip straight to apply, but only for the shards that haven't truncated
- Participants<?> sendTo = route.participants().without(full.truncatedResponse());
+ Route<?> sendTo = route.without(full.truncatedResponse());
if (!sendTo.isEmpty())
{
known = full.knownFor(txnId, sendTo, sendTo);
diff --git a/accord-core/src/main/java/accord/coordinate/SynchronousAwait.java b/accord-core/src/main/java/accord/coordinate/SynchronousAwait.java
index 6f76ff8b..a03f0935 100644
--- a/accord-core/src/main/java/accord/coordinate/SynchronousAwait.java
+++ b/accord-core/src/main/java/accord/coordinate/SynchronousAwait.java
@@ -32,7 +32,6 @@ import accord.messages.Callback;
import accord.primitives.Participants;
import accord.primitives.TxnId;
import accord.topology.Topologies;
-import accord.topology.Topology;
import accord.utils.async.AsyncResults;
import static accord.coordinate.tracking.RequestStatus.Failed;
@@ -60,17 +59,17 @@ public class SynchronousAwait extends AsyncResults.SettableResult<Void> implemen
this.tracker = tracker;
}
- public static SynchronousAwait awaitQuorum(Node node, Topology topology, TxnId txnId, @Nullable RoutingKey homeKey, BlockedUntil awaiting, Participants<?> participants)
+ public static SynchronousAwait awaitQuorum(Node node, Topologies topologies, TxnId txnId, @Nullable RoutingKey homeKey, BlockedUntil awaiting, Participants<?> participants)
{
- QuorumTracker tracker = new QuorumTracker(new Topologies.Single(node.topology().sorter(), topology));
+ QuorumTracker tracker = new QuorumTracker(topologies);
SynchronousAwait result = new SynchronousAwait(txnId, homeKey, tracker);
- result.start(node, topology, participants, awaiting);
+ result.start(node, topologies, participants, awaiting);
return result;
}
- private void start(Node node, Topology topology, Participants<?> participants, BlockedUntil blockedUntil)
+ private void start(Node node, Topologies topologies, Participants<?> participants, BlockedUntil blockedUntil)
{
- node.send(topology.nodes(), to -> new Await(to, topology, txnId, participants, blockedUntil), this);
+ node.send(topologies.nodes(), to -> new Await(to, topologies, txnId, participants, blockedUntil), this);
}
@Override
diff --git a/accord-core/src/main/java/accord/coordinate/tracking/FastPathTracker.java b/accord-core/src/main/java/accord/coordinate/tracking/FastPathTracker.java
index d42e198a..271b1395 100644
--- a/accord-core/src/main/java/accord/coordinate/tracking/FastPathTracker.java
+++ b/accord-core/src/main/java/accord/coordinate/tracking/FastPathTracker.java
@@ -22,6 +22,8 @@ import accord.coordinate.tracking.QuorumTracker.QuorumShardTracker;
import accord.local.Node;
import accord.topology.Shard;
import accord.topology.Topologies;
+import accord.utils.Invariants;
+
import com.google.common.annotations.VisibleForTesting;
import javax.annotation.Nonnull;
@@ -43,6 +45,7 @@ public class FastPathTracker extends PreAcceptTracker<FastPathTracker.FastPathSh
protected int fastPathAccepts;
protected int fastPathFailures;
protected int fastPathDelayed;
+ protected boolean complete;
public FastPathShardTracker(Shard shard)
{
@@ -51,19 +54,43 @@ public class FastPathTracker extends PreAcceptTracker<FastPathTracker.FastPathSh
// return NewQuorumSuccess ONLY once fast path is rejected
public abstract ShardOutcome<? super FastPathTracker> onQuorumSuccess(Node.Id node);
- public abstract ShardOutcome<? super FastPathTracker> onMaybeFastPathSuccess(Node.Id node);
+ final ShardOutcome<? super FastPathTracker> complete(ShardOutcome<? super FastPathTracker> result)
+ {
+ Invariants.checkState(!complete);
+ complete = true;
+ return result;
+ }
+
+ public ShardOutcome<? super FastPathTracker> onMaybeFastPathSuccess(Node.Id node)
+ {
+ if (complete)
+ return NoChange;
+
+ ++successes;
+ if (shard.fastPathElectorate.contains(node))
+ {
+ ++fastPathAccepts;
+ if (hasMetFastPathCriteria())
+ return complete(NewFastPathSuccess);
+ }
+
+ return quorumIfHasRejectedFastPath();
+ }
public final ShardOutcome<? super FastPathTracker> onFailure(@Nonnull Node.Id from)
{
+ if (complete)
+ return NoChange;
+
if (++failures > shard.maxFailures)
- return Fail;
+ return complete(Fail);
if (shard.fastPathElectorate.contains(from))
{
++fastPathFailures;
- if (isNewFastPathReject() && hasReachedQuorum())
- return Success;
+ if (hasRejectedFastPath() && hasReachedQuorum())
+ return complete(Success);
}
return NoChange;
@@ -71,40 +98,28 @@ public class FastPathTracker extends PreAcceptTracker<FastPathTracker.FastPathSh
public final ShardOutcome<? super FastPathTracker> onDelayed(@Nonnull Node.Id from)
{
+ if (complete)
+ return NoChange;
+
if (shard.fastPathElectorate.contains(from))
{
++fastPathDelayed;
- if (isNewFastPathDelayed() && hasReachedQuorum())
- return Success;
+ if (isFastPathDelayed() && hasReachedQuorum())
+ return complete(Success);
}
return NoChange;
}
- final ShardOutcome<? super FastPathTracker> quorumIfNewAndHasRejectedFastPath()
- {
- return isNewSlowPathSuccess() && hasRejectedFastPath() ? Success : NoChange;
- }
-
- final boolean isNewSlowPathSuccess()
- {
- return successes == shard.slowPathQuorumSize;
- }
-
- final boolean isNewFastPathReject()
- {
- return fastPathFailures == shard.fastPathRejectSize && fastPathDelayed < shard.fastPathRejectSize;
- }
-
- final boolean isNewFastPathDelayed()
+ final ShardOutcome<? super FastPathTracker> quorumIfHasRejectedFastPath()
{
- return fastPathDelayed == shard.fastPathRejectSize && fastPathFailures < shard.fastPathRejectSize;
+ return hasReachedQuorum() && hasRejectedFastPath() ? complete(Success) : NoChange;
}
- final boolean isNewFastPathSuccess()
+ final boolean isFastPathDelayed()
{
- return fastPathAccepts == shard.fastPathQuorumSize && (fastPathDelayed < shard.fastPathRejectSize || isNewSlowPathSuccess());
+ return fastPathDelayed >= shard.fastPathRejectSize;
}
@VisibleForTesting
@@ -116,7 +131,7 @@ public class FastPathTracker extends PreAcceptTracker<FastPathTracker.FastPathSh
@VisibleForTesting
public final boolean hasRejectedFastPath()
{
- return Math.max(fastPathFailures, fastPathDelayed) >= shard.fastPathRejectSize;
+ return fastPathFailures >= shard.fastPathRejectSize;
}
}
@@ -129,24 +144,7 @@ public class FastPathTracker extends PreAcceptTracker<FastPathTracker.FastPathSh
public ShardOutcome<? super FastPathTracker> onQuorumSuccess(Node.Id node)
{
- ++successes;
- if (!shard.fastPathElectorate.contains(node))
- return quorumIfNewAndHasRejectedFastPath();
-
- ++fastPathAccepts;
- if (isNewFastPathSuccess())
- return NewFastPathSuccess;
-
- if (isNewSlowPathSuccess() && hasRejectedFastPath())
- return Success;
-
- return NoChange;
- }
-
- @Override
- public ShardOutcome<? super FastPathTracker> onMaybeFastPathSuccess(Node.Id node)
- {
- return onQuorumSuccess(node);
+ return onMaybeFastPathSuccess(node);
}
}
@@ -159,31 +157,14 @@ public class FastPathTracker extends PreAcceptTracker<FastPathTracker.FastPathSh
public ShardOutcome<? super FastPathTracker> onQuorumSuccess(Node.Id node)
{
- ++successes;
- if (!shard.fastPathElectorate.contains(node))
- return quorumIfNewAndHasRejectedFastPath();
-
- ++fastPathFailures;
- if (isNewFastPathReject() && hasReachedQuorum())
- return Success;
-
- if (isNewSlowPathSuccess() && hasRejectedFastPath())
- return Success;
+ if (complete)
+ return NoChange;
- return NoChange;
- }
-
- @Override
- public ShardOutcome<? super FastPathTracker> onMaybeFastPathSuccess(Node.Id node)
- {
++successes;
if (shard.fastPathElectorate.contains(node))
- {
- ++fastPathAccepts;
- if (isNewFastPathSuccess())
- return NewFastPathSuccess;
- }
- return quorumIfNewAndHasRejectedFastPath();
+ ++fastPathFailures; // Quorum success can not count towards fast path success
+
+ return quorumIfHasRejectedFastPath();
}
}
@@ -227,11 +208,6 @@ public class FastPathTracker extends PreAcceptTracker<FastPathTracker.FastPathSh
return any(FastPathShardTracker::hasFailed);
}
- public boolean hasInFlight()
- {
- return any(FastPathShardTracker::hasInFlight);
- }
-
public boolean hasReachedQuorum()
{
return all(FastPathShardTracker::hasReachedQuorum);
diff --git a/accord-core/src/main/java/accord/impl/AbstractSafeCommandStore.java b/accord-core/src/main/java/accord/impl/AbstractSafeCommandStore.java
index 8a740873..bc053ad7 100644
--- a/accord-core/src/main/java/accord/impl/AbstractSafeCommandStore.java
+++ b/accord-core/src/main/java/accord/impl/AbstractSafeCommandStore.java
@@ -18,21 +18,22 @@
package accord.impl;
-import java.util.function.Consumer;
-import java.util.function.Function;
-
-import com.google.common.annotations.VisibleForTesting;
+import java.util.ArrayList;
+import java.util.List;
import accord.api.RoutingKey;
-import accord.api.VisibleForImplementation;
import accord.local.*;
import accord.local.cfk.SafeCommandsForKey;
import accord.primitives.*;
-import accord.utils.Invariants;
-public abstract class AbstractSafeCommandStore<CommandType extends SafeCommand,
- TimestampsForKeyType extends SafeTimestampsForKey,
- CommandsForKeyType extends SafeCommandsForKey> extends SafeCommandStore
+import static accord.local.KeyHistory.TIMESTAMPS;
+import static accord.utils.Invariants.illegalArgument;
+
+public abstract class AbstractSafeCommandStore<C extends SafeCommand,
+ TFK extends SafeTimestampsForKey,
+ CFK extends SafeCommandsForKey,
+ Caches extends AbstractSafeCommandStore.CommandStoreCaches<C, TFK, CFK>>
+extends SafeCommandStore
{
protected final PreLoadContext context;
@@ -41,112 +42,146 @@ public abstract class AbstractSafeCommandStore<CommandType extends SafeCommand,
this.context = context;
}
- private static <K, V> V getIfLoadedUnsafe(K key, Function<K, V> get, Consumer<V> add, Function<K, V> getIfLoaded)
+ public interface CommandStoreCaches<C, TFK, CFK> extends AutoCloseable
{
- V value = get.apply(key);
- if (value != null)
- return value;
+ void close();
- value = getIfLoaded.apply(key);
- if (value == null)
- return null;
- add.accept(value);
- return value;
+ C acquireIfLoaded(TxnId txnId);
+ CFK acquireIfLoaded(RoutingKey key);
+ TFK acquireTfkIfLoaded(RoutingKey key);
}
- protected abstract CommandType getCommandUnsafe(TxnId txnId);
- protected abstract void addCommandUnsafe(CommandType command);
- protected abstract CommandType getIfLoadedUnsafe(TxnId txnId);
+ protected abstract Caches tryGetCaches();
+ protected abstract C add(C safeCommand, Caches caches);
+ protected abstract CFK add(CFK safeCfk, Caches caches);
+ protected abstract TFK add(TFK safeTfk, Caches caches);
- protected abstract TimestampsForKeyType getTimestampsForKeyUnsafe(RoutingKey key);
- protected abstract void addTimestampsForKeyUnsafe(TimestampsForKeyType cfk);
- protected abstract TimestampsForKeyType getTimestampsForKeyIfUnsafe(RoutingKey key);
-
- protected abstract CommandsForKeyType getCommandsForKeyUnsafe(RoutingKey key);
- protected abstract void addCommandsForKeyUnsafe(CommandsForKeyType cfk);
- protected abstract CommandsForKeyType getCommandsForKeyIfUnsafe(RoutingKey key);
+ // get anything we've already loaded and referenced
+ protected abstract TFK timestampsForKeyInternal(RoutingKey key);
@Override
- protected CommandType ifLoadedAndInitialisedAndNotErasedInternal(TxnId txnId)
+ public PreLoadContext canExecute(PreLoadContext with)
{
- CommandType command = getIfLoadedUnsafe(txnId, this::getCommandUnsafe, this::addCommandUnsafe, this::getIfLoadedUnsafe);
- if (command == null || command.isUnset())
+ if (with.isEmpty()) return with;
+ if (with.keys().domain() == Routable.Domain.Range)
+ return with.isSubsetOf(this.context) ? with : null;
+
+ if (!context().keyHistory().satisfies(with.keyHistory()))
return null;
- return command;
- }
- @Override
- public CommandType getInternal(TxnId txnId)
- {
- return getCommandUnsafe(txnId);
+ try (Caches caches = tryGetCaches())
+ {
+ if (caches == null)
+ return with.isSubsetOf(this.context) ? with : null;
+
+ for (TxnId txnId : with.txnIds())
+ {
+ if (null != getInternal(txnId))
+ continue;
+
+ C safeCommand = caches.acquireIfLoaded(txnId);
+ if (safeCommand == null)
+ return null;
+
+ add(safeCommand, caches);
+ }
+
+ KeyHistory keyHistory = with.keyHistory();
+ if (keyHistory == KeyHistory.NONE)
+ return with;
+
+ List<RoutingKey> unavailable = null;
+ Unseekables<?> keys = with.keys();
+ if (keys.isEmpty())
+ return with;
+
+ for (int i = 0 ; i < keys.size() ; ++i)
+ {
+ RoutingKey key = (RoutingKey) keys.get(i);
+ if (keyHistory == TIMESTAMPS)
+ {
+ if (null != timestampsForKeyInternal(key))
+ continue; // already in working set
+
+ TFK safeTfk = caches.acquireTfkIfLoaded(key);
+ if (safeTfk != null)
+ {
+ add(safeTfk, caches);
+ continue;
+ }
+ }
+ else
+ {
+ if (null != getInternal(key))
+ continue; // already in working set
+
+ CFK safeCfk = caches.acquireIfLoaded(key);
+ if (safeCfk != null)
+ {
+ add(safeCfk, caches);
+ continue;
+ }
+ }
+ if (unavailable == null)
+ unavailable = new ArrayList<>();
+ unavailable.add(key);
+ }
+
+ if (unavailable == null)
+ return with;
+
+ if (unavailable.size() == keys.size())
+ return null;
+
+ return PreLoadContext.contextFor(with.primaryTxnId(), with.additionalTxnId(), keys.without(RoutingKeys.ofSortedUnique(unavailable)), keyHistory);
+ }
}
- private CommandsForKeyType getCommandsIfLoadedUnsafe(RoutingKey key)
+ @Override
+ public PreLoadContext context()
{
- return getIfLoadedUnsafe(key, this::getCommandsForKeyUnsafe, this::addCommandsForKeyUnsafe, this::getCommandsForKeyIfUnsafe);
+ return context;
}
- protected CommandsForKeyType ifLoadedInternal(RoutingKey key)
+ @Override
+ protected C ifLoadedInternal(TxnId txnId)
{
- return getCommandsIfLoadedUnsafe(key);
- }
+ try (Caches caches = tryGetCaches())
+ {
+ if (caches == null)
+ return null;
- @VisibleForTesting
- protected CommandsForKeyType getInternal(RoutingKey key)
- {
- return getCommandsIfLoadedUnsafe(key);
- }
+ C command = caches.acquireIfLoaded(txnId);
+ if (command == null)
+ return null;
- @VisibleForImplementation
- public CommandsForKeyType maybeCommandsForKey(RoutingKey key)
- {
- CommandsForKeyType cfk = getCommandsIfLoadedUnsafe(key);
- if (cfk == null || cfk.isUnset())
- return null;
- return cfk;
- }
-
- public TimestampsForKeyType timestampsIfLoadedAndInitialised(RoutingKey key)
- {
- TimestampsForKeyType cfk = getIfLoadedUnsafe(key, this::getTimestampsForKeyUnsafe, this::addTimestampsForKeyUnsafe, this::getTimestampsForKeyIfUnsafe);
- if (cfk == null)
- return null;
- if (cfk.isUnset())
- {
- cfk.initialize();
+ return add(command, caches);
}
- return cfk;
}
- public TimestampsForKeyType timestampsForKey(RoutingKey key)
+ @Override
+ protected CFK ifLoadedInternal(RoutingKey txnId)
{
- TimestampsForKeyType tfk = getIfLoadedUnsafe(key, this::getTimestampsForKeyUnsafe, this::addTimestampsForKeyUnsafe, this::getTimestampsForKeyIfUnsafe);
- Invariants.checkState(tfk != null, "%s was not specified in PreLoadContext", key);
- if (tfk.isUnset())
- tfk.initialize();
- return tfk;
- }
-
+ try (Caches caches = tryGetCaches())
+ {
+ if (caches == null)
+ return null;
- @VisibleForImplementation
- public TimestampsForKeyType maybeTimestampsForKey(RoutingKey key)
- {
- TimestampsForKeyType tfk = getIfLoadedUnsafe(key, this::getTimestampsForKeyUnsafe, this::addTimestampsForKeyUnsafe, this::getTimestampsForKeyIfUnsafe);
- if (tfk == null || tfk.isUnset())
- return null;
- return tfk;
- }
+ CFK cfk = caches.acquireIfLoaded(txnId);
+ if (cfk == null)
+ return null;
- @Override
- public PreLoadContext context()
- {
- return this.context;
+ return add(cfk, caches);
+ }
}
@Override
- public PreLoadContext canExecute(PreLoadContext context)
+ public TFK timestampsForKey(RoutingKey key)
{
- return context.isSubsetOf(this.context) ? context : null;
+ TFK safeTfk = timestampsForKeyInternal(key);
+ if (safeTfk == null)
+ throw illegalArgument("%s not referenced in %s", key, context);
+ return safeTfk;
}
public void postExecute()
diff --git a/accord-core/src/main/java/accord/impl/DefaultLocalListeners.java b/accord-core/src/main/java/accord/impl/DefaultLocalListeners.java
index daa5593c..52fba403 100644
--- a/accord-core/src/main/java/accord/impl/DefaultLocalListeners.java
+++ b/accord-core/src/main/java/accord/impl/DefaultLocalListeners.java
@@ -84,7 +84,7 @@ public class DefaultLocalListeners implements LocalListeners
@Override
public void notify(SafeCommandStore safeStore, SafeCommand safeCommand, TxnId listenerId)
{
- SafeCommand listener = safeStore.ifLoadedAndInitialisedAndNotErased(listenerId);
+ SafeCommand listener = safeStore.ifLoadedAndInitialised(listenerId);
if (listener != null) Commands.listenerUpdate(safeStore, listener, safeCommand);
else
{
diff --git a/accord-core/src/main/java/accord/impl/DefaultRemoteListeners.java b/accord-core/src/main/java/accord/impl/DefaultRemoteListeners.java
index e8a9dc03..a2e63062 100644
--- a/accord-core/src/main/java/accord/impl/DefaultRemoteListeners.java
+++ b/accord-core/src/main/java/accord/impl/DefaultRemoteListeners.java
@@ -462,6 +462,11 @@ public class DefaultRemoteListeners implements RemoteListeners
}
}
+ public void clear()
+ {
+ listeners.clear();
+ }
+
private static int listenerNodeId(long encodedListener)
{
return (int) (encodedListener >>> 31);
diff --git a/accord-core/src/main/java/accord/impl/DurabilityScheduling.java b/accord-core/src/main/java/accord/impl/DurabilityScheduling.java
index 49e57cb1..bd370b93 100644
--- a/accord-core/src/main/java/accord/impl/DurabilityScheduling.java
+++ b/accord-core/src/main/java/accord/impl/DurabilityScheduling.java
@@ -306,7 +306,7 @@ public class DurabilityScheduling implements ConfigurationService.Listener
{
synchronized (ShardScheduler.this)
{
- // TODO (required): try to recover or invalidate prior sync point
+ // TODO (expected): try to recover or invalidate prior sync point
retry();
if (numberOfSplits * 2 <= maxNumberOfSplits)
{
diff --git a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
index eb827e5b..17f16842 100644
--- a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
+++ b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
@@ -94,7 +94,7 @@ import accord.utils.async.Cancellable;
import static accord.local.KeyHistory.ASYNC;
import static accord.local.SafeCommandStore.TestDep.ANY_DEPS;
-import static accord.local.SafeCommandStore.TestDep.WITH;
+import static accord.local.SafeCommandStore.TestDep.WITH_OR_INVALIDATED;
import static accord.local.SafeCommandStore.TestStartedAt.STARTED_BEFORE;
import static accord.local.SafeCommandStore.TestStatus.ANY_STATUS;
import static accord.primitives.SaveStatus.Applying;
@@ -195,7 +195,7 @@ public abstract class InMemoryCommandStore extends CommandStore
long epoch = cur.executeAt().epoch();
Ranges ranges = safeStore.ranges().allAt(epoch);
Participants<?> participants = cur.route().participants(ranges, Minimal);
- // TODO (required): look forwards also, but we only need to look at ?=ReadyToExecute transactions as they have already run their backwards checks
+ // TODO (testing): look forwards also, but we only need to look at ?=ReadyToExecute transactions as they have already run their backwards checks
Iterator<GlobalCommand> iter = commandsByExecuteAt.descendingMap().tailMap(cur.executeAt(), false).values().iterator();
while (iter.hasNext())
{
@@ -391,16 +391,18 @@ public abstract class InMemoryCommandStore extends CommandStore
List<TxnId> resubmit = new ArrayList<>();
for (TxnId txnId : txnIds)
{
- Command command = commands.get(txnId).value();
- if (!command.hasBeen(PreCommitted)) return;
- if (!command.txnId().isVisible()) return;
+ GlobalCommand globalCommand = commands.get(txnId);
+ if (globalCommand == null) continue;
+ Command command = globalCommand.value();
+ if (!command.hasBeen(PreCommitted)) continue;
+ if (!command.txnId().isVisible()) continue;
Ranges allRanges = unsafeRangesForEpoch().allBetween(txnId.epoch(), command.executeAtOrTxnId().epoch());
boolean done = command.hasBeen(Truncated);
if (!done)
{
if (unsafeGetRedundantBefore().status(txnId, command.route()) == RedundantStatus.PRE_BOOTSTRAP_OR_STALE)
- return;
+ continue;
Route<?> route = command.route().slice(allRanges);
done = !route.isEmpty() && ranges.containsAll(route);
@@ -423,7 +425,7 @@ public abstract class InMemoryCommandStore extends CommandStore
if (!resubmit.isEmpty())
{
txnIds = resubmit;
- node.scheduler().once(this, 5L, TimeUnit.SECONDS);
+ node.scheduler().selfRecurring(this, 5L, TimeUnit.SECONDS);
}
}
}
@@ -664,7 +666,40 @@ public abstract class InMemoryCommandStore extends CommandStore
}
}
- public static class InMemorySafeStore extends AbstractSafeCommandStore<InMemorySafeCommand, InMemorySafeTimestampsForKey, InMemorySafeCommandsForKey>
+ public class InMemoryCommandStoreCaches implements AbstractSafeCommandStore.CommandStoreCaches<InMemorySafeCommand, InMemorySafeTimestampsForKey, InMemorySafeCommandsForKey>
+ {
+ @Override
+ public void close() {}
+
+ @Override
+ public InMemorySafeCommand acquireIfLoaded(TxnId txnId)
+ {
+ GlobalCommand command = commands.get(txnId);
+ if (command == null)
+ return null;
+ return command.createSafeReference();
+ }
+
+ @Override
+ public InMemorySafeCommandsForKey acquireIfLoaded(RoutingKey key)
+ {
+ GlobalCommandsForKey cfk = commandsForKey.get(key);
+ if (cfk == null)
+ return null;
+ return cfk.createSafeReference();
+ }
+
+ @Override
+ public InMemorySafeTimestampsForKey acquireTfkIfLoaded(RoutingKey key)
+ {
+ GlobalTimestampsForKey tfk = timestampsForKey.get(key);
+ if (tfk == null)
+ return null;
+ return tfk.createSafeReference();
+ }
+ }
+
+ public static class InMemorySafeStore extends AbstractSafeCommandStore<InMemorySafeCommand, InMemorySafeTimestampsForKey, InMemorySafeCommandsForKey, InMemoryCommandStoreCaches>
{
private final InMemoryCommandStore commandStore;
protected final Map<TxnId, InMemorySafeCommand> commands;
@@ -700,69 +735,53 @@ public abstract class InMemoryCommandStore extends CommandStore
}
@Override
- protected InMemorySafeCommand getCommandUnsafe(TxnId txnId)
+ protected InMemorySafeCommand getInternal(TxnId txnId)
{
return commands.get(txnId);
}
@Override
- protected void addCommandUnsafe(InMemorySafeCommand command)
+ protected InMemoryCommandStoreCaches tryGetCaches()
{
- if (command.isUnset()) command.uninitialised();
- commands.put(command.txnId(), command);
+ if (commandStore.canExposeUnloaded())
+ return commandStore.new InMemoryCommandStoreCaches();
+ return null;
}
@Override
- protected InMemorySafeTimestampsForKey getTimestampsForKeyUnsafe(RoutingKey key)
+ protected InMemorySafeCommand add(InMemorySafeCommand command, InMemoryCommandStoreCaches caches)
{
- return timestampsForKey.get(key);
+ if (command.isUnset()) command.uninitialised();
+ commands.put(command.txnId(), command);
+ return command;
}
@Override
- protected void addTimestampsForKeyUnsafe(InMemorySafeTimestampsForKey tfk)
+ protected InMemorySafeTimestampsForKey add(InMemorySafeTimestampsForKey tfk, InMemoryCommandStoreCaches caches)
{
if (tfk.isUnset()) tfk.initialize();
timestampsForKey.put(tfk.key(), tfk);
+ return tfk;
}
@Override
- protected InMemorySafeTimestampsForKey getTimestampsForKeyIfUnsafe(RoutingKey key)
+ protected InMemorySafeTimestampsForKey timestampsForKeyInternal(RoutingKey key)
{
- if (!commandStore.canExposeUnloaded())
- return null;
- GlobalTimestampsForKey global = commandStore.timestampsForKeyIfPresent(key);
- return global != null ? global.createSafeReference() : null;
- }
-
- @Override
- protected InMemorySafeCommand getIfLoadedUnsafe(TxnId txnId)
- {
- if (!commandStore.canExposeUnloaded())
- return null;
- GlobalCommand global = commandStore.commandIfPresent(txnId);
- return global != null ? global.createSafeReference() : null;
+ return timestampsForKey.get(key);
}
@Override
- protected InMemorySafeCommandsForKey getCommandsForKeyUnsafe(RoutingKey key)
+ protected InMemorySafeCommandsForKey getInternal(RoutingKey key)
{
return commandsForKey.get(key);
}
@Override
- protected void addCommandsForKeyUnsafe(InMemorySafeCommandsForKey cfk)
+ protected InMemorySafeCommandsForKey add(InMemorySafeCommandsForKey cfk, InMemoryCommandStoreCaches caches)
{
if (cfk.isUnset()) cfk.initialize();
commandsForKey.put(cfk.key(), cfk);
- }
-
- @Override
- protected InMemorySafeCommandsForKey getCommandsForKeyIfUnsafe(RoutingKey key)
- {
- if (!commandStore.canExposeUnloaded())
- return null;
- GlobalCommandsForKey global = commandStore.commandsForKeyIfPresent(key);
- return global != null ? global.createSafeReference() : null;
+ return cfk;
}
@Override
@@ -920,6 +939,9 @@ public abstract class InMemoryCommandStore extends CommandStore
case IS_STABLE:
if (command.status().compareTo(Stable) < 0 || command.status().compareTo(Truncated) >= 0)
return;
+ case IS_STABLE_OR_INVALIDATED:
+ if (command.status().compareTo(Stable) < 0 || command.status() == Truncated)
+ return;
}
if (!testKind.test(command.txnId()))
@@ -941,7 +963,7 @@ public abstract class InMemoryCommandStore extends CommandStore
// and so it is safe to execute, when in fact it is only a dependency on a different shard
// (and that other shard, perhaps, does not know that it is a dependency - and so it is not durably known)
// TODO (required): consider this some more
- if ((testDep == WITH) == !command.partialDeps().intersects(testTxnId, rangeCommand.ranges))
+ if ((testDep == WITH_OR_INVALIDATED) == !command.partialDeps().intersects(testTxnId, rangeCommand.ranges))
return;
}
@@ -1194,7 +1216,7 @@ public abstract class InMemoryCommandStore extends CommandStore
@Override
public void shutdown()
{
- executor.shutdown();
+ executor.shutdownNow();
}
}
@@ -1213,10 +1235,10 @@ public abstract class InMemoryCommandStore extends CommandStore
}
@Override
- public InMemorySafeCommand ifLoadedAndInitialisedAndNotErasedInternal(TxnId txnId)
+ public InMemorySafeCommand ifLoadedInternal(TxnId txnId)
{
assertThread();
- return super.ifLoadedAndInitialisedAndNotErasedInternal(txnId);
+ return super.ifLoadedInternal(txnId);
}
@Override
diff --git a/accord-core/src/main/java/accord/impl/progresslog/BaseTxnState.java b/accord-core/src/main/java/accord/impl/progresslog/BaseTxnState.java
index 2c20c6e6..140d5015 100644
--- a/accord-core/src/main/java/accord/impl/progresslog/BaseTxnState.java
+++ b/accord-core/src/main/java/accord/impl/progresslog/BaseTxnState.java
@@ -39,11 +39,11 @@ import static accord.impl.progresslog.TxnStateKind.Waiting;
* - WaitingState defines the methods for updating the WaitingState state machine (i.e. for local commands that are waiting on a dependency's outcome)
* - HomeState defines the methods for updating the HomeState state machine (i.e. for coordinators ensuring a transaction completes)
* <p>
- * TODO (required): unit test all bit fiddling methods
+ * TODO (testing): unit test all bit fiddling methods
*/
abstract class BaseTxnState extends LogGroupTimers.Timer implements Comparable<BaseTxnState>
{
- private static final int PENDING_TIMER_SHIFT = 53;
+ private static final int PENDING_TIMER_SHIFT = HomeState.HOME_STATE_END_SHIFT;
private static final int PENDING_TIMER_BITS = 9;
private static final int PENDING_TIMER_LOW_BITS = PackedLogLinearInteger.validateLowBits(4, PENDING_TIMER_BITS);
private static final long PENDING_TIMER_MASK = (1L << PENDING_TIMER_BITS) - 1;
@@ -51,22 +51,27 @@ abstract class BaseTxnState extends LogGroupTimers.Timer implements Comparable<B
private static final int SCHEDULED_TIMER_SHIFT = 62;
private static final int CONTACT_ALL_SHIFT = 63;
+ static
+ {
+ Invariants.checkState(PENDING_TIMER_SHIFT + PENDING_TIMER_BITS <= SCHEDULED_TIMER_SHIFT);
+ }
+
public final TxnId txnId;
/**
- * bits [0..40) encode WaitingState
+ * bits [0..43) encode WaitingState
* 2 bits for Progress
* 3 bits for BlockedUntil target
* 3 bits for BlockedUntil that home shard can satisfy
* 32 bits for remote progress key counter [note: if we need to in future we can safely and easily reclaim bits here]
* 3 bits for retry counter
* <p>
- * bits [40..45) encode HomeState
+ * bits [43..51) encode HomeState
* 2 bits for Progress
* 3 bits for CoordinatePhase
* 3 bits for retry counter
* <p>
- * bits [53..62) for pending timer delay
+ * bits [51..60) for pending timer delay
* bit 62 for which kind of timer is scheduled
* bit 63 for whether we should contact all candidate replicas (rather than just our preferred group)
*/
diff --git a/accord-core/src/main/java/accord/impl/progresslog/DefaultProgressLog.java b/accord-core/src/main/java/accord/impl/progresslog/DefaultProgressLog.java
index 41c7c7e7..521d07df 100644
--- a/accord-core/src/main/java/accord/impl/progresslog/DefaultProgressLog.java
+++ b/accord-core/src/main/java/accord/impl/progresslog/DefaultProgressLog.java
@@ -183,9 +183,11 @@ public class DefaultProgressLog implements ProgressLog, Runnable
return;
TxnState state = null;
- if (before.route() == null && after.route() != null)
+ Route<?> beforeRoute = before.route();
+ Route<?> afterRoute = after.route();
+ if (beforeRoute == null && afterRoute != null)
{
- RoutingKey homeKey = after.homeKey();
+ RoutingKey homeKey = afterRoute.homeKey();
Ranges coordinateRanges = safeStore.coordinateRanges(txnId);
boolean isHome = coordinateRanges.contains(homeKey);
state = get(txnId);
@@ -200,7 +202,9 @@ public class DefaultProgressLog implements ProgressLog, Runnable
state = maybeFetch(safeStore, txnId, after, state);
}
else
+ {
state.set(safeStore, this, Undecided, Queued);
+ }
}
else if (state != null)
{
@@ -317,7 +321,7 @@ public class DefaultProgressLog implements ProgressLog, Runnable
}
@Override
- public void waiting(BlockedUntil blockedUntil, SafeCommandStore safeStore, SafeCommand blockedBy, Route<?> blockedOnRoute, Participants<?> blockedOnParticipants)
+ public void waiting(BlockedUntil blockedUntil, SafeCommandStore safeStore, SafeCommand blockedBy, Route<?> blockedOnRoute, Participants<?> blockedOnParticipants, StoreParticipants blockedOnStoreParticipants)
{
if (!blockedBy.txnId().isVisible())
return;
@@ -334,6 +338,7 @@ public class DefaultProgressLog implements ProgressLog, Runnable
CommonAttributes update = blockedBy.current();
StoreParticipants participants = update.participants();
StoreParticipants updatedParticipants = participants.supplement(blockedOnRoute, null, blockedOnParticipants);
+ if (blockedOnStoreParticipants != null) updatedParticipants = updatedParticipants.supplement(blockedOnStoreParticipants);
if (participants != updatedParticipants)
update = update.mutable().updateParticipants(updatedParticipants);
@@ -342,7 +347,8 @@ public class DefaultProgressLog implements ProgressLog, Runnable
// TODO (required): tighten up ExclusiveSyncPoint range bounds
Invariants.checkState((command.txnId().is(Txn.Kind.ExclusiveSyncPoint) ? safeStore.ranges().all()
- : safeStore.ranges().allSince(command.txnId().epoch())).intersects(command.participants().hasTouched()));
+ : safeStore.ranges().allSince(command.txnId().epoch())
+ ).intersects(command.participants().hasTouched()));
// TODO (consider): consider triggering a preemption of existing coordinator (if any) in some circumstances;
// today, an LWT can pre-empt more efficiently (i.e. instantly) a failed operation whereas Accord will
@@ -459,6 +465,10 @@ public class DefaultProgressLog implements ProgressLog, Runnable
if (safeCommand == null)
return;
+ // check this after fetching SafeCommand, as doing so can erase the command (and invalidate our state)
+ if (run.isDone(runKind))
+ return;
+
Invariants.checkState(get(run.txnId) == run, "Transaction state for %s does not match expected one %s", run.txnId, run);
Invariants.checkState(run.scheduledTimer() != runKind, "We are actively executing %s, but we are also scheduled to run this same TxnState later. This should not happen.", runKind);
Invariants.checkState(run.pendingTimer() != runKind, "We are actively executing %s, but we also have a pending scheduled task to run this same TxnState later. This should not happen.", runKind);
diff --git a/accord-core/src/main/java/accord/impl/progresslog/DefaultProgressLogs.java b/accord-core/src/main/java/accord/impl/progresslog/DefaultProgressLogs.java
index 7f4fceaa..8ca59d6c 100644
--- a/accord-core/src/main/java/accord/impl/progresslog/DefaultProgressLogs.java
+++ b/accord-core/src/main/java/accord/impl/progresslog/DefaultProgressLogs.java
@@ -28,7 +28,7 @@ import accord.local.Node;
// TODO (desired, consider): consider propagating invalidations in the same way as we do applied
// TODO (expected): report transactions not making progress
-// TODO (required): evict to disk
+// TODO (expected): evict to disk
public class DefaultProgressLogs implements ProgressLog.Factory
{
static ConcurrentHashMap<DefaultProgressLog, Runnable> PAUSE_FOR_TEST;
diff --git a/accord-core/src/main/java/accord/impl/progresslog/HomeState.java b/accord-core/src/main/java/accord/impl/progresslog/HomeState.java
index d8378f53..f8302f6d 100644
--- a/accord-core/src/main/java/accord/impl/progresslog/HomeState.java
+++ b/accord-core/src/main/java/accord/impl/progresslog/HomeState.java
@@ -41,7 +41,7 @@ import static accord.impl.progresslog.Progress.Queued;
import static accord.impl.progresslog.TxnStateKind.Home;
/**
- * TODO (required): describe state machine
+ * TODO (documentation): describe state machine
*
* TODO (expected): do not attempt recovery every run; simply check the coordinator is still active
* TODO (expected): do not attempt execution until all shards are ready; use the WaitingState to achieve this
@@ -52,11 +52,11 @@ abstract class HomeState extends WaitingState
private static final long PROGRESS_MASK = 0x3;
private static final int STATUS_SHIFT = PROGRESS_SHIFT + 2;
private static final long STATUS_MASK = 0x7;
- private static final int RETRY_COUNTER_SHIFT = PROGRESS_SHIFT + 3;
+ private static final int RETRY_COUNTER_SHIFT = STATUS_SHIFT + 3;
private static final long RETRY_COUNTER_MASK = 0x7;
private static final long SET_MASK = ~((PROGRESS_MASK << PROGRESS_SHIFT)
| (STATUS_MASK << STATUS_SHIFT));
- final int HOME_STATE_END_SHIFT = RETRY_COUNTER_SHIFT + 3;
+ static final int HOME_STATE_END_SHIFT = RETRY_COUNTER_SHIFT + 3;
HomeState(TxnId txnId)
{
@@ -141,7 +141,7 @@ abstract class HomeState extends WaitingState
Invariants.checkState(command.durability() != null);
// TODO (expected): when invalidated, safer to maintain HomeState until known to be globally invalidated
- // TODO (now): validate that we clear HomeState when we receive a Durable reply, to replace the token check logic
+ // TODO (expected): validate that we clear HomeState when we receive a Durable reply, to replace the token check logic
Invariants.checkState(!command.durability().isDurableOrInvalidated(), "Command is durable or invalidated, but we have not cleared the ProgressLog");
ProgressToken maxProgressToken = instance.savedProgressToken(txnId).merge(command);
diff --git a/accord-core/src/main/java/accord/impl/progresslog/TxnState.java b/accord-core/src/main/java/accord/impl/progresslog/TxnState.java
index c189f6a2..e5441995 100644
--- a/accord-core/src/main/java/accord/impl/progresslog/TxnState.java
+++ b/accord-core/src/main/java/accord/impl/progresslog/TxnState.java
@@ -154,4 +154,9 @@ final class TxnState extends HomeState
{
return txnId + ": " + toStateString();
}
+
+ public boolean isDone(TxnStateKind runKind)
+ {
+ return runKind == TxnStateKind.Home ? isHomeDone() : isWaitingDone();
+ }
}
diff --git a/accord-core/src/main/java/accord/impl/progresslog/WaitingState.java b/accord-core/src/main/java/accord/impl/progresslog/WaitingState.java
index 87c90ae3..35e80b8a 100644
--- a/accord-core/src/main/java/accord/impl/progresslog/WaitingState.java
+++ b/accord-core/src/main/java/accord/impl/progresslog/WaitingState.java
@@ -87,13 +87,13 @@ abstract class WaitingState extends BaseTxnState
private static final long BLOCKED_UNTIL_MASK = 0x7;
private static final int HOME_SATISFIES_SHIFT = 5;
private static final long HOME_SATISFIES_MASK = 0x7;
- private static final int AWAIT_BITS = 32;
private static final int AWAIT_SHIFT = 8;
+ private static final int AWAIT_BITS = 32;
private static final long AWAIT_MASK = (1L << AWAIT_BITS) - 1;
private static final long SET_MASK = ~((PROGRESS_MASK << PROGRESS_SHIFT) | (BLOCKED_UNTIL_MASK << BLOCKED_UNTIL_SHIFT));
private static final long INITIALISED_MASK = (PROGRESS_MASK << PROGRESS_SHIFT) | (BLOCKED_UNTIL_MASK << BLOCKED_UNTIL_SHIFT) | (HOME_SATISFIES_MASK << HOME_SATISFIES_SHIFT);
- private static final int RETRY_COUNTER_SHIFT = 40;
+ private static final int RETRY_COUNTER_SHIFT = AWAIT_SHIFT + AWAIT_BITS;
private static final long RETRY_COUNTER_MASK = 0x7;
static final int WAITING_STATE_END_SHIFT = RETRY_COUNTER_SHIFT + 3;
diff --git a/accord-core/src/main/java/accord/local/Bootstrap.java b/accord-core/src/main/java/accord/local/Bootstrap.java
index 2c9799e8..459952aa 100644
--- a/accord-core/src/main/java/accord/local/Bootstrap.java
+++ b/accord-core/src/main/java/accord/local/Bootstrap.java
@@ -44,7 +44,6 @@ import accord.utils.async.AsyncResults;
import static accord.local.PreLoadContext.empty;
import static accord.primitives.Routables.Slice.Minimal;
import static accord.primitives.Txn.Kind.ExclusiveSyncPoint;
-import static accord.primitives.Txn.Kind.LocalOnly;
import static accord.utils.Invariants.illegalState;
/**
@@ -109,7 +108,7 @@ class Bootstrap
boolean fetchCompleted;
boolean completed; // we have finished fetching all the data we are able to, but we may still have in-flight fetches
Throwable fetchOutcome;
- TxnId globalSyncId, localSyncId;
+ TxnId globalSyncId;
Attempt(Ranges ranges)
{
@@ -125,7 +124,6 @@ class Bootstrap
}
globalSyncId = node.nextTxnId(ExclusiveSyncPoint, Routable.Domain.Range);
- localSyncId = globalSyncId.as(LocalOnly).withEpoch(epoch);
Invariants.checkArgument(epoch <= globalSyncId.epoch(), "Attempting to use local epoch %d which is larger than global epoch %d", epoch, globalSyncId.epoch());
if (!node.topology().hasEpoch(globalSyncId.epoch()))
diff --git a/accord-core/src/main/java/accord/local/Cleanup.java b/accord-core/src/main/java/accord/local/Cleanup.java
index 195f9fd6..028f068e 100644
--- a/accord-core/src/main/java/accord/local/Cleanup.java
+++ b/accord-core/src/main/java/accord/local/Cleanup.java
@@ -30,9 +30,10 @@ import accord.primitives.Status.Durability;
import accord.primitives.TxnId;
import static accord.local.RedundantBefore.PreBootstrapOrStale.FULLY;
-import static accord.local.RedundantStatus.LIVE;
+import static accord.local.RedundantStatus.GC_BEFORE;
import static accord.local.RedundantStatus.NOT_OWNED;
import static accord.local.RedundantStatus.SHARD_REDUNDANT;
+import static accord.local.RedundantStatus.WAS_OWNED_RETIRED;
import static accord.primitives.SaveStatus.Erased;
import static accord.primitives.SaveStatus.ErasedOrVestigial;
import static accord.primitives.SaveStatus.Invalidated;
@@ -40,7 +41,6 @@ import static accord.primitives.SaveStatus.TruncatedApply;
import static accord.primitives.SaveStatus.TruncatedApplyWithOutcome;
import static accord.primitives.SaveStatus.Uninitialised;
import static accord.primitives.Status.Applied;
-import static accord.primitives.Status.Durability.Majority;
import static accord.primitives.Status.Durability.UniversalOrInvalidated;
import static accord.primitives.Status.PreCommitted;
import static accord.primitives.Txn.Kind.EphemeralRead;
@@ -79,26 +79,9 @@ public enum Cleanup
return saveStatus.compareTo(appliesIfNot) >= 0 ? NO : this;
}
- public static Cleanup shouldCleanup(SafeCommandStore safeStore, Command command)
- {
- return shouldCleanup(safeStore, command, command.participants());
- }
-
- public static Cleanup shouldCleanup(SafeCommandStore safeStore, Command command, @Nonnull StoreParticipants participants)
- {
- return shouldCleanup(safeStore.agent(), command.txnId(), command.saveStatus(), command.durability(), participants,
- safeStore.redundantBefore(), safeStore.durableBefore());
- }
-
- public static Cleanup shouldCleanup(Agent agent, Command command, RedundantBefore redundantBefore, DurableBefore durableBefore)
+ static Cleanup min(Cleanup a, Cleanup b)
{
- return shouldCleanup(agent, command.txnId(), command.saveStatus(), command.durability(), command.participants(),
- redundantBefore, durableBefore);
- }
-
- public static Cleanup shouldCleanup(Agent agent, TxnId txnId, SaveStatus status, Durability durability, StoreParticipants participants, RedundantBefore redundantBefore, DurableBefore durableBefore)
- {
- return shouldCleanupInternal(agent, txnId, status, durability, participants, redundantBefore, durableBefore).filter(status);
+ return a.compareTo(b) <= 0 ? a : b;
}
// TODO (required): simulate compaction of log records in burn test
@@ -108,44 +91,6 @@ public enum Cleanup
return shouldCleanupPartialInternal(agent, txnId, status, durability, participants, redundantBefore, durableBefore).filter(status);
}
- private static Cleanup shouldCleanupInternal(Agent agent, TxnId txnId, SaveStatus saveStatus, Durability durability, StoreParticipants participants, RedundantBefore redundantBefore, DurableBefore durableBefore)
- {
- if (txnId.kind() == EphemeralRead)
- return NO; // TODO (required): clean-up based on timeout
-
- if (expunge(txnId, saveStatus, durableBefore, redundantBefore))
- return EXPUNGE;
-
- if (saveStatus == Uninitialised)
- {
- if (!redundantBefore.isAnyOnAnyEpoch(txnId, participants.touches, SHARD_REDUNDANT))
- return NO;
-
- // participants.touches() means e.g. we used to or will own the participant, but shard redundant means all
- // owners of the command have applied it - if we aren't guaranteed to know it and we don't then it is
- // "vestigial" i.e. represents some attempt to coordinate the command against us (e.g. failed propose or calculateDeps)
- Cleanup cleanup = VESTIGIAL;
- if (redundantBefore.isAnyOnCoordinationEpoch(txnId, participants.owns, SHARD_REDUNDANT))
- {
- // owns means the key is known to interact with us is shard redundant then we
- cleanup = ERASE;
- if (redundantBefore.isAnyOnCoordinationEpoch(txnId, participants.owns, LIVE))
- cleanup = INVALIDATE;
- }
- return cleanup;
- }
-
- if (!participants.hasFullRoute())
- {
- if (!saveStatus.hasBeen(PreCommitted) && redundantBefore.isAnyOnCoordinationEpoch(txnId, participants.owns, SHARD_REDUNDANT))
- return Cleanup.INVALIDATE;
-
- return Cleanup.NO;
- }
-
- return cleanupWithFullRoute(agent, false, participants, txnId, saveStatus, durability, redundantBefore, durableBefore);
- }
-
private static Cleanup shouldCleanupPartialInternal(Agent agent, TxnId txnId, SaveStatus status, @Nullable Durability durability, @Nullable StoreParticipants participants, RedundantBefore redundantBefore, DurableBefore durableBefore)
{
if (txnId.kind() == EphemeralRead)
@@ -159,7 +104,7 @@ public enum Cleanup
if (!participants.hasFullRoute())
{
- if (!redundantBefore.isAnyOnCoordinationEpoch(txnId, participants.owns, SHARD_REDUNDANT))
+ if (!redundantBefore.isAnyOnCoordinationEpoch(txnId, participants.owns, GC_BEFORE))
return NO;
// we only need to keep the outcome if we have it; otherwise we can expunge
@@ -181,6 +126,65 @@ public enum Cleanup
return cleanupWithFullRoute(agent, true, participants, txnId, status, durability, redundantBefore, durableBefore);
}
+ public static Cleanup shouldCleanup(SafeCommandStore safeStore, Command command)
+ {
+ return shouldCleanup(safeStore, command, command.participants());
+ }
+
+ public static Cleanup shouldCleanup(SafeCommandStore safeStore, Command command, @Nonnull StoreParticipants participants)
+ {
+ return shouldCleanup(safeStore.agent(), command.txnId(), command.saveStatus(), command.durability(), participants,
+ safeStore.redundantBefore(), safeStore.durableBefore());
+ }
+
+ public static Cleanup shouldCleanup(Agent agent, Command command, RedundantBefore redundantBefore, DurableBefore durableBefore)
+ {
+ return shouldCleanup(agent, command.txnId(), command.saveStatus(), command.durability(), command.participants(),
+ redundantBefore, durableBefore);
+ }
+
+ public static Cleanup shouldCleanup(Agent agent, TxnId txnId, SaveStatus status, Durability durability, StoreParticipants participants, RedundantBefore redundantBefore, DurableBefore durableBefore)
+ {
+ return shouldCleanupInternal(agent, txnId, status, durability, participants, redundantBefore, durableBefore).filter(status);
+ }
+
+ private static Cleanup shouldCleanupInternal(Agent agent, TxnId txnId, SaveStatus saveStatus, Durability durability, StoreParticipants participants, RedundantBefore redundantBefore, DurableBefore durableBefore)
+ {
+ if (txnId.kind() == EphemeralRead)
+ return NO;
+
+ if (expunge(txnId, saveStatus, durableBefore, redundantBefore))
+ return EXPUNGE;
+
+ if (!participants.hasFullRoute())
+ {
+ if (!saveStatus.hasBeen(PreCommitted) && redundantBefore.isAnyOnCoordinationEpoch(txnId, participants.owns, GC_BEFORE))
+ return Cleanup.INVALIDATE;
+
+ // TODO (required): consider if our uninitialised special-casing is fine
+ return saveStatus != Uninitialised ? NO : cleanupUninitialised(txnId, participants, redundantBefore);
+ }
+
+ Cleanup result = cleanupWithFullRoute(agent, false, participants, txnId, saveStatus, durability, redundantBefore, durableBefore);
+ if (result == NO && saveStatus == Uninitialised)
+ result = cleanupUninitialised(txnId, participants, redundantBefore);
+ return result;
+ }
+
+ private static Cleanup cleanupUninitialised(TxnId txnId, StoreParticipants participants, RedundantBefore redundantBefore)
+ {
+ if (!redundantBefore.isAnyOnAnyEpoch(txnId, participants.touches, SHARD_REDUNDANT))
+ return NO;
+
+ // participants.touches() means e.g. we used to or will own the participant, but shard redundant means all
+ // owners of the command have applied it - if we aren't guaranteed to know it and we don't then it is
+ // "vestigial" i.e. represents some attempt to coordinate the command against us (e.g. failed propose or calculateDeps)
+ Cleanup cleanup = VESTIGIAL;
+ if (redundantBefore.isAnyOnCoordinationEpoch(txnId, participants.owns, SHARD_REDUNDANT))
+ cleanup = INVALIDATE;
+ return cleanup;
+ }
+
private static Cleanup cleanupWithFullRoute(Agent agent, boolean isPartial, StoreParticipants participants, TxnId txnId, SaveStatus saveStatus, Durability durability, RedundantBefore redundantBefore, DurableBefore durableBefore)
{
// We first check if the command is redundant locally, i.e. whether it has been applied to all non-faulty replicas of the local shard
@@ -193,44 +197,54 @@ public enum Cleanup
switch (redundant)
{
default: throw new AssertionError();
+ case WAS_OWNED:
+ case WAS_OWNED_CLOSED:
case LIVE:
case PARTIALLY_PRE_BOOTSTRAP_OR_STALE:
case PRE_BOOTSTRAP_OR_STALE:
case REDUNDANT_PRE_BOOTSTRAP_OR_STALE:
case LOCALLY_REDUNDANT:
- return Cleanup.NO;
+ return NO;
+
case SHARD_REDUNDANT:
- if (!isPartial && saveStatus.hasBeen(PreCommitted) && !saveStatus.hasBeen(Applied) && redundantBefore.preBootstrapOrStale(txnId, participants.owns) != FULLY)
+ return isPartial || saveStatus.hasBeen(PreCommitted) ? NO : INVALIDATE;
+
+ case GC_BEFORE:
+ if (!isPartial)
{
- agent.onViolation(String.format("Loading SHARD_REDUNDANT command %s with status %s (that should have been Applied). Expected to be witnessed and executed by %s.", txnId, saveStatus, redundantBefore.max(participants.route, e -> e.shardAppliedOrInvalidatedBefore)));
- return TRUNCATE;
+ if (!saveStatus.hasBeen(PreCommitted))
+ return INVALIDATE;
+
+ if (!saveStatus.hasBeen(Applied) && redundantBefore.preBootstrapOrStale(txnId, participants.owns) != FULLY)
+ {
+ agent.onViolation(String.format("Loading SHARD_REDUNDANT command %s with status %s (that should have been Applied). Expected to be witnessed and executed by %s.", txnId, saveStatus, redundantBefore.max(participants.route, e -> e.shardAppliedOrInvalidatedBefore)));
+ return TRUNCATE;
+ }
}
- case WAS_OWNED:
- }
+ break;
- if (!isPartial && !saveStatus.hasBeen(PreCommitted))
- return INVALIDATE;
+ case WAS_OWNED_RETIRED:
+ }
- Durability min = durableBefore.min(txnId, participants.route);
- switch (min)
+ durability = Durability.max(durability, durableBefore.min(txnId, participants.route));
+ switch (durability)
{
- default:
+ default: throw new AssertionError("Unexpected durability: " + durability);
case Local:
- throw new AssertionError("Unexpected durability: " + min);
-
case NotDurable:
- if (durability == Majority)
- return Cleanup.TRUNCATE;
+ case ShardUniversal:
+ if (redundant == WAS_OWNED_RETIRED)
+ return NO; // TODO (expected): document why we treat this differently
return Cleanup.TRUNCATE_WITH_OUTCOME;
case MajorityOrInvalidated:
case Majority:
- return Cleanup.TRUNCATE;
+ return TRUNCATE;
case UniversalOrInvalidated:
case Universal:
// TODO (expected): can we EXPUNGE here?
- return Cleanup.ERASE;
+ return ERASE;
}
}
@@ -245,7 +259,7 @@ public enum Cleanup
// TODO (required): we should perhaps weaken this to separately account whether remotely and locally redundant?
// i.e., if we know that the shard is remotely durable and we know we don't need it locally (e.g. due to bootstrap)
// then we can safely erase. Revisit as part of rationalising RedundantBefore registers.
- return redundantBefore.shardStatus(txnId) == SHARD_REDUNDANT;
+ return redundantBefore.shardStatus(txnId) == GC_BEFORE;
}
public static Cleanup forOrdinal(int ordinal)
diff --git a/accord-core/src/main/java/accord/local/CommandStore.java b/accord-core/src/main/java/accord/local/CommandStore.java
index bcd175cb..6fb4ae04 100644
--- a/accord-core/src/main/java/accord/local/CommandStore.java
+++ b/accord-core/src/main/java/accord/local/CommandStore.java
@@ -165,6 +165,7 @@ public abstract class CommandStore implements AgentExecutor
* But they may still be ordered for other key ranges they participate in.
*
* TODO (expected): merge with redundantBefore
+ * TODO (required): if we ever actually erase data on disk (after losing ownership), make sure to mark the range unsafe to read
*/
private NavigableMap<Timestamp, Ranges> safeToRead = ImmutableSortedMap.of(Timestamp.NONE, Ranges.EMPTY);
private final Set<Bootstrap> bootstraps = Collections.synchronizedSet(new DeterministicIdentitySet<>());
@@ -592,7 +593,7 @@ public abstract class CommandStore implements AgentExecutor
e.getValue().ranges = remaining = remaining.without(ranges);
if (e.getValue().ranges.isEmpty())
{
- logger.info("Completed full sync for {} on epoch {} using {}", e.getValue().allRanges, e.getKey(), syncId);
+ logger.debug("Completed full sync for {} on epoch {} using {}", e.getValue().allRanges, e.getKey(), syncId);
e.getValue().whenDone.trySuccess(null);
if (remove == null)
remove = new LongHashSet();
@@ -600,7 +601,7 @@ public abstract class CommandStore implements AgentExecutor
}
else
{
- logger.info("Completed partial sync for {} on epoch {} using {}; {} still to sync", synced, e.getKey(), syncId, remaining);
+ logger.debug("Completed partial sync for {} on epoch {} using {}; {} still to sync", synced, e.getKey(), syncId, remaining);
}
}
if (remove != null)
diff --git a/accord-core/src/main/java/accord/local/Commands.java b/accord-core/src/main/java/accord/local/Commands.java
index 3974d86d..c69b1c3b 100644
--- a/accord-core/src/main/java/accord/local/Commands.java
+++ b/accord-core/src/main/java/accord/local/Commands.java
@@ -573,7 +573,7 @@ public class Commands
if (logger.isTraceEnabled())
logger.trace("{}: Maybe executing with status {}. Will notify listeners on noop: {}", command.txnId(), command.status(), alwaysNotifyListeners);
- if (command.status() != Stable && command.status() != PreApplied)
+ if (command.status() != Stable && command.saveStatus() != SaveStatus.PreApplied)
{
if (alwaysNotifyListeners)
safeStore.notifyListeners(safeCommand, command);
@@ -648,7 +648,7 @@ public class Commands
redundantBefore.removeRedundantDependencies(participants, update);
update.forEachWaitingOnId(safeStore, update, waiting, executeAt, (store, upd, w, exec, i) -> {
- SafeCommand dep = store.ifLoadedAndInitialisedAndNotErased(upd.txnId(i));
+ SafeCommand dep = store.ifLoadedAndInitialised(upd.txnId(i));
if (dep == null || !dep.current().hasBeen(PreCommitted))
return;
updateWaitingOn(store, w, exec, upd, dep);
@@ -737,7 +737,7 @@ public class Commands
{
TxnId nextWaitingOn = command.waitingOn().nextWaitingOn();
if (nextWaitingOn != null && nextWaitingOn.equals(pred.txnId()) && !pred.hasBeen(PreApplied))
- safeStore.progressLog().waiting(CanApply, safeStore, predecessor, pred.route(), null);
+ safeStore.progressLog().waiting(CanApply, safeStore, predecessor, pred.route(), null, null);
}
}
}
@@ -846,13 +846,13 @@ public class Commands
case TRUNCATE_WITH_OUTCOME:
Invariants.checkArgument(!command.hasBeen(Truncated), "%s", command);
- Invariants.checkState(command.hasBeen(PreApplied));
+ Invariants.checkState(command.hasBeen(PreApplied) || command.participants().touches().isEmpty());
result = truncatedApplyWithOutcome(command.asExecuted());
break;
case TRUNCATE:
Invariants.checkState(command.saveStatus().compareTo(TruncatedApply) < 0);
- Invariants.checkState(command.hasBeen(PreApplied));
+ Invariants.checkState(command.hasBeen(PreApplied) || command.participants().touches().isEmpty());
result = truncatedApply(command, participants);
break;
@@ -870,7 +870,7 @@ public class Commands
return result;
}
- private static boolean validateSafeToCleanup(SafeCommandStore safeStore, Command command, StoreParticipants participants)
+ private static boolean validateSafeToCleanup(SafeCommandStore safeStore, Command command, @Nullable StoreParticipants participants)
{
TxnId txnId = command.txnId();
participants = command.participants().supplement(participants);
@@ -879,13 +879,16 @@ public class Commands
{
default: throw new AssertionError("Unhandled RedundantStatus: " + status);
case LIVE:
+ case WAS_OWNED:
+ case WAS_OWNED_CLOSED:
+ case WAS_OWNED_RETIRED:
case REDUNDANT_PRE_BOOTSTRAP_OR_STALE:
case PARTIALLY_PRE_BOOTSTRAP_OR_STALE:
return false;
case LOCALLY_REDUNDANT:
case SHARD_REDUNDANT:
+ case GC_BEFORE:
Invariants.checkState(!command.hasBeen(PreCommitted));
- case WAS_OWNED:
case PRE_BOOTSTRAP_OR_STALE:
case NOT_OWNED:
return true;
@@ -894,7 +897,8 @@ public class Commands
public static boolean maybeCleanup(SafeCommandStore safeStore, SafeCommand safeCommand, Command command, @Nonnull StoreParticipants participants)
{
- Cleanup cleanup = shouldCleanup(safeStore, command, participants);
+ // TODO (desired): cheaper version of supplement for here
+ Cleanup cleanup = shouldCleanup(safeStore, command, command.participants().supplement(participants));
if (cleanup == NO)
return false;
@@ -956,13 +960,24 @@ public class Commands
switch (redundantStatus)
{
default: throw new AssertionError("Unexpected redundant status: " + redundantStatus);
- case NOT_OWNED: throw new AssertionError("Invalid state: waiting for execution of command that is not owned at the execution time");
+ case NOT_OWNED:
+ throw new AssertionError("Invalid state: waiting for execution of command that is not owned at the execution time");
+
+ case GC_BEFORE:
case SHARD_REDUNDANT:
case LOCALLY_REDUNDANT:
case REDUNDANT_PRE_BOOTSTRAP_OR_STALE:
case PRE_BOOTSTRAP_OR_STALE:
+ case WAS_OWNED_RETIRED:
removeRedundantDependencies(safeStore, waitingSafe, loadDepId);
break;
+
+ case WAS_OWNED:
+ case WAS_OWNED_CLOSED:
+ if (!waitingId.awaitsPreviouslyOwned())
+ removeNoLongerOwnedDependency(safeStore, waitingSafe, loadDepId);
+ break;
+
case LIVE:
case PARTIALLY_PRE_BOOTSTRAP_OR_STALE:
}
@@ -998,7 +1013,7 @@ public class Commands
}
}
- depSafe = safeStore.ifLoadedAndInitialisedAndNotErased(directlyBlockedOn);
+ depSafe = safeStore.ifLoadedAndInitialised(directlyBlockedOn);
if (depSafe == null)
{
loadDepId = directlyBlockedOn;
@@ -1032,16 +1047,22 @@ public class Commands
switch (redundantStatus)
{
default: throw new AssertionError("Unknown redundant status: " + redundantStatus);
- case NOT_OWNED: throw new AssertionError("Invalid state: waiting for execution of command that is not owned at the execution time");
+ case NOT_OWNED:
+ case WAS_OWNED:
+ case WAS_OWNED_CLOSED:
+ case WAS_OWNED_RETIRED:
+ throw new AssertionError("Invalid state: waiting for execution of command that is not owned at the execution time");
+
case LIVE:
case PARTIALLY_PRE_BOOTSTRAP_OR_STALE:
if (logger.isTraceEnabled()) logger.trace("{} blocked on {} until ReadyToExclude", waitingId, dep.txnId());
safeStore.registerListener(depSafe, HasDecidedExecuteAt.minSaveStatus, waitingId);
- safeStore.progressLog().waiting(HasDecidedExecuteAt, safeStore, depSafe, null, participants);
+ safeStore.progressLog().waiting(HasDecidedExecuteAt, safeStore, depSafe, null, participants, null);
return;
case LOCALLY_REDUNDANT:
case SHARD_REDUNDANT:
+ case GC_BEFORE:
case PRE_BOOTSTRAP_OR_STALE:
case REDUNDANT_PRE_BOOTSTRAP_OR_STALE:
Invariants.checkState(dep.hasBeen(Applied) || !dep.hasBeen(PreCommitted) || redundantStatus == PRE_BOOTSTRAP_OR_STALE);
@@ -1060,13 +1081,13 @@ public class Commands
case NotReady:
safeStore.registerListener(depSafe, HasDecidedExecuteAt.minSaveStatus, waitingId);
- safeStore.progressLog().waiting(HasDecidedExecuteAt, safeStore, depSafe, dep.route(), null);
+ safeStore.progressLog().waiting(HasDecidedExecuteAt, safeStore, depSafe, dep.route(), null, null);
return;
case ReadyToExclude:
case WaitingToExecute:
case ReadyToExecute:
- safeStore.progressLog().waiting(CanApply, safeStore, depSafe, dep.route(), null);
+ safeStore.progressLog().waiting(CanApply, safeStore, depSafe, dep.route(), null, null);
case Applying:
safeStore.registerListener(depSafe, SaveStatus.Applied, waitingId);
@@ -1130,6 +1151,17 @@ public class Commands
return safeCommand.updateWaitingOn(update);
}
+ static Command removeNoLongerOwnedDependency(SafeCommandStore safeStore, SafeCommand safeCommand, @Nonnull TxnId wasOwned)
+ {
+ Command.Committed current = safeCommand.current().asCommitted();
+ if (!current.waitingOn.isWaitingOn(wasOwned))
+ return current;
+
+ WaitingOn.Update update = new WaitingOn.Update(current.waitingOn);
+ update.removeWaitingOn(wasOwned);
+ return safeCommand.updateWaitingOn(update);
+ }
+
/**
* A key nominated to represent the "home" shard - only members of the home shard may be nominated to recover
* a transaction, to reduce the cluster-wide overhead of ensuring progress. A transaction that has only been
diff --git a/accord-core/src/main/java/accord/local/PreLoadContext.java b/accord-core/src/main/java/accord/local/PreLoadContext.java
index 97697d06..b4d5e15b 100644
--- a/accord-core/src/main/java/accord/local/PreLoadContext.java
+++ b/accord-core/src/main/java/accord/local/PreLoadContext.java
@@ -156,6 +156,13 @@ public interface PreLoadContext
return new Standard(primary, additional, keys, keyHistory);
}
+ default boolean contains(TxnId txnId)
+ {
+ TxnId primaryTxnId = primaryTxnId();
+ return primaryTxnId != null && (txnId.equals(primaryTxnId) || txnId.equals(additionalTxnId()));
+ }
+
+
class Standard implements PreLoadContext
{
private final @Nullable TxnId primary;
diff --git a/accord-core/src/main/java/accord/local/RedundantBefore.java b/accord-core/src/main/java/accord/local/RedundantBefore.java
index 4aef63a2..c13a229e 100644
--- a/accord-core/src/main/java/accord/local/RedundantBefore.java
+++ b/accord-core/src/main/java/accord/local/RedundantBefore.java
@@ -21,6 +21,7 @@ package accord.local;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
+import java.util.function.BiPredicate;
import java.util.function.Function;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@@ -38,6 +39,7 @@ import accord.primitives.Ranges;
import accord.primitives.Routables;
import accord.primitives.RoutingKeys;
import accord.primitives.Timestamp;
+import accord.primitives.Txn;
import accord.primitives.TxnId;
import accord.primitives.Unseekables;
import accord.utils.Invariants;
@@ -48,12 +50,15 @@ import org.agrona.collections.Int2ObjectHashMap;
import static accord.local.RedundantBefore.PreBootstrapOrStale.FULLY;
import static accord.local.RedundantBefore.PreBootstrapOrStale.POST_BOOTSTRAP;
import static accord.local.RedundantBefore.PreBootstrapOrStale.PARTIALLY;
+import static accord.local.RedundantStatus.GC_BEFORE;
import static accord.local.RedundantStatus.LIVE;
import static accord.local.RedundantStatus.LOCALLY_REDUNDANT;
import static accord.local.RedundantStatus.NOT_OWNED;
import static accord.local.RedundantStatus.PRE_BOOTSTRAP_OR_STALE;
import static accord.local.RedundantStatus.SHARD_REDUNDANT;
import static accord.local.RedundantStatus.WAS_OWNED;
+import static accord.local.RedundantStatus.WAS_OWNED_CLOSED;
+import static accord.local.RedundantStatus.WAS_OWNED_RETIRED;
import static accord.utils.Invariants.illegalState;
public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
@@ -155,14 +160,27 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
this.gcBefore = gcBefore;
this.bootstrappedAt = bootstrappedAt;
this.staleUntilAtLeast = staleUntilAtLeast;
- Invariants.checkArgument(locallyWitnessedOrInvalidatedBefore.equals(TxnId.NONE) || locallyWitnessedOrInvalidatedBefore.domain().isRange());
- Invariants.checkArgument(locallyAppliedOrInvalidatedBefore.equals(TxnId.NONE) || locallyAppliedOrInvalidatedBefore.domain().isRange());
- Invariants.checkArgument(locallyDecidedAndAppliedOrInvalidatedBefore.equals(TxnId.NONE) || locallyDecidedAndAppliedOrInvalidatedBefore.domain().isRange());
- Invariants.checkArgument(shardAppliedOrInvalidatedBefore.equals(TxnId.NONE) || shardAppliedOrInvalidatedBefore.domain().isRange());
- Invariants.checkArgument(gcBefore.equals(TxnId.NONE) || gcBefore.domain().isRange());
- Invariants.checkArgument(locallyDecidedAndAppliedOrInvalidatedBefore.compareTo(locallyAppliedOrInvalidatedBefore) <= 0);
- Invariants.checkArgument(shardAppliedOrInvalidatedBefore.compareTo(shardOnlyAppliedOrInvalidatedBefore) <= 0);
- Invariants.checkArgument(gcBefore.compareTo(shardAppliedOrInvalidatedBefore) <= 0);
+ checkNoneOrRX(locallyWitnessedOrInvalidatedBefore, locallyAppliedOrInvalidatedBefore, locallyDecidedAndAppliedOrInvalidatedBefore,
+ shardAppliedOrInvalidatedBefore, gcBefore);
+ checkPartiallyOrdered(locallyDecidedAndAppliedOrInvalidatedBefore, locallyAppliedOrInvalidatedBefore);
+ checkPartiallyOrdered(shardAppliedOrInvalidatedBefore, locallyAppliedOrInvalidatedBefore);
+ checkPartiallyOrdered(gcBefore, shardAppliedOrInvalidatedBefore, shardOnlyAppliedOrInvalidatedBefore);
+ }
+
+ private static void checkPartiallyOrdered(TxnId ... txnIds)
+ {
+ for (int i = 1 ; i < txnIds.length ; ++i)
+ Invariants.checkArgument(txnIds[i - 1].compareTo(txnIds[i]) <= 0);
+ }
+
+ private static void checkNoneOrRX(TxnId ... txnIds)
+ {
+ for (TxnId txnId : txnIds)
+ checkNoneOrRX(txnId);
+ }
+ private static void checkNoneOrRX(TxnId txnId)
+ {
+ Invariants.checkArgument(txnId.equals(TxnId.NONE) || (txnId.domain().isRange() && txnId.is(Txn.Kind.ExclusiveSyncPoint)));
}
public static Entry reduce(Entry a, Entry b)
@@ -201,7 +219,6 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
TxnId gcBefore = cg >= 0 ? cur.gcBefore : add.gcBefore;
TxnId bootstrappedAt = cb >= 0 ? cur.bootstrappedAt : add.bootstrappedAt;
Timestamp staleUntilAtLeast = csu >= 0 ? cur.staleUntilAtLeast : add.staleUntilAtLeast;
- TxnId shardAppliedOrInvalidatedBefore = TxnId.min(shardOnlyAppliedOrInvalidatedBefore, locallyAppliedOrInvalidatedBefore);
// if a NEW redundantBefore predates our current bootstrappedAt, we should not update it to avoid erroneously
// treating transactions prior as locally redundant when they may simply have not applied yet, since we may
@@ -212,6 +229,8 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
// TODO (desired): revisit later as semantics here evolve
if (bootstrappedAt.compareTo(locallyAppliedOrInvalidatedBefore) >= 0)
locallyAppliedOrInvalidatedBefore = cur.locallyAppliedOrInvalidatedBefore;
+
+ TxnId shardAppliedOrInvalidatedBefore = TxnId.min(shardOnlyAppliedOrInvalidatedBefore, locallyAppliedOrInvalidatedBefore);
if (bootstrappedAt.compareTo(shardAppliedOrInvalidatedBefore) >= 0)
locallyDecidedAndAppliedOrInvalidatedBefore = cur.locallyDecidedAndAppliedOrInvalidatedBefore;
if (staleUntilAtLeast != null && bootstrappedAt.compareTo(staleUntilAtLeast) >= 0)
@@ -245,6 +264,16 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
}
static @Nonnull Boolean isAnyOnCoordinationEpoch(Entry entry, @Nonnull Boolean prev, TxnId txnId, RedundantStatus status)
+ {
+ return isAnyOnCoordinationEpoch(entry, prev, txnId, status, (a, b) -> a == b);
+ }
+
+ static @Nonnull Boolean isAnyOnCoordinationEpochAtLeast(Entry entry, @Nonnull Boolean prev, TxnId txnId, RedundantStatus status)
+ {
+ return isAnyOnCoordinationEpoch(entry, prev, txnId, status, (a, b) -> a.compareTo(b) >= 0);
+ }
+
+ static @Nonnull Boolean isAnyOnCoordinationEpoch(Entry entry, @Nonnull Boolean prev, TxnId txnId, RedundantStatus test, BiPredicate<RedundantStatus, RedundantStatus> predicate)
{
if (entry == null || prev)
return prev;
@@ -253,10 +282,10 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
if (entry.startOwnershipEpoch > epoch || entry.endOwnershipEpoch <= epoch)
return false;
- return entry.getIgnoringOwnership(txnId) == status;
+ return predicate.test(entry.getIgnoringOwnership(txnId), test);
}
- static @Nonnull Boolean isAnyOnAnyEpoch(Entry entry, @Nonnull Boolean prev, TxnId txnId, RedundantStatus status)
+ static @Nonnull Boolean isAnyOnAnyEpoch(Entry entry, @Nonnull Boolean prev, TxnId txnId, RedundantStatus test, BiPredicate<RedundantStatus, RedundantStatus> predicate)
{
if (entry == null || prev)
return prev;
@@ -265,7 +294,17 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
if (entry.startOwnershipEpoch > epoch || entry.endOwnershipEpoch <= epoch)
return false;
- return entry.getIgnoringOwnership(txnId) == status;
+ return predicate.test(entry.getIgnoringOwnership(txnId), test);
+ }
+
+ static @Nonnull Boolean isAnyOnAnyEpoch(Entry entry, @Nonnull Boolean prev, TxnId txnId, RedundantStatus status)
+ {
+ return isAnyOnAnyEpoch(entry, prev, txnId, status, (a, b) -> a == b);
+ }
+
+ static @Nonnull Boolean isAnyOnAnyEpochAtLeast(Entry entry, @Nonnull Boolean prev, TxnId txnId, RedundantStatus status)
+ {
+ return isAnyOnAnyEpoch(entry, prev, txnId, status, (a, b) -> a.compareTo(b) >= 0);
}
static RedundantStatus get(Entry entry, TxnId txnId, EpochSupplier executeAt)
@@ -282,7 +321,7 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
return prev;
// TODO (required): consider all call-sites and confirm the answers when wasOwned and willBeOwned are reasonable
- if (entry.wasOwned(txnId) && entry.isComplete())
+ if (entry.wasOwned(txnId) && entry.isRetired())
return prev;
boolean isPreBootstrapOrStale = entry.staleUntilAtLeast != null || entry.bootstrappedAt.compareTo(txnId) > 0;
@@ -363,8 +402,8 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
RedundantStatus get(TxnId txnId)
{
- if (wasOwned(txnId) && isComplete())
- return WAS_OWNED;
+ if (wasOwned(txnId))
+ return isRetired() ? WAS_OWNED_RETIRED : isClosed() ? WAS_OWNED_CLOSED : WAS_OWNED;
return getIgnoringOwnership(txnId);
}
@@ -378,6 +417,8 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
if (locallyAppliedOrInvalidatedBefore.compareTo(txnId) > 0)
{
if (gcBefore.compareTo(txnId) > 0)
+ return GC_BEFORE;
+ if (shardAppliedOrInvalidatedBefore.compareTo(txnId) > 0)
return SHARD_REDUNDANT;
return LOCALLY_REDUNDANT;
}
@@ -395,11 +436,16 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
return aIsNull ? 0 : a.compareTo(b);
}
- public final TxnId shardRedundantBefore()
+ public final TxnId gcBefore()
{
return gcBefore;
}
+ public final TxnId shardRedundantBefore()
+ {
+ return shardAppliedOrInvalidatedBefore;
+ }
+
public final TxnId locallyWitnessedBefore()
{
return locallyWitnessedOrInvalidatedBefore;
@@ -426,10 +472,16 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
}
// TODO (required): do we still need this, or can we stick to just the explicit endOwnershipEpoch
- private boolean isComplete()
+ private boolean isRetired()
{
// TODO (required): carefully consider whether we should ALSO expect some local property to be met here
- return endOwnershipEpoch <= gcBefore.epoch();
+ return endOwnershipEpoch <= shardAppliedOrInvalidatedBefore.epoch();
+ }
+
+ private boolean isClosed()
+ {
+ // TODO (required): carefully consider whether we should ALSO expect some local property to be met here
+ return endOwnershipEpoch <= locallyWitnessedOrInvalidatedBefore.epoch();
}
private boolean outOfBounds(Timestamp lb)
@@ -598,11 +650,21 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
return foldl(participants, Entry::isAnyOnCoordinationEpoch, false, txnId, status, isDone -> isDone);
}
+ public boolean isAnyOnCoordinationEpochAtLeast(TxnId txnId, Unseekables<?> participants, RedundantStatus status)
+ {
+ return foldl(participants, Entry::isAnyOnCoordinationEpochAtLeast, false, txnId, status, isDone -> isDone);
+ }
+
public boolean isAnyOnAnyEpoch(TxnId txnId, Unseekables<?> participants, RedundantStatus status)
{
return foldl(participants, Entry::isAnyOnAnyEpoch, false, txnId, status, isDone -> isDone);
}
+ public boolean isAnyOnAnyEpochAtLeast(TxnId txnId, Unseekables<?> participants, RedundantStatus status)
+ {
+ return foldl(participants, Entry::isAnyOnAnyEpochAtLeast, false, txnId, status, isDone -> isDone);
+ }
+
/**
* RedundantStatus.REDUNDANT overrides PRE_BOOTSTRAP; to avoid complicating that state machine,
* for cases where we care independently about the overall pre-bootstrap state we have a separate mechanism
diff --git a/accord-core/src/main/java/accord/local/RedundantStatus.java b/accord-core/src/main/java/accord/local/RedundantStatus.java
index 3ecdc073..279621bb 100644
--- a/accord-core/src/main/java/accord/local/RedundantStatus.java
+++ b/accord-core/src/main/java/accord/local/RedundantStatus.java
@@ -34,6 +34,18 @@ public enum RedundantStatus
*/
WAS_OWNED,
+ /**
+ * None of the relevant ranges are owned by the command store anymore, and all of those ranges are closed
+ * (i.e. we should know all participating commands already)
+ */
+ WAS_OWNED_CLOSED,
+
+ /**
+ * None of the relevant ranges are owned by the command store anymore, and all of those ranges are retired
+ * (i.e. we should not participate in any decisions)
+ */
+ WAS_OWNED_RETIRED,
+
/**
* Some of the relevant ranges are owned by the command store and valid for execution
*/
@@ -71,11 +83,20 @@ public enum RedundantStatus
/**
* The relevant owned ranges are redundant, meaning any intersecting transaction is known to be either applied or invalidated
* via a sync point that has applied locally AND on all healthy shards. Note that this status CANNOT be taken if
- * we are not ALSO LOCALLY_REDUNDANT, so we can use this to cleanup commands < PreCommitted.
+ * we are not ALSO LOCALLY_REDUNDANT, so we can use this to cleanup transactions < PreCommitted (or Erased).
*
* Note that this status overrides PRE_BOOTSTRAP_OR_STALE, since it implies the transaction has applied.
*/
SHARD_REDUNDANT,
+
+ /**
+ * The relevant owned ranges are redundant, meaning any intersecting transaction is known to be either applied or invalidated
+ * via a sync point that has applied locally AND on all healthy shards. Note that this status CANNOT be taken if
+ * we are not ALSO LOCALLY_REDUNDANT, so we can use this to cleanup local state.
+ *
+ * Note that this status overrides PRE_BOOTSTRAP_OR_STALE, since it implies the transaction has applied.
+ */
+ GC_BEFORE,
;
private EnumMap<RedundantStatus, RedundantStatus> merge;
@@ -85,75 +106,135 @@ public enum RedundantStatus
NOT_OWNED.merge = new EnumMap<>(RedundantStatus.class);
NOT_OWNED.merge.put(NOT_OWNED, NOT_OWNED);
NOT_OWNED.merge.put(WAS_OWNED, WAS_OWNED);
+ NOT_OWNED.merge.put(WAS_OWNED_CLOSED, WAS_OWNED_CLOSED);
+ NOT_OWNED.merge.put(WAS_OWNED_RETIRED, WAS_OWNED_RETIRED);
NOT_OWNED.merge.put(LIVE, LIVE);
NOT_OWNED.merge.put(PARTIALLY_PRE_BOOTSTRAP_OR_STALE, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
NOT_OWNED.merge.put(PRE_BOOTSTRAP_OR_STALE, PRE_BOOTSTRAP_OR_STALE);
NOT_OWNED.merge.put(REDUNDANT_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
NOT_OWNED.merge.put(LOCALLY_REDUNDANT, LOCALLY_REDUNDANT);
NOT_OWNED.merge.put(SHARD_REDUNDANT, SHARD_REDUNDANT);
+ NOT_OWNED.merge.put(GC_BEFORE, GC_BEFORE);
WAS_OWNED.merge = new EnumMap<>(RedundantStatus.class);
WAS_OWNED.merge.put(NOT_OWNED, NOT_OWNED);
WAS_OWNED.merge.put(WAS_OWNED, WAS_OWNED);
+ WAS_OWNED.merge.put(WAS_OWNED_CLOSED, WAS_OWNED);
+ WAS_OWNED.merge.put(WAS_OWNED_RETIRED, WAS_OWNED);
WAS_OWNED.merge.put(LIVE, LIVE);
WAS_OWNED.merge.put(PARTIALLY_PRE_BOOTSTRAP_OR_STALE, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
WAS_OWNED.merge.put(PRE_BOOTSTRAP_OR_STALE, PRE_BOOTSTRAP_OR_STALE);
WAS_OWNED.merge.put(REDUNDANT_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
WAS_OWNED.merge.put(LOCALLY_REDUNDANT, LOCALLY_REDUNDANT);
WAS_OWNED.merge.put(SHARD_REDUNDANT, SHARD_REDUNDANT);
+ WAS_OWNED.merge.put(GC_BEFORE, GC_BEFORE);
+ WAS_OWNED_CLOSED.merge = new EnumMap<>(RedundantStatus.class);
+ WAS_OWNED_CLOSED.merge.put(NOT_OWNED, NOT_OWNED);
+ WAS_OWNED_CLOSED.merge.put(WAS_OWNED, WAS_OWNED);
+ WAS_OWNED_CLOSED.merge.put(WAS_OWNED_CLOSED, WAS_OWNED_CLOSED);
+ WAS_OWNED_CLOSED.merge.put(WAS_OWNED_RETIRED, WAS_OWNED_CLOSED);
+ WAS_OWNED_CLOSED.merge.put(LIVE, LIVE);
+ WAS_OWNED_CLOSED.merge.put(PARTIALLY_PRE_BOOTSTRAP_OR_STALE, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
+ WAS_OWNED_CLOSED.merge.put(PRE_BOOTSTRAP_OR_STALE, PRE_BOOTSTRAP_OR_STALE);
+ WAS_OWNED_CLOSED.merge.put(REDUNDANT_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+ WAS_OWNED_CLOSED.merge.put(LOCALLY_REDUNDANT, LOCALLY_REDUNDANT);
+ WAS_OWNED_CLOSED.merge.put(SHARD_REDUNDANT, SHARD_REDUNDANT);
+ WAS_OWNED_CLOSED.merge.put(GC_BEFORE, GC_BEFORE);
+ WAS_OWNED_RETIRED.merge = new EnumMap<>(RedundantStatus.class);
+ WAS_OWNED_RETIRED.merge.put(NOT_OWNED, NOT_OWNED);
+ WAS_OWNED_RETIRED.merge.put(WAS_OWNED, WAS_OWNED);
+ WAS_OWNED_RETIRED.merge.put(WAS_OWNED_CLOSED, WAS_OWNED_CLOSED);
+ WAS_OWNED_RETIRED.merge.put(WAS_OWNED_RETIRED, WAS_OWNED_RETIRED);
+ WAS_OWNED_RETIRED.merge.put(LIVE, LIVE);
+ WAS_OWNED_RETIRED.merge.put(PARTIALLY_PRE_BOOTSTRAP_OR_STALE, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
+ WAS_OWNED_RETIRED.merge.put(PRE_BOOTSTRAP_OR_STALE, PRE_BOOTSTRAP_OR_STALE);
+ WAS_OWNED_RETIRED.merge.put(REDUNDANT_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+ WAS_OWNED_RETIRED.merge.put(LOCALLY_REDUNDANT, LOCALLY_REDUNDANT);
+ WAS_OWNED_RETIRED.merge.put(SHARD_REDUNDANT, SHARD_REDUNDANT);
+ WAS_OWNED_RETIRED.merge.put(GC_BEFORE, GC_BEFORE);
LIVE.merge = new EnumMap<>(RedundantStatus.class);
LIVE.merge.put(NOT_OWNED, LIVE);
LIVE.merge.put(WAS_OWNED, LIVE);
+ LIVE.merge.put(WAS_OWNED_CLOSED, LIVE);
+ LIVE.merge.put(WAS_OWNED_RETIRED, LIVE);
LIVE.merge.put(LIVE, LIVE);
LIVE.merge.put(PARTIALLY_PRE_BOOTSTRAP_OR_STALE, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
LIVE.merge.put(PRE_BOOTSTRAP_OR_STALE, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
LIVE.merge.put(REDUNDANT_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
LIVE.merge.put(LOCALLY_REDUNDANT, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
LIVE.merge.put(SHARD_REDUNDANT, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+ LIVE.merge.put(GC_BEFORE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
PARTIALLY_PRE_BOOTSTRAP_OR_STALE.merge = new EnumMap<>(RedundantStatus.class);
PARTIALLY_PRE_BOOTSTRAP_OR_STALE.merge.put(NOT_OWNED, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
PARTIALLY_PRE_BOOTSTRAP_OR_STALE.merge.put(WAS_OWNED, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
+ PARTIALLY_PRE_BOOTSTRAP_OR_STALE.merge.put(WAS_OWNED_CLOSED, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
+ PARTIALLY_PRE_BOOTSTRAP_OR_STALE.merge.put(WAS_OWNED_RETIRED, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
PARTIALLY_PRE_BOOTSTRAP_OR_STALE.merge.put(LIVE, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
PARTIALLY_PRE_BOOTSTRAP_OR_STALE.merge.put(PARTIALLY_PRE_BOOTSTRAP_OR_STALE, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
PARTIALLY_PRE_BOOTSTRAP_OR_STALE.merge.put(PRE_BOOTSTRAP_OR_STALE, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
PARTIALLY_PRE_BOOTSTRAP_OR_STALE.merge.put(REDUNDANT_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
PARTIALLY_PRE_BOOTSTRAP_OR_STALE.merge.put(LOCALLY_REDUNDANT, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
PARTIALLY_PRE_BOOTSTRAP_OR_STALE.merge.put(SHARD_REDUNDANT, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+ PARTIALLY_PRE_BOOTSTRAP_OR_STALE.merge.put(GC_BEFORE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
PRE_BOOTSTRAP_OR_STALE.merge = new EnumMap<>(RedundantStatus.class);
PRE_BOOTSTRAP_OR_STALE.merge.put(NOT_OWNED, PRE_BOOTSTRAP_OR_STALE);
PRE_BOOTSTRAP_OR_STALE.merge.put(WAS_OWNED, PRE_BOOTSTRAP_OR_STALE);
+ PRE_BOOTSTRAP_OR_STALE.merge.put(WAS_OWNED_CLOSED, PRE_BOOTSTRAP_OR_STALE);
+ PRE_BOOTSTRAP_OR_STALE.merge.put(WAS_OWNED_RETIRED, PRE_BOOTSTRAP_OR_STALE);
PRE_BOOTSTRAP_OR_STALE.merge.put(LIVE, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
PRE_BOOTSTRAP_OR_STALE.merge.put(PARTIALLY_PRE_BOOTSTRAP_OR_STALE, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
PRE_BOOTSTRAP_OR_STALE.merge.put(PRE_BOOTSTRAP_OR_STALE, PRE_BOOTSTRAP_OR_STALE);
PRE_BOOTSTRAP_OR_STALE.merge.put(REDUNDANT_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
PRE_BOOTSTRAP_OR_STALE.merge.put(LOCALLY_REDUNDANT, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
PRE_BOOTSTRAP_OR_STALE.merge.put(SHARD_REDUNDANT, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+ PRE_BOOTSTRAP_OR_STALE.merge.put(GC_BEFORE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
REDUNDANT_PRE_BOOTSTRAP_OR_STALE.merge = new EnumMap<>(RedundantStatus.class);
REDUNDANT_PRE_BOOTSTRAP_OR_STALE.merge.put(NOT_OWNED, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
REDUNDANT_PRE_BOOTSTRAP_OR_STALE.merge.put(WAS_OWNED, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+ REDUNDANT_PRE_BOOTSTRAP_OR_STALE.merge.put(WAS_OWNED_CLOSED, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+ REDUNDANT_PRE_BOOTSTRAP_OR_STALE.merge.put(WAS_OWNED_RETIRED, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
REDUNDANT_PRE_BOOTSTRAP_OR_STALE.merge.put(LIVE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE); // TODO (expected): should this be an invalid combination?
REDUNDANT_PRE_BOOTSTRAP_OR_STALE.merge.put(PARTIALLY_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
REDUNDANT_PRE_BOOTSTRAP_OR_STALE.merge.put(PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
REDUNDANT_PRE_BOOTSTRAP_OR_STALE.merge.put(REDUNDANT_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
REDUNDANT_PRE_BOOTSTRAP_OR_STALE.merge.put(LOCALLY_REDUNDANT, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
REDUNDANT_PRE_BOOTSTRAP_OR_STALE.merge.put(SHARD_REDUNDANT, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+ REDUNDANT_PRE_BOOTSTRAP_OR_STALE.merge.put(GC_BEFORE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
LOCALLY_REDUNDANT.merge = new EnumMap<>(RedundantStatus.class);
LOCALLY_REDUNDANT.merge.put(NOT_OWNED, LOCALLY_REDUNDANT);
LOCALLY_REDUNDANT.merge.put(WAS_OWNED, LOCALLY_REDUNDANT);
+ LOCALLY_REDUNDANT.merge.put(WAS_OWNED_CLOSED, LOCALLY_REDUNDANT);
+ LOCALLY_REDUNDANT.merge.put(WAS_OWNED_RETIRED, LOCALLY_REDUNDANT);
LOCALLY_REDUNDANT.merge.put(LIVE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE); // TODO (expected): should this be an invalid combination?
LOCALLY_REDUNDANT.merge.put(PARTIALLY_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
LOCALLY_REDUNDANT.merge.put(PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
LOCALLY_REDUNDANT.merge.put(REDUNDANT_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
LOCALLY_REDUNDANT.merge.put(LOCALLY_REDUNDANT, LOCALLY_REDUNDANT);
LOCALLY_REDUNDANT.merge.put(SHARD_REDUNDANT, LOCALLY_REDUNDANT);
+ LOCALLY_REDUNDANT.merge.put(GC_BEFORE, LOCALLY_REDUNDANT);
SHARD_REDUNDANT.merge = new EnumMap<>(RedundantStatus.class);
SHARD_REDUNDANT.merge.put(NOT_OWNED, SHARD_REDUNDANT);
SHARD_REDUNDANT.merge.put(WAS_OWNED, SHARD_REDUNDANT);
+ SHARD_REDUNDANT.merge.put(WAS_OWNED_CLOSED, SHARD_REDUNDANT);
+ SHARD_REDUNDANT.merge.put(WAS_OWNED_RETIRED, SHARD_REDUNDANT);
SHARD_REDUNDANT.merge.put(LIVE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE); // TODO (expected): should this be an invalid combination?
SHARD_REDUNDANT.merge.put(PARTIALLY_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
SHARD_REDUNDANT.merge.put(PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
SHARD_REDUNDANT.merge.put(REDUNDANT_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
SHARD_REDUNDANT.merge.put(LOCALLY_REDUNDANT, LOCALLY_REDUNDANT);
SHARD_REDUNDANT.merge.put(SHARD_REDUNDANT, SHARD_REDUNDANT);
+ SHARD_REDUNDANT.merge.put(GC_BEFORE, SHARD_REDUNDANT);
+ GC_BEFORE.merge = new EnumMap<>(RedundantStatus.class);
+ GC_BEFORE.merge.put(NOT_OWNED, SHARD_REDUNDANT);
+ GC_BEFORE.merge.put(WAS_OWNED, GC_BEFORE);
+ GC_BEFORE.merge.put(WAS_OWNED_CLOSED, GC_BEFORE);
+ GC_BEFORE.merge.put(WAS_OWNED_RETIRED, GC_BEFORE);
+ GC_BEFORE.merge.put(LIVE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE); // TODO (expected): should this be an invalid combination?
+ GC_BEFORE.merge.put(PARTIALLY_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+ GC_BEFORE.merge.put(PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+ GC_BEFORE.merge.put(REDUNDANT_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+ GC_BEFORE.merge.put(LOCALLY_REDUNDANT, LOCALLY_REDUNDANT);
+ GC_BEFORE.merge.put(SHARD_REDUNDANT, SHARD_REDUNDANT);
+ GC_BEFORE.merge.put(GC_BEFORE, GC_BEFORE);
}
public RedundantStatus merge(RedundantStatus that)
diff --git a/accord-core/src/main/java/accord/local/SafeCommandStore.java b/accord-core/src/main/java/accord/local/SafeCommandStore.java
index 72fb5bed..0c7a06f1 100644
--- a/accord-core/src/main/java/accord/local/SafeCommandStore.java
+++ b/accord-core/src/main/java/accord/local/SafeCommandStore.java
@@ -74,8 +74,8 @@ public abstract class SafeCommandStore
STARTED_AFTER,
ANY
}
- public enum TestDep { WITH, WITHOUT, ANY_DEPS }
- public enum TestStatus { ANY_STATUS, IS_PROPOSED, IS_STABLE }
+ public enum TestDep { WITH_OR_INVALIDATED, WITHOUT, ANY_DEPS }
+ public enum TestStatus { ANY_STATUS, IS_PROPOSED, IS_STABLE, IS_STABLE_OR_INVALIDATED }
/**
* If the transaction exists (with some associated data) in the CommandStore, return it. Otherwise return null.
@@ -164,17 +164,18 @@ public abstract class SafeCommandStore
*
* This permits efficient operation when a transaction involved in processing another transaction happens to be in memory.
*/
- public SafeCommand ifLoadedAndInitialisedAndNotErased(TxnId txnId)
+ public SafeCommand ifLoadedAndInitialised(TxnId txnId)
{
SafeCommand safeCommand = getInternal(txnId);
- if (safeCommand != null)
- return safeCommand.current().saveStatus().isUninitialised() ? null : safeCommand;
-
- safeCommand = ifLoadedAndInitialisedAndNotErasedInternal(txnId);
- if (safeCommand == null || safeCommand.current().saveStatus().isUninitialised())
- return null;
+ if (safeCommand == null)
+ {
+ safeCommand = ifLoadedInternal(txnId);
+ if (safeCommand == null)
+ return null;
+ }
- return maybeCleanup(safeCommand, safeCommand.current(), StoreParticipants.empty(txnId));
+ safeCommand = maybeCleanup(safeCommand, safeCommand.current(), StoreParticipants.empty(txnId));
+ return safeCommand.isUnset() ? null : safeCommand;
}
protected SafeCommandsForKey maybeCleanup(SafeCommandsForKey safeCfk)
@@ -220,8 +221,8 @@ public abstract class SafeCommandStore
/** Get anything already referenced (should include anything in PreLoadContext). If returned, should be initialised. */
protected abstract SafeCommand getInternal(TxnId txnId);
- /** Get if available and initialised */
- protected abstract SafeCommand ifLoadedAndInitialisedAndNotErasedInternal(TxnId txnId);
+ /** Get if available */
+ protected abstract SafeCommand ifLoadedInternal(TxnId txnId);
/** Get anything already referenced (should include anything in PreLoadContext) */
protected abstract SafeCommandsForKey getInternal(RoutingKey key);
/** Get if available */
@@ -412,7 +413,8 @@ public abstract class SafeCommandStore
CommandStore commandStore = safeStore.commandStore();
Ranges ranges = next.participants().touches.toRanges();
commandStore.registerTransitive(safeStore, next.partialDeps().rangeDeps);
- commandStore.markSynced(safeStore, txnId, ranges);
+ if (txnId.is(Kind.ExclusiveSyncPoint))
+ commandStore.markSynced(safeStore, txnId, ranges);
}
/**
diff --git a/accord-core/src/main/java/accord/local/StoreParticipants.java b/accord-core/src/main/java/accord/local/StoreParticipants.java
index 7c8a319b..349e423e 100644
--- a/accord-core/src/main/java/accord/local/StoreParticipants.java
+++ b/accord-core/src/main/java/accord/local/StoreParticipants.java
@@ -126,8 +126,9 @@ public class StoreParticipants
return new StoreParticipants(route, owns, touches, touches);
}
- public final StoreParticipants supplement(StoreParticipants that)
+ public final StoreParticipants supplement(@Nullable StoreParticipants that)
{
+ if (that == null) return this;
if (this == that) return this;
return supplement(that.route, that.owns, that.hasTouched);
}
@@ -242,9 +243,14 @@ public class StoreParticipants
public static StoreParticipants execute(SafeCommandStore safeStore, Participants<?> participants, TxnId txnId, long executeAtEpoch)
{
- long txnIdEpoch = txnId.epoch();
- Participants<?> owns = participants.slice(safeStore.ranges().allBetween(txnIdEpoch, executeAtEpoch), Minimal);
- return new StoreParticipants(tryCastToRoute(participants), owns, owns, owns);
+ return execute(safeStore, participants, txnId, txnId.epoch(), executeAtEpoch);
+ }
+
+ public static StoreParticipants execute(SafeCommandStore safeStore, Participants<?> participants, TxnId txnId, long minEpoch, long executeAtEpoch)
+ {
+ Participants<?> owns = participants.slice(safeStore.ranges().allBetween(txnId.epoch(), executeAtEpoch), Minimal);
+ Participants<?> touches = minEpoch >= txnId.epoch() ? owns : participants.slice(safeStore.ranges().allBetween(minEpoch, executeAtEpoch), Minimal);
+ return new StoreParticipants(tryCastToRoute(participants), owns, touches, touches);
}
public static StoreParticipants read(SafeCommandStore safeStore, Participants<?> participants, TxnId txnId, long minEpoch, long maxEpoch)
diff --git a/accord-core/src/main/java/accord/local/cfk/CommandsForKey.java b/accord-core/src/main/java/accord/local/cfk/CommandsForKey.java
index fa4162ee..fba7d689 100644
--- a/accord-core/src/main/java/accord/local/cfk/CommandsForKey.java
+++ b/accord-core/src/main/java/accord/local/cfk/CommandsForKey.java
@@ -58,11 +58,11 @@ import static accord.api.ProgressLog.BlockedUntil.CanApply;
import static accord.api.ProgressLog.BlockedUntil.HasStableDeps;
import static accord.local.cfk.CommandsForKey.InternalStatus.ACCEPTED;
import static accord.local.cfk.CommandsForKey.InternalStatus.APPLIED;
+import static accord.local.cfk.CommandsForKey.InternalStatus.INVALIDATED;
import static accord.local.cfk.CommandsForKey.InternalStatus.PREACCEPTED_OR_ACCEPTED_INVALIDATE;
import static accord.local.cfk.CommandsForKey.InternalStatus.STABLE;
-import static accord.local.cfk.CommandsForKey.InternalStatus.HISTORICAL;
-import static accord.local.cfk.CommandsForKey.InternalStatus.INVALID_OR_TRUNCATED_OR_PRUNED;
-import static accord.local.cfk.CommandsForKey.InternalStatus.TRANSITIVELY_KNOWN;
+import static accord.local.cfk.CommandsForKey.InternalStatus.TRANSITIVE;
+import static accord.local.cfk.CommandsForKey.InternalStatus.TRUNCATED_OR_PRUNED;
import static accord.local.cfk.PostProcess.notifyManagedPreBootstrap;
import static accord.local.cfk.Pruning.isAnyPredecessorWaitingOnPruned;
import static accord.local.cfk.Pruning.isWaitingOnPruned;
@@ -72,7 +72,7 @@ import static accord.local.cfk.Pruning.prunedBeforeId;
import static accord.local.cfk.UpdateUnmanagedMode.UPDATE;
import static accord.local.cfk.Updating.insertOrUpdate;
import static accord.local.SafeCommandStore.TestDep.ANY_DEPS;
-import static accord.local.SafeCommandStore.TestDep.WITH;
+import static accord.local.SafeCommandStore.TestDep.WITH_OR_INVALIDATED;
import static accord.primitives.Routable.Domain.Key;
import static accord.primitives.Txn.Kind.Kinds.AnyGloballyVisible;
import static accord.primitives.Txn.Kind.Write;
@@ -209,7 +209,7 @@ public class CommandsForKey extends CommandsForKeyUpdate implements CommandsSumm
private static final boolean ELIDE_TRANSITIVE_DEPENDENCIES = true;
public static final RedundantBefore.Entry NO_BOUNDS_INFO = new RedundantBefore.Entry(null, Long.MIN_VALUE, Long.MAX_VALUE, TxnId.NONE, TxnId.NONE, TxnId.NONE, TxnId.NONE, TxnId.NONE, TxnId.NONE, TxnId.NONE, null);
- public static final TxnInfo NO_INFO = TxnInfo.create(TxnId.NONE, HISTORICAL, false, TxnId.NONE, Ballot.ZERO);
+ public static final TxnInfo NO_INFO = TxnInfo.create(TxnId.NONE, TRANSITIVE, false, TxnId.NONE, Ballot.ZERO);
public static final TxnInfo[] NO_INFOS = new TxnInfo[0];
public static final Unmanaged[] NO_PENDING_UNMANAGED = new Unmanaged[0];
@@ -313,13 +313,13 @@ public class CommandsForKey extends CommandsForKeyUpdate implements CommandsSumm
{
public static CommandsForKey create(RoutingKey key, TxnInfo[] txns, Unmanaged[] unmanageds, TxnId prunedBefore, RedundantBefore.Entry boundsInfo)
{
- return reconstruct(key, boundsInfo, txns, prunedBefore, unmanageds);
+ return reconstruct(key, boundsInfo, true, txns, prunedBefore, unmanageds);
}
}
interface Updater<O>
{
- O update(RoutingKey key, RedundantBefore.Entry boundsInfo, TxnInfo[] byId, TxnInfo[] committedByExecuteAt, int minUndecidedById, int maxAppliedWriteByExecuteAt, Object[] loadingPruned, int newPrunedBeforeById, Unmanaged[] unmanageds);
+ O update(RoutingKey key, RedundantBefore.Entry boundsInfo, boolean isNewBoundsInfo, TxnInfo[] byId, TxnInfo[] committedByExecuteAt, int minUndecidedById, int maxAppliedWriteByExecuteAt, Object[] loadingPruned, int newPrunedBeforeById, Unmanaged[] unmanageds);
}
/**
@@ -337,7 +337,7 @@ public class CommandsForKey extends CommandsForKeyUpdate implements CommandsSumm
static final int NOTIFIED_READY = 0x1000;
static final int NOTIFIED_WAITING = 0x2000;
- final int encodedStatus;
+ int encodedStatus;
public final Timestamp executeAt;
private TxnInfo(TxnId txnId, int encodedStatus, Timestamp executeAt)
@@ -392,6 +392,21 @@ public class CommandsForKey extends CommandsForKeyUpdate implements CommandsSumm
return (encodedStatus & 0xf) == status.ordinal();
}
+ boolean isNot(InternalStatus status)
+ {
+ return (encodedStatus & 0xf) != status.ordinal();
+ }
+
+ boolean isAtLeast(InternalStatus status)
+ {
+ return (encodedStatus & 0xf) >= status.ordinal();
+ }
+
+ int compareTo(InternalStatus status)
+ {
+ return (encodedStatus & 0xf) - status.ordinal();
+ }
+
boolean is(TestStatus testStatus)
{
InternalStatus status = status();
@@ -402,8 +417,10 @@ public class CommandsForKey extends CommandsForKeyUpdate implements CommandsSumm
return status.isProposed();
case IS_STABLE:
return status.isStable();
+ case IS_STABLE_OR_INVALIDATED:
+ return status.isStableOrInvalidated();
case ANY_STATUS:
- return status != TRANSITIVELY_KNOWN;
+ return status != TRANSITIVE;
}
}
@@ -543,14 +560,14 @@ public class CommandsForKey extends CommandsForKeyUpdate implements CommandsSumm
return this.executeAt.compareTo(that.executeAt);
}
- public int compareExecuteAtEpoch(TxnInfo that)
+ public int compareExecuteAt(Timestamp that)
{
- return Long.compare(this.executeAt.epoch(), that.executeAt.epoch());
+ return this.executeAt.compareTo(that);
}
- Timestamp executeAtIfKnownElseTxnId()
+ public int compareExecuteAtEpoch(TxnInfo that)
{
- return status() == INVALID_OR_TRUNCATED_OR_PRUNED ? this : executeAt;
+ return Long.compare(this.executeAt.epoch(), that.executeAt.epoch());
}
private static int encode(TxnId txnId, InternalStatus internalStatus, boolean mayExecute)
@@ -583,14 +600,14 @@ public class CommandsForKey extends CommandsForKeyUpdate implements CommandsSumm
return 0 != (encodedStatus & NOTIFIED_WAITING);
}
- public TxnInfo asNotifiedReady()
+ void setNotifiedReadyInPlace()
{
- return withEncodedStatus(encodedStatus | NOTIFIED_READY);
+ encodedStatus |= NOTIFIED_READY;
}
- public TxnInfo asNotifiedWaiting()
+ void setNotifiedWaitingInPlace()
{
- return withEncodedStatus(encodedStatus | NOTIFIED_WAITING);
+ encodedStatus |= NOTIFIED_WAITING;
}
}
@@ -678,7 +695,7 @@ public class CommandsForKey extends CommandsForKeyUpdate implements CommandsSumm
{
this.pending = pending;
this.txnId = txnId;
- this.waitingUntil = waitingUntil;
+ this.waitingUntil = Invariants.checkArgument(waitingUntil, waitingUntil != null);
}
@Override
@@ -716,14 +733,15 @@ public class CommandsForKey extends CommandsForKeyUpdate implements CommandsSumm
{
// TODO (expected): collapse historical and transitively known into a single concept;
// no reason to not witness transitive deps, and then registerHistorical can rely on ExclusiveSyncPoints.
- TRANSITIVELY_KNOWN(false, false),
- HISTORICAL(false, false),
+ TRANSITIVE(false, false),
PREACCEPTED_OR_ACCEPTED_INVALIDATE(false, true),
ACCEPTED(true, true),
COMMITTED(true, true),
STABLE(true, false),
APPLIED(true, false),
- INVALID_OR_TRUNCATED_OR_PRUNED(false, false);
+ INVALIDATED(false, false),
+ TRUNCATED_OR_PRUNED(false, false)
+ ;
static final EnumMap<SaveStatus, InternalStatus> convert = new EnumMap<>(SaveStatus.class);
static final InternalStatus[] VALUES = values();
@@ -743,13 +761,12 @@ public class CommandsForKey extends CommandsForKeyUpdate implements CommandsSumm
convert.put(SaveStatus.PreApplied, STABLE);
convert.put(SaveStatus.Applying, STABLE);
convert.put(SaveStatus.Applied, APPLIED);
+ // We don't map TruncatedApplyX as we want to retain them as APPLIED
+ // esp. to support pruning where we expect the prunedBefore entry to be APPLIED
// TODO (expected): can we simply delete in the following cases?
- convert.put(SaveStatus.TruncatedApplyWithDeps, INVALID_OR_TRUNCATED_OR_PRUNED);
- convert.put(SaveStatus.TruncatedApplyWithOutcome, INVALID_OR_TRUNCATED_OR_PRUNED);
- convert.put(SaveStatus.TruncatedApply, INVALID_OR_TRUNCATED_OR_PRUNED);
- convert.put(SaveStatus.Erased, INVALID_OR_TRUNCATED_OR_PRUNED);
- convert.put(SaveStatus.ErasedOrVestigial, INVALID_OR_TRUNCATED_OR_PRUNED);
- convert.put(SaveStatus.Invalidated, INVALID_OR_TRUNCATED_OR_PRUNED);
+ convert.put(SaveStatus.Erased, TRUNCATED_OR_PRUNED);
+ convert.put(SaveStatus.ErasedOrVestigial, TRUNCATED_OR_PRUNED);
+ convert.put(SaveStatus.Invalidated, INVALIDATED);
}
public final boolean hasInfo;
@@ -783,6 +800,11 @@ public class CommandsForKey extends CommandsForKeyUpdate implements CommandsSumm
return this == STABLE | this == APPLIED;
}
+ public boolean isStableOrInvalidated()
+ {
+ return this == STABLE | this == APPLIED | this == INVALIDATED;
+ }
+
public boolean isCommittedToExecute()
{
return this == COMMITTED | this == STABLE | this == APPLIED;
@@ -830,6 +852,11 @@ public class CommandsForKey extends CommandsForKeyUpdate implements CommandsSumm
final Unmanaged[] unmanageds;
+ CommandsForKey(RoutingKey key, RedundantBefore.Entry boundsInfo, boolean isNewBoundsInfo, TxnInfo[] byId, TxnInfo[] committedByExecuteAt, int minUndecidedById, int maxAppliedWriteByExecuteAt, Object[] loadingPruned, int prunedBeforeById, Unmanaged[] unmanageds)
+ {
+ this(key, boundsInfo, byId, committedByExecuteAt, minUndecidedById, maxAppliedWriteByExecuteAt, loadingPruned, prunedBeforeById, unmanageds);
+ }
+
CommandsForKey(RoutingKey key, RedundantBefore.Entry boundsInfo, TxnInfo[] byId, TxnInfo[] committedByExecuteAt, int minUndecidedById, int maxAppliedWriteByExecuteAt, Object[] loadingPruned, int prunedBeforeById, Unmanaged[] unmanageds)
{
this.key = key;
@@ -1078,7 +1105,7 @@ public class CommandsForKey extends CommandsForKeyUpdate implements CommandsSumm
case ANY_DEPS:
break;
- case WITH:
+ case WITH_OR_INVALIDATED:
if (testTxnId.compareTo(prunedBefore) >= 0)
return initialValue;
@@ -1110,32 +1137,38 @@ public class CommandsForKey extends CommandsForKeyUpdate implements CommandsSumm
if (testDep != ANY_DEPS)
{
if (!txn.hasDeps())
- continue;
-
- if (executeAt.compareTo(testTxnId) <= 0)
- continue;
-
- boolean hasAsDep;
- if (loadingFor == null)
{
- TxnId[] missing = txn.missing();
- hasAsDep = missing == NO_TXNIDS || Arrays.binarySearch(txn.missing(), testTxnId) < 0;
- }
- else if (loadingFor == NO_TXNIDS)
- {
- hasAsDep = false;
+ if (testDep == TestDep.WITHOUT || !txn.is(INVALIDATED))
+ continue;
}
else
{
- // we could use expontentialSearch and moving index for improved algorithmic complexity,
- // but since should be rarely taken path probably not worth code complexity
- loadingIndex = SortedArrays.exponentialSearch(loadingFor, loadingIndex, loadingFor.length, txn);
- if (hasAsDep = (loadingIndex >= 0)) ++loadingIndex;
- else loadingIndex = -1 - loadingIndex;
+ if (executeAt.compareTo(testTxnId) <= 0)
+ continue;
+
+ boolean hasAsDep;
+ if (loadingFor == null)
+ {
+ TxnId[] missing = txn.missing();
+ hasAsDep = missing == NO_TXNIDS || Arrays.binarySearch(txn.missing(), testTxnId) < 0;
+ }
+ else if (loadingFor == NO_TXNIDS)
+ {
+ hasAsDep = false;
+ }
+ else
+ {
+ // we could use expontentialSearch and moving index for improved algorithmic complexity,
+ // but since should be rarely taken path probably not worth code complexity
+ loadingIndex = SortedArrays.exponentialSearch(loadingFor, loadingIndex, loadingFor.length, txn);
+ if (hasAsDep = (loadingIndex >= 0)) ++loadingIndex;
+ else loadingIndex = -1 - loadingIndex;
+ }
+
+ if (hasAsDep != (testDep == WITH_OR_INVALIDATED))
+ continue;
}
- if (hasAsDep != (testDep == WITH))
- continue;
}
initialValue = map.apply(p1, key, txn.plainTxnId(), executeAt, initialValue);
@@ -1178,8 +1211,9 @@ public class CommandsForKey extends CommandsForKeyUpdate implements CommandsSumm
case APPLIED:
if (!ELIDE_TRANSITIVE_DEPENDENCIES || maxCommittedWriteBefore == null || txn.executeAt.compareTo(maxCommittedWriteBefore) >= 0 || !Write.witnesses(txn))
break;
- case TRANSITIVELY_KNOWN:
- case INVALID_OR_TRUNCATED_OR_PRUNED:
+ case TRANSITIVE:
+ case INVALIDATED:
+ case TRUNCATED_OR_PRUNED:
continue;
}
@@ -1234,9 +1268,9 @@ public class CommandsForKey extends CommandsForKeyUpdate implements CommandsSumm
{
InternalStatus newStatus = InternalStatus.from(next.saveStatus());
if (newStatus == null)
- newStatus = TRANSITIVELY_KNOWN;
+ newStatus = TRANSITIVE;
if (!manages(next.txnId()))
- newStatus = newStatus.compareTo(InternalStatus.COMMITTED) < 0 ? TRANSITIVELY_KNOWN : INVALID_OR_TRUNCATED_OR_PRUNED;
+ newStatus = newStatus.compareTo(InternalStatus.COMMITTED) < 0 ? TRANSITIVE : TRUNCATED_OR_PRUNED;
return update(newStatus, next, false, true);
}
@@ -1249,7 +1283,7 @@ public class CommandsForKey extends CommandsForKeyUpdate implements CommandsSumm
return this;
boolean mayExecute = mayExecute(txnId, newStatus, next);
- if (isOutOfRange && newStatus == INVALID_OR_TRUNCATED_OR_PRUNED) isOutOfRange = false; // invalidated is safe to use anywhere, and erases deps
+ if (isOutOfRange && newStatus.compareTo(INVALIDATED) >= 0) isOutOfRange = false; // invalidated is safe to use anywhere, and erases deps
TxnId[] loadingAsPrunedFor = loadingPrunedFor(loadingPruned, txnId, null); // we default to null to distinguish between no match, and a match with NO_TXNIDS
wasPruned |= loadingAsPrunedFor != null;
@@ -1273,12 +1307,12 @@ public class CommandsForKey extends CommandsForKeyUpdate implements CommandsSumm
if (cur != null)
{
- int c = newStatus.compareTo(cur.status());
- if (c <= 0)
+ int c = cur.compareTo(newStatus);
+ if (c >= 0)
{
- if (c < 0)
+ if (c > 0)
{
- if (!(newStatus == PREACCEPTED_OR_ACCEPTED_INVALIDATE && cur.status() == ACCEPTED && next.acceptedOrCommitted().compareTo(cur.ballot()) > 0))
+ if (!(newStatus == PREACCEPTED_OR_ACCEPTED_INVALIDATE && cur.is(ACCEPTED) && next.acceptedOrCommitted().compareTo(cur.ballot()) > 0))
return this;
}
else
@@ -1338,31 +1372,31 @@ public class CommandsForKey extends CommandsForKeyUpdate implements CommandsSumm
CommandsForKeyUpdate update(TxnInfo[] newById, int newMinUndecidedById, TxnInfo[] newCommittedByExecuteAt, int newMaxAppliedWriteByExecuteAt, Object[] newLoadingPruned, int newPrunedBeforeById, @Nullable TxnInfo curInfo, @Nonnull TxnInfo newInfo)
{
Invariants.checkState(prunedBeforeById < 0 || newById[newPrunedBeforeById].equals(byId[prunedBeforeById]));
- return updateAndNotifyUnmanageds(key, boundsInfo,
+ return updateAndNotifyUnmanageds(key, boundsInfo, false,
newById, newCommittedByExecuteAt, newMinUndecidedById, newMaxAppliedWriteByExecuteAt,
newLoadingPruned, newPrunedBeforeById, unmanageds, curInfo, newInfo);
}
- static CommandsForKey reconstruct(RoutingKey key, RedundantBefore.Entry boundsInfo, TxnInfo[] byId, TxnId prunedBefore, Unmanaged[] unmanageds)
+ static CommandsForKey reconstruct(RoutingKey key, RedundantBefore.Entry boundsInfo, boolean isNewBoundsInfo, TxnInfo[] byId, TxnId prunedBefore, Unmanaged[] unmanageds)
{
int prunedBeforeById = Arrays.binarySearch(byId, prunedBefore);
Invariants.checkState(prunedBeforeById >= 0 || prunedBefore.equals(TxnId.NONE));
- return reconstruct(key, boundsInfo, byId, BTree.empty(), prunedBeforeById, unmanageds);
+ return reconstruct(key, boundsInfo, isNewBoundsInfo, byId, BTree.empty(), prunedBeforeById, unmanageds);
}
- static CommandsForKey reconstruct(RoutingKey key, RedundantBefore.Entry newBoundsInfo, TxnInfo[] byId, Object[] loadingPruned, int newPrunedBeforeById, Unmanaged[] unmanageds)
+ static CommandsForKey reconstruct(RoutingKey key, RedundantBefore.Entry boundsInfo, boolean isNewBoundsInfo, TxnInfo[] byId, Object[] loadingPruned, int newPrunedBeforeById, Unmanaged[] unmanageds)
{
- return reconstruct(key, newBoundsInfo, byId, loadingPruned, newPrunedBeforeById, unmanageds, CommandsForKey::new);
+ return reconstruct(key, boundsInfo, isNewBoundsInfo, byId, loadingPruned, newPrunedBeforeById, unmanageds, CommandsForKey::new);
}
- static <O> O reconstruct(RoutingKey key, RedundantBefore.Entry newBoundsInfo, TxnInfo[] byId, Object[] loadingPruned, int newPrunedBeforeById, Unmanaged[] unmanageds, Updater<O> updater)
+ static <O> O reconstruct(RoutingKey key, RedundantBefore.Entry boundsInfo, boolean isNewBoundsInfo, TxnInfo[] byId, Object[] loadingPruned, int newPrunedBeforeById, Unmanaged[] unmanageds, Updater<O> updater)
{
int countCommitted = 0;
int minUndecidedById = -1;
for (int i = 0; i < byId.length ; ++i)
{
TxnInfo txn = byId[i];
- if (txn.status() == INVALID_OR_TRUNCATED_OR_PRUNED) continue;
+ if (txn.isAtLeast(INVALIDATED)) continue;
if (txn.isCommittedAndExecutes()) ++countCommitted;
else if (minUndecidedById == -1 && !txn.isCommittedToExecute() && txn.mayExecute())
minUndecidedById = i;
@@ -1379,26 +1413,26 @@ public class CommandsForKey extends CommandsForKeyUpdate implements CommandsSumm
while (--maxAppliedWriteByExecuteAt >= 0)
{
TxnInfo txn = committedByExecuteAt[maxAppliedWriteByExecuteAt];
- if (txn.kind() == Write && (txn.status() == APPLIED || isPreBootstrap(txn, newBoundsInfo)))
+ if (txn.kind() == Write && (txn.is(APPLIED) || isPreBootstrap(txn, boundsInfo)))
break;
}
- return updater.update(key, newBoundsInfo, byId, committedByExecuteAt, minUndecidedById, maxAppliedWriteByExecuteAt, loadingPruned, newPrunedBeforeById, unmanageds);
+ return updater.update(key, boundsInfo, isNewBoundsInfo, byId, committedByExecuteAt, minUndecidedById, maxAppliedWriteByExecuteAt, loadingPruned, newPrunedBeforeById, unmanageds);
}
- static CommandsForKeyUpdate reconstructAndUpdateUnmanaged(RoutingKey key, RedundantBefore.Entry boundsInfo, TxnInfo[] byId, Object[] loadingPruned, int newPrunedBeforeById, Unmanaged[] unmanageds)
+ static CommandsForKeyUpdate reconstructAndUpdateUnmanaged(RoutingKey key, RedundantBefore.Entry boundsInfo, boolean isNewBoundsInfo, TxnInfo[] byId, Object[] loadingPruned, int newPrunedBeforeById, Unmanaged[] unmanageds)
{
- return reconstruct(key, boundsInfo, byId, loadingPruned, newPrunedBeforeById, unmanageds, CommandsForKey::updateAndNotifyUnmanageds);
+ return reconstruct(key, boundsInfo, isNewBoundsInfo, byId, loadingPruned, newPrunedBeforeById, unmanageds, CommandsForKey::updateAndNotifyUnmanageds);
}
- static CommandsForKeyUpdate updateAndNotifyUnmanageds(RoutingKey key, RedundantBefore.Entry boundsInfo, TxnInfo[] byId, TxnInfo[] committedByExecuteAt, int minUndecidedById, int maxAppliedWriteByExecuteAt, Object[] loadingPruned, int newPrunedBeforeById, Unmanaged[] unmanageds)
+ static CommandsForKeyUpdate updateAndNotifyUnmanageds(RoutingKey key, RedundantBefore.Entry boundsInfo, boolean isNewBoundsInfo, TxnInfo[] byId, TxnInfo[] committedByExecuteAt, int minUndecidedById, int maxAppliedWriteByExecuteAt, Object[] loadingPruned, int newPrunedBeforeById, Unmanaged[] unmanageds)
{
- return updateAndNotifyUnmanageds(key, boundsInfo, byId, committedByExecuteAt, minUndecidedById, maxAppliedWriteByExecuteAt, loadingPruned, newPrunedBeforeById, unmanageds, null, null);
+ return updateAndNotifyUnmanageds(key, boundsInfo, isNewBoundsInfo, byId, committedByExecuteAt, minUndecidedById, maxAppliedWriteByExecuteAt, loadingPruned, newPrunedBeforeById, unmanageds, null, null);
}
- static CommandsForKeyUpdate updateAndNotifyUnmanageds(RoutingKey key, RedundantBefore.Entry boundsInfo, TxnInfo[] byId, TxnInfo[] committedByExecuteAt, int minUndecidedById, int maxAppliedWriteByExecuteAt, Object[] loadingPruned, int newPrunedBeforeById, Unmanaged[] unmanageds, @Nullable TxnInfo curInfo, @Nullable TxnInfo newInfo)
+ static CommandsForKeyUpdate updateAndNotifyUnmanageds(RoutingKey key, RedundantBefore.Entry boundsInfo, boolean isNewBoundsInfo, TxnInfo[] byId, TxnInfo[] committedByExecuteAt, int minUndecidedById, int maxAppliedWriteByExecuteAt, Object[] loadingPruned, int newPrunedBeforeById, Unmanaged[] unmanageds, @Nullable TxnInfo curInfo, @Nullable TxnInfo newInfo)
{
- NotifyUnmanagedResult notifyUnmanaged = PostProcess.notifyUnmanaged(unmanageds, byId, minUndecidedById, committedByExecuteAt, maxAppliedWriteByExecuteAt, loadingPruned, boundsInfo, curInfo, newInfo);
+ NotifyUnmanagedResult notifyUnmanaged = PostProcess.notifyUnmanaged(unmanageds, byId, minUndecidedById, committedByExecuteAt, maxAppliedWriteByExecuteAt, loadingPruned, boundsInfo, isNewBoundsInfo, curInfo, newInfo);
if (notifyUnmanaged != null)
unmanageds = notifyUnmanaged.newUnmanaged;
CommandsForKey result = new CommandsForKey(key, boundsInfo, byId, committedByExecuteAt, minUndecidedById, maxAppliedWriteByExecuteAt, loadingPruned, newPrunedBeforeById, unmanageds);
@@ -1471,7 +1505,7 @@ public class CommandsForKey extends CommandsForKeyUpdate implements CommandsSumm
}
TxnInfo prevInfo = prevCfk.get(updatedTxnId);
- InternalStatus prevStatus = prevInfo == null ? TRANSITIVELY_KNOWN : prevInfo.status();
+ InternalStatus prevStatus = prevInfo == null ? TRANSITIVE : prevInfo.status();
int mayExecuteToIndex;
int mayExecuteAnyAtIndex = -1;
@@ -1527,7 +1561,7 @@ public class CommandsForKey extends CommandsForKeyUpdate implements CommandsSumm
// cannot execute as dependencies not stable, so notify progress log to get or decide stable deps
notifySink.waitingOn(safeStore, txn, key, SaveStatus.Stable, HasStableDeps, true);
- updateCommittedByExecuteAtInSitu(i, txn.asNotifiedWaiting());
+ txn.setNotifiedWaitingInPlace();
break;
}
@@ -1543,7 +1577,7 @@ public class CommandsForKey extends CommandsForKeyUpdate implements CommandsSumm
while (undecidedIndex < nextUndecidedIndex)
{
TxnInfo backfillTxn = byId[undecidedIndex++];
- if (backfillTxn.status().compareTo(InternalStatus.COMMITTED) >= 0 || !mayExecute(backfillTxn)) continue;
+ if (backfillTxn.compareTo(InternalStatus.COMMITTED) >= 0 || !mayExecute(backfillTxn)) continue;
unappliedCounters += unappliedCountersDelta(backfillTxn.kind());
}
}
@@ -1578,7 +1612,7 @@ public class CommandsForKey extends CommandsForKeyUpdate implements CommandsSumm
notifySink.notWaiting(safeStore, txnId, key);
// TODO (required): avoid invoking this here; we may do redundant work if we have local dependencies we're already waiting on
notifySink.waitingOn(safeStore, txn, key, SaveStatus.PreApplied, CanApply, false);
- updateCommittedByExecuteAtInSitu(i, txn.asNotifiedReady());
+ txn.setNotifiedReadyInPlace();
}
}
}
@@ -1602,7 +1636,6 @@ public class CommandsForKey extends CommandsForKeyUpdate implements CommandsSumm
switch (kind)
{
default: throw new AssertionError("Unhandled Txn.Kind: " + kind);
- case LocalOnly:
case EphemeralRead:
throw illegalState("Invalid Txn.Kind for CommandsForKey: " + kind);
@@ -1623,7 +1656,6 @@ public class CommandsForKey extends CommandsForKeyUpdate implements CommandsSumm
switch (kind)
{
default: throw new AssertionError("Unhandled Txn.Kind: " + kind);
- case LocalOnly:
case EphemeralRead:
case ExclusiveSyncPoint:
throw illegalState("Invalid Txn.Kind for CommandsForKey: " + kind);
@@ -1671,7 +1703,7 @@ public class CommandsForKey extends CommandsForKeyUpdate implements CommandsSumm
int newPrunedBeforeById = prunedBeforeId(newById, prunedBefore(), redundantBefore(newBoundsInfo));
Object[] newLoadingPruned = Pruning.removeRedundantLoadingPruned(loadingPruned, redundantBefore(newBoundsInfo));
- return notifyManagedPreBootstrap(this, newBoundsInfo, reconstructAndUpdateUnmanaged(key, newBoundsInfo, newById, newLoadingPruned, newPrunedBeforeById, unmanageds));
+ return notifyManagedPreBootstrap(this, newBoundsInfo, reconstructAndUpdateUnmanaged(key, newBoundsInfo, true, newById, newLoadingPruned, newPrunedBeforeById, unmanageds));
}
/**
@@ -1688,7 +1720,7 @@ public class CommandsForKey extends CommandsForKeyUpdate implements CommandsSumm
int newPrunedBeforeById = prunedBeforeId(newById, prunedBefore(), newRedundantBefore);
Object[] newLoadingPruned = Pruning.removeRedundantLoadingPruned(loadingPruned, newRedundantBefore);
- return reconstruct(key, newBoundsInfo, newById, newLoadingPruned, newPrunedBeforeById, unmanageds);
+ return reconstruct(key, newBoundsInfo, true, newById, newLoadingPruned, newPrunedBeforeById, unmanageds);
}
/**
@@ -1769,7 +1801,7 @@ public class CommandsForKey extends CommandsForKeyUpdate implements CommandsSumm
{
TxnInfo txn = committedByExecuteAt[i];
// TODO (expected): should we count any final run of !managesExecution()? i.e. if we have Y(es)N(o)YNYNNN, should we not stop after only YNYNY?
- if (txn.status() != APPLIED && managesExecution(txn) && (bootstrappedAt == null || bootstrappedAt.compareTo(txn) <= 0))
+ if (!txn.is(APPLIED) && managesExecution(txn) && (bootstrappedAt == null || bootstrappedAt.compareTo(txn) <= 0))
break;
++i;
}
diff --git a/accord-core/src/main/java/accord/local/cfk/NotifySink.java b/accord-core/src/main/java/accord/local/cfk/NotifySink.java
index b6a415df..e9875b8c 100644
--- a/accord-core/src/main/java/accord/local/cfk/NotifySink.java
+++ b/accord-core/src/main/java/accord/local/cfk/NotifySink.java
@@ -49,7 +49,7 @@ interface NotifySink
@Override
public void notWaiting(SafeCommandStore safeStore, TxnId txnId, RoutingKey key)
{
- SafeCommand safeCommand = safeStore.ifLoadedAndInitialisedAndNotErased(txnId);
+ SafeCommand safeCommand = safeStore.ifLoadedAndInitialised(txnId);
if (safeCommand != null) notWaiting(safeStore, safeCommand, key);
else
{
@@ -106,7 +106,7 @@ interface NotifySink
}
else
{
- safeStore.progressLog().waiting(blockedUntil, safeStore, safeCommand, null, RoutingKeys.of(key.toUnseekable()));
+ safeStore.progressLog().waiting(blockedUntil, safeStore, safeCommand, null, RoutingKeys.of(key.toUnseekable()), null);
}
}
diff --git a/accord-core/src/main/java/accord/local/cfk/PostProcess.java b/accord-core/src/main/java/accord/local/cfk/PostProcess.java
index 1c187a00..d49b39c6 100644
--- a/accord-core/src/main/java/accord/local/cfk/PostProcess.java
+++ b/accord-core/src/main/java/accord/local/cfk/PostProcess.java
@@ -21,6 +21,7 @@ package accord.local.cfk;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.function.Predicate;
import javax.annotation.Nullable;
@@ -42,7 +43,7 @@ import accord.utils.btree.BTree;
import static accord.local.KeyHistory.SYNC;
import static accord.local.cfk.CommandsForKey.InternalStatus.APPLIED;
-import static accord.local.cfk.CommandsForKey.InternalStatus.INVALID_OR_TRUNCATED_OR_PRUNED;
+import static accord.local.cfk.CommandsForKey.InternalStatus.INVALIDATED;
import static accord.local.cfk.CommandsForKey.InternalStatus.STABLE;
import static accord.local.cfk.CommandsForKey.Unmanaged.Pending.APPLY;
import static accord.local.cfk.CommandsForKey.maxContiguousManagedAppliedIndex;
@@ -56,6 +57,7 @@ import static accord.local.cfk.Utils.removeUnmanaged;
import static accord.local.cfk.Utils.selectUnmanaged;
import static accord.primitives.TxnId.NO_TXNIDS;
import static accord.utils.ArrayBuffers.cachedTxnIds;
+import static accord.utils.SortedArrays.Search.FAST;
abstract class PostProcess
{
@@ -91,7 +93,7 @@ abstract class PostProcess
for (TxnId txnId : load)
{
safeStore = safeStore; // make it unsafe for use in lambda
- SafeCommand safeCommand = safeStore.ifLoadedAndInitialisedAndNotErased(txnId);
+ SafeCommand safeCommand = safeStore.ifLoadedAndInitialised(txnId);
if (safeCommand != null) load(safeStore, safeCommand, safeCfk, notifySink);
else
safeStore.commandStore().execute(PreLoadContext.contextFor(txnId, RoutingKeys.of(key), SYNC), safeStore0 -> {
@@ -148,7 +150,7 @@ abstract class PostProcess
if (txn.compareTo(newBoundsInfo.bootstrappedAt) >= 0)
break;
- if (txn.status() == STABLE)
+ if (txn.is(STABLE))
{
if (notifyCount == notify.length)
notify = cachedTxnIds().resize(notify, notifyCount, Math.max(notifyCount * 2, 8));
@@ -196,7 +198,7 @@ abstract class PostProcess
List<PostProcess> nestedNotify = new ArrayList<>();
for (TxnId txnId : notify)
{
- SafeCommand safeCommand = safeStore.ifLoadedAndInitialisedAndNotErased(txnId);
+ SafeCommand safeCommand = safeStore.ifLoadedAndInitialised(txnId);
if (safeCommand != null)
{
CommandsForKeyUpdate update = updateUnmanaged(cfk, safeCommand, UPDATE, addUnmanageds);
@@ -246,10 +248,12 @@ abstract class PostProcess
int maxAppliedWriteByExecuteAt,
Object[] loadingPruned,
RedundantBefore.Entry boundsInfo,
+ boolean isNewBoundsInfo,
@Nullable TxnInfo curInfo,
@Nullable TxnInfo newInfo)
{
- TxnId redundantBefore = boundsInfo.shardRedundantBefore();
+ // TODO (expected): can we relax this to shardRedundantBefore?
+ TxnId redundantBefore = boundsInfo.gcBefore();
TxnId bootstrappedAt = boundsInfo.bootstrappedAt;
if (bootstrappedAt.compareTo(redundantBefore) <= 0) bootstrappedAt = null;
@@ -272,7 +276,7 @@ abstract class PostProcess
{
Timestamp applyTo = null;
- if (newInfo != null && newInfo.status() == APPLIED)
+ if (newInfo != null && newInfo.is(APPLIED))
{
TxnInfo maxContiguousApplied = CommandsForKey.maxContiguousManagedApplied(committedByExecuteAt, maxAppliedWriteByExecuteAt, bootstrappedAt);
if (maxContiguousApplied != null && maxContiguousApplied.compareExecuteAt(newInfo) >= 0)
@@ -283,7 +287,7 @@ abstract class PostProcess
TxnInfo maxContiguousApplied = CommandsForKey.maxContiguousManagedApplied(committedByExecuteAt, maxAppliedWriteByExecuteAt, bootstrappedAt);
if (maxContiguousApplied != null)
applyTo = maxContiguousApplied.executeAt;
- // if we're updating bootstrappedAt, we can fire anyone waiting on an executeAt before the bootstrappedAt
+
applyTo = Timestamp.nonNullOrMax(applyTo, TxnId.nonNullOrMax(redundantBefore, bootstrappedAt));
}
@@ -300,39 +304,73 @@ abstract class PostProcess
}
}
- if (newInfo != null && newInfo.status() == INVALID_OR_TRUNCATED_OR_PRUNED && curInfo != null && curInfo.status().isCommittedToExecute())
+ Predicate<Timestamp> rescheduleOrNotifyIf = null;
+ if ((newInfo != null && newInfo.isAtLeast(INVALIDATED) && curInfo != null && curInfo.isCommittedToExecute()))
{
- // this is a rare edge case, but we might have unmanaged transactions waiting on this command we must re-schedule or notify
- int start = findFirstApply(unmanageds);
- int end = start;
- while (end < unmanageds.length && unmanageds[end].waitingUntil.equals(curInfo.executeAt))
- ++end;
+ rescheduleOrNotifyIf = curInfo.executeAt::equals;
+ }
- if (start != end)
+ if (isNewBoundsInfo && bootstrappedAt != null)
+ {
+ Timestamp maxPreBootstrap;
{
- // find committed predecessor, if any
- int predecessor = -2 - Arrays.binarySearch(committedByExecuteAt, curInfo, TxnInfo::compareExecuteAt);
-
- if (predecessor >= 0)
+ Timestamp tmp = bootstrappedAt;
+ for (int i = 0; i < byId.length; ++i)
{
- int maxContiguousApplied = maxContiguousManagedAppliedIndex(committedByExecuteAt, maxAppliedWriteByExecuteAt, bootstrappedAt);
- if (maxContiguousApplied >= predecessor)
- predecessor = -1;
+ TxnInfo txn = byId[i];
+ if (txn.compareTo(bootstrappedAt) > 0)
+ break;
+ tmp = Timestamp.nonNullOrMax(tmp, txn.executeAt);
}
+ maxPreBootstrap = tmp;
+ }
+ if (rescheduleOrNotifyIf == null)
+ rescheduleOrNotifyIf = test -> test.compareTo(maxPreBootstrap) <= 0;
+ else
+ rescheduleOrNotifyIf = test -> curInfo.executeAt.equals(test) || test.compareTo(maxPreBootstrap) <= 0;
+ }
- if (predecessor >= 0)
- {
- Timestamp waitingUntil = committedByExecuteAt[predecessor].plainExecuteAt();
- unmanageds = unmanageds.clone();
- for (int i = start ; i < end ; ++i)
- unmanageds[i] = new Unmanaged(APPLY, unmanageds[i].txnId, waitingUntil);
- }
- else
+ if (rescheduleOrNotifyIf != null)
+ {
+ // this is a rare edge case, but we might have unmanaged transactions waiting on this command we must re-schedule or notify
+ boolean clone = true;
+ int start = findFirstApply(unmanageds);
+ while (start < unmanageds.length)
+ {
+ if (rescheduleOrNotifyIf.test(unmanageds[start].waitingUntil))
{
- TxnId[] notifyNotWaiting = selectUnmanaged(unmanageds, start, end);
- unmanageds = removeUnmanaged(unmanageds, start, end);
- notifier = new PostProcess.NotifyNotWaiting(notifier, notifyNotWaiting);
+ int end = start + 1;
+ while (end < unmanageds.length && rescheduleOrNotifyIf.test(unmanageds[end].waitingUntil))
+ ++end;
+
+ // find committed predecessor, if any
+ int predecessor = -2 - SortedArrays.binarySearch(committedByExecuteAt, 0, committedByExecuteAt.length, unmanageds[start].waitingUntil, (t, i) -> t.compareTo(i.executeAt), FAST);
+
+ if (predecessor >= 0)
+ {
+ int maxContiguousApplied = maxContiguousManagedAppliedIndex(committedByExecuteAt, maxAppliedWriteByExecuteAt, bootstrappedAt);
+ if (maxContiguousApplied >= predecessor)
+ predecessor = -1;
+ }
+
+ if (predecessor >= 0)
+ {
+ Timestamp waitingUntil = committedByExecuteAt[predecessor].plainExecuteAt();
+ if (clone) unmanageds = unmanageds.clone();
+ clone = false;
+ for (int i = start ; i < end ; ++i)
+ unmanageds[i] = new Unmanaged(APPLY, unmanageds[i].txnId, waitingUntil);
+ }
+ else
+ {
+ TxnId[] notifyNotWaiting = selectUnmanaged(unmanageds, start, end);
+ unmanageds = removeUnmanaged(unmanageds, start, end);
+ notifier = new PostProcess.NotifyNotWaiting(notifier, notifyNotWaiting);
+ clone = false;
+ }
+ start = end - 1;
}
+ ++start;
}
}
diff --git a/accord-core/src/main/java/accord/local/cfk/Pruning.java b/accord-core/src/main/java/accord/local/cfk/Pruning.java
index a4f9851a..e977c9c2 100644
--- a/accord-core/src/main/java/accord/local/cfk/Pruning.java
+++ b/accord-core/src/main/java/accord/local/cfk/Pruning.java
@@ -189,7 +189,7 @@ public class Pruning
while (--i >= 0)
{
TxnInfo txn = cfk.committedByExecuteAt[i];
- if (txn.is(Write) && txn.executeAt.hlc() <= maxPruneHlc && txn.status() == APPLIED)
+ if (txn.is(Write) && txn.executeAt.hlc() <= maxPruneHlc && txn.is(APPLIED))
break;
}
@@ -264,8 +264,7 @@ public class Pruning
}
break;
- case HISTORICAL:
- case TRANSITIVELY_KNOWN:
+ case TRANSITIVE:
case PREACCEPTED_OR_ACCEPTED_INVALIDATE:
case ACCEPTED:
newByIdBuffer[pos - ++retainCount] = txn;
@@ -281,10 +280,9 @@ public class Pruning
activePruneEpoch = epoch;
}
- boolean tryPrune = activePruneEpochBefore == newPrunedBefore
- ? txn.executeAt.compareTo(newPrunedBefore.executeAt) < 0
- : activePruneEpochBefore != txn && activePruneEpochBefore != null;
-
+ boolean tryPrune = activePruneEpochBefore != null
+ && txn.executeAt.compareTo(activePruneEpochBefore.executeAt) < 0
+ && txn.compareTo(activePruneEpochBefore) < 0;
if (tryPrune)
{
TxnId[] missing = txn.missing();
@@ -311,7 +309,8 @@ public class Pruning
}
newByIdBuffer[pos - ++retainCount] = txn;
- case INVALID_OR_TRUNCATED_OR_PRUNED:
+ case INVALIDATED:
+ case TRUNCATED_OR_PRUNED:
break;
}
}
@@ -345,7 +344,7 @@ public class Pruning
for (int i = sourcePos - 1; i >= 0 ; --i)
{
TxnInfo txn = committedByExecuteAt[i];
- if (txn.status() == APPLIED)
+ if (txn.is(APPLIED))
{
long epoch = txn.executeAt.epoch();
if (epoch != activePruneEpoch && epochPrunedBefores != null)
@@ -354,10 +353,9 @@ public class Pruning
activePruneEpoch = epoch;
}
- boolean tryPrune = activePruneEpochBefore == newPrunedBefore
- ? txn.compareTo(newPrunedBefore) < 0
- : activePruneEpochBefore != txn && activePruneEpochBefore != null;
-
+ boolean tryPrune = activePruneEpochBefore != null
+ && txn.compareTo(activePruneEpochBefore) < 0
+ && txn.executeAt.compareTo(activePruneEpochBefore.executeAt) < 0;
if (tryPrune)
{
TxnId[] missing = txn.missing();
@@ -435,7 +433,7 @@ public class Pruning
{
int startPos = prevBootstrappedAt == null ? 0 : insertPos(byId, prevBootstrappedAt);
for (int i = startPos ; i < pos ; ++i)
- Invariants.checkState(byId[i].status() != COMMITTED, "%s expected to be applied or undecided, as marked redundant", byId[i]);
+ Invariants.checkState(byId[i].isNot(COMMITTED), "%s expected to be applied or undecided, as marked redundant", byId[i]);
}
newById = Arrays.copyOfRange(byId, pos, byId.length);
diff --git a/accord-core/src/main/java/accord/local/cfk/Updating.java b/accord-core/src/main/java/accord/local/cfk/Updating.java
index 079f8b45..cb28ee5f 100644
--- a/accord-core/src/main/java/accord/local/cfk/Updating.java
+++ b/accord-core/src/main/java/accord/local/cfk/Updating.java
@@ -49,8 +49,8 @@ import accord.utils.SortedCursor;
import static accord.local.KeyHistory.SYNC;
import static accord.local.cfk.CommandsForKey.InternalStatus.APPLIED;
import static accord.local.cfk.CommandsForKey.InternalStatus.COMMITTED;
-import static accord.local.cfk.CommandsForKey.InternalStatus.INVALID_OR_TRUNCATED_OR_PRUNED;
-import static accord.local.cfk.CommandsForKey.InternalStatus.TRANSITIVELY_KNOWN;
+import static accord.local.cfk.CommandsForKey.InternalStatus.INVALIDATED;
+import static accord.local.cfk.CommandsForKey.InternalStatus.TRANSITIVE;
import static accord.local.cfk.CommandsForKey.Unmanaged.Pending.APPLY;
import static accord.local.cfk.CommandsForKey.Unmanaged.Pending.COMMIT;
import static accord.local.cfk.CommandsForKey.reportLinearizabilityViolations;
@@ -64,7 +64,6 @@ import static accord.local.cfk.Utils.missingTo;
import static accord.local.cfk.Utils.removeOneMissing;
import static accord.local.cfk.Utils.removePrunedAdditions;
import static accord.local.cfk.Utils.validateMissing;
-import static accord.primitives.Txn.Kind.EphemeralRead;
import static accord.primitives.Txn.Kind.ExclusiveSyncPoint;
import static accord.primitives.Txn.Kind.Write;
import static accord.primitives.TxnId.NO_TXNIDS;
@@ -151,7 +150,7 @@ class Updating
break;
}
- if ((newMinUndecided == null || newMinUndecided.compareTo(newInfo) > 0) && newInfo.status().compareTo(COMMITTED) < 0 && cfk.mayExecute(newInfo) && !cfk.isPreBootstrap(newInfo))
+ if ((newMinUndecided == null || newMinUndecided.compareTo(newInfo) > 0) && newInfo.compareTo(COMMITTED) < 0 && cfk.mayExecute(newInfo) && !cfk.isPreBootstrap(newInfo))
newMinUndecided = newInfo;
if (newMinUndecided != null && curMinUndecided != null && curMinUndecided.compareTo(newMinUndecided) < 0)
@@ -273,7 +272,7 @@ class Updating
{
while (txnIdsIndex < depsKnownBeforePos)
{
- if (txnIdsIndex != updatePos && byId[txnIdsIndex].status().compareTo(COMMITTED) < 0)
+ if (txnIdsIndex != updatePos && byId[txnIdsIndex].compareTo(COMMITTED) < 0)
{
TxnId txnId = byId[txnIdsIndex].plainTxnId();
if ((plainTxnId.kind().witnesses(txnId)))
@@ -317,7 +316,7 @@ class Updating
System.arraycopy(byId, pos, newById, pos + 1, byId.length - pos);
if (newInfo.mayExecute())
{
- if (newInfo.status().compareTo(COMMITTED) >= 0)
+ if (newInfo.compareTo(COMMITTED) >= 0)
{
if (newMinUndecidedById < 0 || pos <= newMinUndecidedById)
{
@@ -338,24 +337,24 @@ class Updating
{
newById = byId.clone();
newById[pos] = newInfo;
- if (pos == newMinUndecidedById && curInfo.status().compareTo(COMMITTED) < 0 && newInfo.status().compareTo(COMMITTED) >= 0)
+ if (pos == newMinUndecidedById && curInfo.compareTo(COMMITTED) < 0 && newInfo.compareTo(COMMITTED) >= 0)
newMinUndecidedById = nextUndecided(newById, pos + 1, cfk);
}
if (loadingAsPrunedFor == null)
loadingAsPrunedFor = NO_TXNIDS;
- if (curInfo != null && curInfo.status().compareTo(COMMITTED) < 0 && newInfo.status().compareTo(COMMITTED) >= 0)
+ if (curInfo != null && curInfo.compareTo(COMMITTED) < 0 && newInfo.compareTo(COMMITTED) >= 0)
{
Utils.removeFromMissingArrays(newById, newCommittedByExecuteAt, plainTxnId);
}
- else if (curInfo == null && newInfo.status().compareTo(COMMITTED) < 0)
+ else if (curInfo == null && newInfo.compareTo(COMMITTED) < 0)
{
// TODO (desired): for consistency, move this to insertOrUpdate (without additions), while maintaining the efficiency
Utils.addToMissingArrays(newById, newCommittedByExecuteAt, newInfo, plainTxnId, loadingAsPrunedFor);
}
- if (testParanoia(SUPERLINEAR, NONE, LOW) && curInfo == null && newInfo.status().compareTo(COMMITTED) < 0)
+ if (testParanoia(SUPERLINEAR, NONE, LOW) && curInfo == null && newInfo.compareTo(COMMITTED) < 0)
validateMissing(newById, NO_TXNIDS, 0, curInfo, newInfo, loadingAsPrunedFor);
int newPrunedBeforeById = cfk.prunedBeforeById;
@@ -376,8 +375,8 @@ class Updating
int targetInsertPos = sourceInsertPos + additionInsertPos;
// we may need to insert or remove ourselves, depending on the new and prior status
- boolean insertSelfMissing = sourceUpdatePos < 0 && newInfo.status().compareTo(COMMITTED) < 0;
- boolean removeSelfMissing = sourceUpdatePos >= 0 && newInfo.status().compareTo(COMMITTED) >= 0 && byId[sourceUpdatePos].status().compareTo(COMMITTED) < 0;
+ boolean insertSelfMissing = sourceUpdatePos < 0 && newInfo.compareTo(COMMITTED) < 0;
+ boolean removeSelfMissing = sourceUpdatePos >= 0 && newInfo.compareTo(COMMITTED) >= 0 && byId[sourceUpdatePos].compareTo(COMMITTED) < 0;
// missingSource starts as additions, but if we insertSelfMissing at the relevant moment it becomes the merge of additions and plainTxnId
TxnId[] missingSource = additions;
@@ -461,7 +460,7 @@ class Updating
else if (c > 0)
{
TxnId txnId = additions[j++];
- newById[count] = TxnInfo.create(txnId, TRANSITIVELY_KNOWN, mayExecute(boundsInfo, txnId), txnId, Ballot.ZERO);
+ newById[count] = TxnInfo.create(txnId, TRANSITIVE, mayExecute(boundsInfo, txnId), txnId, Ballot.ZERO);
++missingCount;
}
else
@@ -478,7 +477,7 @@ class Updating
while (count < targetInsertPos)
{
TxnId txnId = additions[j++];
- newById[count++] = TxnInfo.create(txnId, TRANSITIVELY_KNOWN, mayExecute(boundsInfo, txnId), txnId, Ballot.ZERO);
+ newById[count++] = TxnInfo.create(txnId, TRANSITIVE, mayExecute(boundsInfo, txnId), txnId, Ballot.ZERO);
}
newById[targetInsertPos] = newInfo;
count = targetInsertPos + 1;
@@ -486,7 +485,7 @@ class Updating
while (j < additionCount)
{
TxnId txnId = additions[j++];
- newById[count++] = TxnInfo.create(txnId, TRANSITIVELY_KNOWN, mayExecute(boundsInfo, txnId), txnId, Ballot.ZERO);
+ newById[count++] = TxnInfo.create(txnId, TRANSITIVE, mayExecute(boundsInfo, txnId), txnId, Ballot.ZERO);
}
}
else if (count == targetInsertPos)
@@ -551,7 +550,7 @@ class Updating
else if (c > 0)
{
TxnId txnId = additions[j];
- newInfos[count] = TxnInfo.create(txnId, TRANSITIVELY_KNOWN, mayExecute(boundsInfo, txnId), txnId, Ballot.ZERO);
+ newInfos[count] = TxnInfo.create(txnId, TRANSITIVE, mayExecute(boundsInfo, txnId), txnId, Ballot.ZERO);
++j;
++missingCount;
}
@@ -565,7 +564,7 @@ class Updating
while (j < additionCount)
{
TxnId txnId = additions[j];
- newInfos[count++] = TxnInfo.create(txnId, TRANSITIVELY_KNOWN, mayExecute(boundsInfo, txnId), txnId, Ballot.ZERO);
+ newInfos[count++] = TxnInfo.create(txnId, TRANSITIVE, mayExecute(boundsInfo, txnId), txnId, Ballot.ZERO);
j++;
}
@@ -580,7 +579,7 @@ class Updating
{
return Arrays.binarySearch(committedByExecuteAt, 0, committedByExecuteAt.length, newInfo, TxnInfo::compareExecuteAt);
}
- else if (curInfo != null && curInfo.isCommittedAndExecutes() && newInfo.status() == INVALID_OR_TRUNCATED_OR_PRUNED)
+ else if (curInfo != null && curInfo.isCommittedAndExecutes() && newInfo.isAtLeast(INVALIDATED))
{
return Arrays.binarySearch(committedByExecuteAt, 0, committedByExecuteAt.length, curInfo, TxnInfo::compareExecuteAt);
}
@@ -593,7 +592,7 @@ class Updating
if (pos == Integer.MAX_VALUE)
return committedByExecuteAt;
- if (newInfo.status() != INVALID_OR_TRUNCATED_OR_PRUNED)
+ if (newInfo.compareTo(INVALIDATED) < 0)
{
if (pos >= 0 && committedByExecuteAt[pos].equals(newInfo))
{
@@ -653,7 +652,7 @@ class Updating
{
return maxAppliedWriteByExecuteAt + 1;
}
- else if (newInfo.status() == INVALID_OR_TRUNCATED_OR_PRUNED)
+ else if (newInfo.isAtLeast(INVALIDATED))
{
// deleted
int newMaxAppliedWriteByExecuteAt = maxAppliedWriteByExecuteAt - 1;
@@ -662,7 +661,7 @@ class Updating
return newMaxAppliedWriteByExecuteAt;
}
}
- else if (newInfo.status() == APPLIED || (cfk.isPreBootstrap(newInfo) && pos - 1 == maxAppliedWriteByExecuteAt))
+ else if (newInfo.is(APPLIED) || (cfk.isPreBootstrap(newInfo) && pos - 1 == maxAppliedWriteByExecuteAt))
{
return maybeAdvanceMaxAppliedAndCheckForLinearizabilityViolations(cfk, pos, newInfo.kind(), newInfo, wasPruned);
}
@@ -676,7 +675,7 @@ class Updating
TxnInfo[] committedByExecuteAt = cfk.committedByExecuteAt;
for (int i = cfk.maxAppliedWriteByExecuteAt + 1; i < appliedPos ; ++i)
{
- if (committedByExecuteAt[i].status() != APPLIED
+ if (committedByExecuteAt[i].isNot(APPLIED)
&& appliedKind.witnesses(committedByExecuteAt[i])
&& reportLinearizabilityViolations())
logger.error("Linearizability violation on key {}: {} is committed to execute (at {}) before {} that should witness it but has already applied (at {})", cfk.key, committedByExecuteAt[i].plainTxnId(), committedByExecuteAt[i].plainExecuteAt(), applied.plainTxnId(), applied.plainExecuteAt());
@@ -693,7 +692,7 @@ class Updating
if (pos == infos.length)
return -1;
- if (infos[pos].status().compareTo(COMMITTED) < 0 && cfk.mayExecute(infos[pos]))
+ if (infos[pos].compareTo(COMMITTED) < 0 && cfk.mayExecute(infos[pos]))
return pos;
++pos;
@@ -760,6 +759,7 @@ class Updating
Txn.Kind waitingKind = waitingTxnId.kind();
boolean readyToApply = true; // our dependencies have applied, so we are ready to apply
boolean waitingToApply = true; // our dependencies have committed, so we know when we execute and are waiting
+ boolean hasFutureDependency = false;
Timestamp executesAt = null;
int j = SortedArrays.binarySearch(byId, 0, byId.length, txnIds.get(i), Timestamp::compareTo, FAST);
@@ -772,46 +772,30 @@ class Updating
TxnInfo txn = byId[j];
if (txn.mayExecute())
{
- if (txn.status().compareTo(COMMITTED) < 0) waitingToApply = readyToApply = false;
- else if (txn.status() != INVALID_OR_TRUNCATED_OR_PRUNED
- && (txn.executeAt.compareTo(waitingExecuteAt) < 0
- || waitingKind == EphemeralRead
- || (waitingKind == ExclusiveSyncPoint && txn.compareTo(waitingTxnId) < 0)
- ))
+ if (txn.compareTo(COMMITTED) < 0) waitingToApply = readyToApply = false;
+ else if (txn.compareTo(INVALIDATED) < 0
+ && (txn.executeAt.compareTo(waitingExecuteAt) < 0 || waitingKind.awaitsOnlyDeps))
{
- readyToApply &= txn.is(APPLIED);
- executesAt = Timestamp.nonNullOrMax(executesAt, txn.executeAt);
- }
- }
- ++i;
- ++j;
- }
- else if (c > 0)
- {
- if (waitingKind.isSyncPoint())
- {
- // we special-case sync points to handle pruning of transactions they should have witnessed as dependencies.
- // We require all earlier TxnId to be decided before we can execute; we must be informed of all the necessary TxnId
- // transitively by the "future" dependency provided for this purpose in mapReduceActive, so simply ensuring
- // all earlier TxnId are committed is sufficient to compute the correct executeAt.
- TxnInfo txn = byId[j];
- if (txn.mayExecute())
- {
- if (txn.status().compareTo(COMMITTED) < 0) readyToApply = waitingToApply = false;
- else if (txn.status() != INVALID_OR_TRUNCATED_OR_PRUNED
- && cfk.executes(txn.executeAt)
- && (txn.executeAt.compareTo(waitingExecuteAt) < 0
- || waitingKind == EphemeralRead
- || (waitingKind == ExclusiveSyncPoint && txn.compareTo(waitingTxnId) < 0)
- ))
+ if (waitingKind == ExclusiveSyncPoint && txn.compareTo(waitingTxnId) > 0)
+ {
+ // we don't wait for this txn itself to apply, but we do require it to be transitively committed
+ // (i.e., not just it must be committed, but all of its transitive dependencies;
+ // and then for the highest committed transaction < waitingTxnId to apply
+ if (cfk.minUndecidedById >= 0 && cfk.minUndecidedById <= j)
+ waitingToApply = readyToApply = false;
+ hasFutureDependency = true;
+ }
+ else
{
readyToApply &= txn.is(APPLIED);
executesAt = Timestamp.nonNullOrMax(executesAt, txn.executeAt);
}
}
}
+ ++i;
++j;
}
+ else if (c > 0) ++j;
else if (!cfk.mayExecute(txnIds.get(i))) ++i;
else if (register)
{
@@ -826,8 +810,26 @@ class Updating
}
}
- if (waitingKind.isSyncPoint() && Pruning.isAnyPredecessorWaitingOnPruned(cfk.loadingPruned, waitingTxnId))
- readyToApply = waitingToApply = false;
+ if (waitingKind.isSyncPoint())
+ {
+ if (Pruning.isAnyPredecessorWaitingOnPruned(cfk.loadingPruned, waitingTxnId))
+ {
+ readyToApply = waitingToApply = false;
+ }
+ else if (waitingKind == ExclusiveSyncPoint && hasFutureDependency && waitingToApply)
+ {
+ int w = Arrays.binarySearch(byId, waitingTxnId);
+ if (w < 0) w = -1 - w;
+ while (--w >= 0)
+ {
+ TxnInfo txn = byId[w];
+ if (!txn.mayExecute()) continue;
+ readyToApply &= txn.isAtLeast(APPLIED);
+ if (txn.isCommittedToExecute())
+ executesAt = Timestamp.nonNullOrMax(executesAt, byId[w].executeAt);
+ }
+ }
+ }
if (!readyToApply)
{
diff --git a/accord-core/src/main/java/accord/local/cfk/Utils.java b/accord-core/src/main/java/accord/local/cfk/Utils.java
index 484d6e7a..c4301a45 100644
--- a/accord-core/src/main/java/accord/local/cfk/Utils.java
+++ b/accord-core/src/main/java/accord/local/cfk/Utils.java
@@ -56,7 +56,7 @@ class Utils
if (shouldNotHaveMissing != NO_TXNIDS && Arrays.binarySearch(shouldNotHaveMissing, txn) >= 0) Invariants.checkState(j < 0);
else Invariants.checkState(j >= 0);
}
- if (curInfo == null && newInfo.status().compareTo(COMMITTED) < 0 && txn.kind().witnesses(newInfo) && txn.depsKnownBefore().compareTo(newInfo) > 0 && (shouldNotHaveMissing == NO_TXNIDS || Arrays.binarySearch(shouldNotHaveMissing, txn) < 0))
+ if (curInfo == null && newInfo.compareTo(COMMITTED) < 0 && txn.kind().witnesses(newInfo) && txn.depsKnownBefore().compareTo(newInfo) > 0 && (shouldNotHaveMissing == NO_TXNIDS || Arrays.binarySearch(shouldNotHaveMissing, txn) < 0))
Invariants.checkState(Arrays.binarySearch(missing, newInfo) >= 0);
}
}
@@ -107,7 +107,7 @@ class Utils
if (txn.getClass() == TxnInfo.class) continue;
if (!txn.hasDeps()) continue;
if (!txn.kind().witnesses(removeTxnId)) continue;
- if (txn.status() != ACCEPTED && txn.mayExecute()) continue;
+ if (txn.isNot(ACCEPTED) && txn.mayExecute()) continue;
TxnId[] missing = txn.missing();
TxnId[] newMissing = removeOneMissing(missing, removeTxnId);
@@ -188,7 +188,7 @@ class Utils
if (txn == newInfo) continue;
if (!txn.hasDeps()) continue;
if (!txn.kind().witnesses(insertTxnId)) continue;
- if (txn.status() != ACCEPTED && txn.mayExecute()) continue;
+ if (txn.isNot(ACCEPTED) && txn.mayExecute()) continue;
if (minDoNotInsertSearchIndex < doNotInsert.length && doNotInsert[minDoNotInsertSearchIndex].equals(txn))
{
++minDoNotInsertSearchIndex;
@@ -209,7 +209,7 @@ class Utils
// TODO (expected): optimise this with flag bits
if (!txn.hasDeps()) continue;
if (!txn.kind().witnesses(insertTxnId)) continue;
- if (txn.status() != ACCEPTED && txn.mayExecute()) continue;
+ if (txn.isNot(ACCEPTED) && txn.mayExecute()) continue;
if (minDoNotInsertSearchIndex < doNotInsert.length && doNotInsert[minDoNotInsertSearchIndex].equals(txn))
{
++minDoNotInsertSearchIndex;
diff --git a/accord-core/src/main/java/accord/messages/Apply.java b/accord-core/src/main/java/accord/messages/Apply.java
index 862e2658..a8c6e6c5 100644
--- a/accord-core/src/main/java/accord/messages/Apply.java
+++ b/accord-core/src/main/java/accord/messages/Apply.java
@@ -57,12 +57,12 @@ public class Apply extends TxnRequest<ApplyReply>
public interface Factory
{
- Apply create(Kind kind, Id to, Topologies participates, TxnId txnId, FullRoute<?> route, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result);
+ Apply create(Kind kind, Id to, Topologies participates, TxnId txnId, Route<?> scope, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, FullRoute<?> fullRoute);
}
public static Factory wrapForExclusiveSyncPoint(Factory factory)
{
- return (kind, to, participates, txnId, route, txn, executeAt, deps, writes, result) -> {
+ return (kind, to, participates, txnId, sendTo, txn, executeAt, deps, writes, result, fullRoute) -> {
while (true)
{
long minEpoch = participates.oldestEpoch();
@@ -81,7 +81,7 @@ public class Apply extends TxnRequest<ApplyReply>
break;
}
- return factory.create(kind, to, participates, txnId, route, txn, executeAt, deps, writes, result);
+ return factory.create(kind, to, participates, txnId, sendTo, txn, executeAt, deps, writes, result, fullRoute);
};
}
@@ -95,26 +95,32 @@ public class Apply extends TxnRequest<ApplyReply>
public enum Kind { Minimal, Maximal }
- protected Apply(Kind kind, Id to, Topologies participates, TxnId txnId, FullRoute<?> route, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result)
+ protected Apply(Kind kind, Id to, Topologies participates, TxnId txnId, Route<?> sendTo, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, FullRoute<?> fullRoute)
{
- super(to, participates, route, txnId);
+ super(to, participates, sendTo, txnId);
Invariants.checkState(txnId.kind() != Txn.Kind.Write || writes != null);
// TODO (desired): it's wasteful to encode the full set of ranges owned by the recipient node;
// often it will be cheaper to include the FullRoute for Deps scope (or come up with some other safety-preserving encoding scheme)
this.kind = kind;
this.deps = deps.intersecting(scope);
this.txn = kind == Kind.Maximal ? txn.intersecting(scope, true) : null;
- this.fullRoute = kind == Kind.Maximal ? route : null;
+ this.fullRoute = kind == Kind.Maximal ? fullRoute : null;
this.executeAt = executeAt;
this.writes = writes;
this.result = result;
}
- public static void sendMaximal(Node node, Id to, TxnId txnId, FullRoute<?> route, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result)
+ public static void sendMaximal(Node node, Id to, TxnId txnId, Route<?> sendTo, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, FullRoute<?> fullRoute)
{
- Topologies executes = executes(node, route, executeAt);
- Topologies participates = participates(node, route, txnId, executeAt, executes);
- node.send(to, applyMaximal(FACTORY, to, participates, txnId, route, txn, executeAt, deps, writes, result));
+ Topologies executes = executes(node, sendTo, executeAt);
+ sendMaximal(node, to, executes, txnId, sendTo, txn, executeAt, deps, writes, result, fullRoute);
+ }
+
+ public static void sendMaximal(Node node, Id to, Topologies executes, TxnId txnId, Route<?> sendTo, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, FullRoute<?> fullRoute)
+ {
+ Topologies participates = participates(node, sendTo, txnId, executeAt, executes);
+ // TODO (expected): should this FACTORY be user configurable? Perhaps via CoordinationAdapter?
+ node.send(to, applyMaximal(FACTORY, to, participates, txnId, sendTo, txn, executeAt, deps, writes, result, fullRoute));
}
public static Topologies executes(Node node, Unseekables<?> route, Timestamp executeAt)
@@ -127,9 +133,9 @@ public class Apply extends TxnRequest<ApplyReply>
return txnId.epoch() == executeAt.epoch() ? executes : node.topology().preciseEpochs(route, txnId.epoch(), executeAt.epoch());
}
- public static Apply applyMaximal(Factory factory, Id to, Topologies participates, TxnId txnId, FullRoute<?> route, Txn txn, Timestamp executeAt, Deps stableDeps, Writes writes, Result result)
+ public static Apply applyMaximal(Factory factory, Id to, Topologies participates, TxnId txnId, Route<?> sendTo, Txn txn, Timestamp executeAt, Deps stableDeps, Writes writes, Result result, FullRoute<?> fullRoute)
{
- return factory.create(Kind.Maximal, to, participates, txnId, route, txn, executeAt, stableDeps, writes, result);
+ return factory.create(Kind.Maximal, to, participates, txnId, sendTo, txn, executeAt, stableDeps, writes, result, fullRoute);
}
protected Apply(Kind kind, TxnId txnId, Route<?> route, long waitForEpoch, Timestamp executeAt, PartialDeps deps, @Nullable PartialTxn txn, @Nullable FullRoute<?> fullRoute, Writes writes, Result result)
diff --git a/accord-core/src/main/java/accord/messages/ApplyThenWaitUntilApplied.java b/accord-core/src/main/java/accord/messages/ApplyThenWaitUntilApplied.java
index e71660c7..34b2dac8 100644
--- a/accord-core/src/main/java/accord/messages/ApplyThenWaitUntilApplied.java
+++ b/accord-core/src/main/java/accord/messages/ApplyThenWaitUntilApplied.java
@@ -54,9 +54,9 @@ public class ApplyThenWaitUntilApplied extends WaitUntilApplied
@SuppressWarnings("unused")
public static class SerializerSupport
{
- public static ApplyThenWaitUntilApplied create(TxnId txnId, Participants<?> readScope, Timestamp executeAt, FullRoute<?> route, PartialTxn txn, PartialDeps deps, Writes writes, Result result)
+ public static ApplyThenWaitUntilApplied create(TxnId txnId, Participants<?> readScope, long minEpoch, Timestamp executeAt, FullRoute<?> route, PartialTxn txn, PartialDeps deps, Writes writes, Result result)
{
- return new ApplyThenWaitUntilApplied(txnId, readScope, executeAt, route, txn, deps, writes, result);
+ return new ApplyThenWaitUntilApplied(txnId, readScope, minEpoch, executeAt, route, txn, deps, writes, result);
}
}
@@ -79,9 +79,9 @@ public class ApplyThenWaitUntilApplied extends WaitUntilApplied
this.result = result;
}
- protected ApplyThenWaitUntilApplied(TxnId txnId, Participants<?> readScope, Timestamp executeAt, FullRoute<?> route, PartialTxn txn, PartialDeps deps, Writes writes, Result result)
+ protected ApplyThenWaitUntilApplied(TxnId txnId, Participants<?> readScope, long minEpoch, Timestamp executeAt, FullRoute<?> route, PartialTxn txn, PartialDeps deps, Writes writes, Result result)
{
- super(txnId, readScope, executeAt.epoch());
+ super(txnId, readScope, minEpoch, executeAt.epoch());
this.executeAt = executeAt;
this.route = route;
this.txn = txn;
diff --git a/accord-core/src/main/java/accord/messages/Await.java b/accord-core/src/main/java/accord/messages/Await.java
index 53a89a0e..450db36a 100644
--- a/accord-core/src/main/java/accord/messages/Await.java
+++ b/accord-core/src/main/java/accord/messages/Await.java
@@ -44,7 +44,6 @@ import accord.primitives.Participants;
import accord.primitives.Route;
import accord.primitives.TxnId;
import accord.topology.Topologies;
-import accord.topology.Topology;
import accord.utils.Invariants;
import accord.utils.MapReduceConsume;
@@ -63,16 +62,16 @@ public class Await implements Request, MapReduceConsume<SafeCommandStore, Void>,
{
public static class SerializerSupport
{
- public static Await create(TxnId txnId, Participants<?> scope, BlockedUntil blockedUntil, long awaitEpoch, int callbackId)
+ public static Await create(TxnId txnId, Participants<?> scope, BlockedUntil blockedUntil, long minAwaitEpoch, long maxAwaitEpoch, int callbackId)
{
- return new Await(txnId, scope, blockedUntil, awaitEpoch, callbackId);
+ return new Await(txnId, scope, blockedUntil, minAwaitEpoch, maxAwaitEpoch, callbackId);
}
}
public final TxnId txnId;
public final Participants<?> scope;
public final BlockedUntil blockedUntil;
- public final long awaitEpoch;
+ public final long minAwaitEpoch, maxAwaitEpoch;
public final int callbackId; // < 0 means synchronous await
private transient Node node;
@@ -94,34 +93,26 @@ public class Await implements Request, MapReduceConsume<SafeCommandStore, Void>,
public Await(Id to, Topologies topologies, TxnId txnId, Participants<?> participants, BlockedUntil blockedUntil, int callbackId)
{
- Invariants.checkArgument(topologies.size() == 1);
this.txnId = txnId;
this.callbackId = callbackId;
this.scope = computeScope(to, topologies, participants);
this.blockedUntil = blockedUntil;
- this.awaitEpoch = topologies.currentEpoch();
+ this.maxAwaitEpoch = topologies.currentEpoch();
+ this.minAwaitEpoch = topologies.oldestEpoch();
}
- public Await(Id to, Topology topology, TxnId txnId, Participants<?> participants, BlockedUntil blockedUntil)
+ public Await(Id to, Topologies topologies, TxnId txnId, Participants<?> participants, BlockedUntil blockedUntil)
{
- this(to, topology, txnId, participants, blockedUntil, SYNCHRONOUS_CALLBACKID);
+ this(to, topologies, txnId, participants, blockedUntil, SYNCHRONOUS_CALLBACKID);
}
- public Await(Id to, Topology topology, TxnId txnId, Participants<?> participants, BlockedUntil blockedUntil, int callbackId)
- {
- this.txnId = txnId;
- this.scope = participants.slice(topology.rangesForNode(to));
- this.blockedUntil = blockedUntil;
- this.awaitEpoch = topology.epoch();
- this.callbackId = callbackId;
- }
-
- private Await(TxnId txnId, Participants<?> scope, BlockedUntil blockedUntil, long awaitEpoch, int callbackId)
+ private Await(TxnId txnId, Participants<?> scope, BlockedUntil blockedUntil, long minAwaitEpoch, long maxAwaitEpoch, int callbackId)
{
this.txnId = txnId;
this.scope = scope;
this.blockedUntil = blockedUntil;
- this.awaitEpoch = awaitEpoch;
+ this.minAwaitEpoch = minAwaitEpoch;
+ this.maxAwaitEpoch = maxAwaitEpoch;
this.callbackId = callbackId;
}
@@ -131,16 +122,16 @@ public class Await implements Request, MapReduceConsume<SafeCommandStore, Void>,
this.node = node;
this.replyTo = replyToNode;
this.replyContext = replyContext;
- node.mapReduceConsumeLocal(this, scope, awaitEpoch, awaitEpoch, this);
+ node.mapReduceConsumeLocal(this, scope, minAwaitEpoch, maxAwaitEpoch, this);
}
@Override
public Void apply(SafeCommandStore safeStore)
{
- StoreParticipants participants = StoreParticipants.execute(safeStore, scope, txnId, awaitEpoch);
+ StoreParticipants participants = StoreParticipants.execute(safeStore, scope, txnId, minAwaitEpoch, maxAwaitEpoch);
SafeCommand safeCommand = safeStore.get(txnId, participants);
Command command = safeCommand.current();
- Invariants.checkState(awaitEpoch == txnId.epoch() || !command.known().executeAt.isDecidedAndKnownToExecute() || awaitEpoch == command.executeAt().epoch());
+ Invariants.checkState(minAwaitEpoch >= txnId.epoch());
if (command.saveStatus().compareTo(blockedUntil.minSaveStatus) >= 0)
return null;
@@ -165,7 +156,7 @@ public class Await implements Request, MapReduceConsume<SafeCommandStore, Void>,
synchronouslyWaitingOnUpdater.incrementAndGet(this);
}
- safeStore.progressLog().waiting(blockedUntil, safeStore, safeCommand, null, participants.owns());
+ safeStore.progressLog().waiting(blockedUntil, safeStore, safeCommand, null, null, participants);
return null;
}
diff --git a/accord-core/src/main/java/accord/messages/BeginRecovery.java b/accord-core/src/main/java/accord/messages/BeginRecovery.java
index 8d4b483d..4a925a83 100644
--- a/accord-core/src/main/java/accord/messages/BeginRecovery.java
+++ b/accord-core/src/main/java/accord/messages/BeginRecovery.java
@@ -41,18 +41,18 @@ import accord.topology.Topologies;
import accord.utils.Invariants;
import accord.utils.async.Cancellable;
-import static accord.local.SafeCommandStore.TestDep.WITH;
import static accord.local.SafeCommandStore.TestDep.WITHOUT;
+import static accord.local.SafeCommandStore.TestDep.WITH_OR_INVALIDATED;
import static accord.local.SafeCommandStore.TestStartedAt.ANY;
-import static accord.local.SafeCommandStore.TestStatus.IS_STABLE;
-import static accord.local.SafeCommandStore.TestStatus.IS_PROPOSED;
import static accord.local.SafeCommandStore.TestStartedAt.STARTED_AFTER;
import static accord.local.SafeCommandStore.TestStartedAt.STARTED_BEFORE;
+import static accord.local.SafeCommandStore.TestStatus.IS_PROPOSED;
+import static accord.local.SafeCommandStore.TestStatus.IS_STABLE;
+import static accord.messages.PreAccept.calculateDeps;
+import static accord.primitives.EpochSupplier.constant;
import static accord.primitives.Status.Accepted;
import static accord.primitives.Status.Phase;
import static accord.primitives.Status.PreAccepted;
-import static accord.messages.PreAccept.calculateDeps;
-import static accord.primitives.EpochSupplier.constant;
import static accord.utils.Invariants.illegalState;
public class BeginRecovery extends TxnRequest.WithUnsynced<BeginRecovery.RecoverReply>
@@ -144,7 +144,7 @@ public class BeginRecovery extends TxnRequest.WithUnsynced<BeginRecovery.Recover
// TODO (expected, testing): introduce some good unit tests for verifying these two functions in a real repair scenario
// committed txns with an earlier txnid and have our txnid as a dependency
- earlierCommittedWitness = stableStartedBeforeAndWitnessed(safeStore, txnId, participants);
+ earlierCommittedWitness = stableStartedBeforeAndWitnessedOrInvalidated(safeStore, txnId, participants);
// accepted txns with an earlier txnid that don't have our txnid as a dependency
earlierAcceptedNoWitness = acceptedOrCommittedStartedBeforeWithoutWitnessing(safeStore, txnId, participants);
@@ -345,11 +345,11 @@ public class BeginRecovery extends TxnRequest.WithUnsynced<BeginRecovery.Recover
}
}
- private static Deps stableStartedBeforeAndWitnessed(SafeCommandStore safeStore, TxnId startedBefore, StoreParticipants participants)
+ private static Deps stableStartedBeforeAndWitnessedOrInvalidated(SafeCommandStore safeStore, TxnId startedBefore, StoreParticipants participants)
{
try (Deps.Builder builder = Deps.builder())
{
- safeStore.mapReduceFull(participants.owns(), startedBefore, startedBefore.witnessedBy(), STARTED_BEFORE, WITH, IS_STABLE,
+ safeStore.mapReduceFull(participants.owns(), startedBefore, startedBefore.witnessedBy(), STARTED_BEFORE, WITH_OR_INVALIDATED, IS_STABLE,
(p1, keyOrRange, txnId, executeAt, prev) -> builder.add(keyOrRange, txnId), null, (Deps.AbstractBuilder<Deps>)builder);
return builder.build();
}
diff --git a/accord-core/src/main/java/accord/messages/CheckStatus.java b/accord-core/src/main/java/accord/messages/CheckStatus.java
index 24a32e0d..59be6dde 100644
--- a/accord-core/src/main/java/accord/messages/CheckStatus.java
+++ b/accord-core/src/main/java/accord/messages/CheckStatus.java
@@ -288,7 +288,7 @@ public class CheckStatus extends AbstractRequest<CheckStatus.CheckStatusReply>
return null;
}
- public CheckStatusOk finish(Unseekables<?> routeOrParticipants, WithQuorum withQuorum)
+ public CheckStatusOk finish(Unseekables<?> queried, Unseekables<?> routeOrParticipants, WithQuorum withQuorum)
{
CheckStatusOk finished = this;
if (withQuorum == HasQuorum)
@@ -310,7 +310,7 @@ public class CheckStatus extends AbstractRequest<CheckStatus.CheckStatusReply>
}
Known validForAll = map.computeValidForAll(routeOrParticipants);
- if (withQuorum == HasQuorum && invalidIf.inferInvalidWithQuorum(finished.maxKnown()))
+ if (withQuorum == HasQuorum && invalidIf.inferInvalidWithQuorum(finished.minKnown(queried)))
validForAll = validForAll.atLeast(Known.Invalidated);
return finished.with(map.with(validForAll));
@@ -447,6 +447,11 @@ public class CheckStatus extends AbstractRequest<CheckStatus.CheckStatusReply>
return map.foldl(Known::atLeast, Known.Nothing, i -> false);
}
+ public Known minKnown(Unseekables<?> query)
+ {
+ return map.foldlWithDefault(query, Known::nonNullOrMin, Known.Nothing, null);
+ }
+
@Override
public MessageType type()
{
@@ -482,9 +487,9 @@ public class CheckStatus extends AbstractRequest<CheckStatus.CheckStatusReply>
this.result = result;
}
- public CheckStatusOkFull finish(Unseekables<?> unseekables, WithQuorum withQuorum)
+ public CheckStatusOkFull finish(Unseekables<?> queried, Unseekables<?> routeOrParticipants, WithQuorum withQuorum)
{
- return (CheckStatusOkFull) super.finish(unseekables, withQuorum);
+ return (CheckStatusOkFull) super.finish(queried, routeOrParticipants, withQuorum);
}
public CheckStatusOkFull merge(@Nonnull Route<?> route)
diff --git a/accord-core/src/main/java/accord/messages/PreAccept.java b/accord-core/src/main/java/accord/messages/PreAccept.java
index 8a5fbf59..0b7a0efb 100644
--- a/accord-core/src/main/java/accord/messages/PreAccept.java
+++ b/accord-core/src/main/java/accord/messages/PreAccept.java
@@ -127,6 +127,9 @@ public class PreAccept extends WithUnsynced<PreAccept.PreAcceptReply>
if (command.saveStatus().compareTo(SaveStatus.PreAccepted) > 0)
return PreAcceptNack.INSTANCE;
+ if (command.executeAt().isRejected())
+ return new PreAcceptOk(txnId, command.executeAt(), Deps.NONE);
+
Deps deps = calculateDeps(safeStore, txnId, participants, EpochSupplier.constant(minEpoch), txnId);
// NOTE: we CANNOT test whether we adopt a future dependency here because it might be that this command
// is guaranteed to not reach agreement, but that this replica is unaware of that fact and has pruned
diff --git a/accord-core/src/main/java/accord/messages/Propagate.java b/accord-core/src/main/java/accord/messages/Propagate.java
index f872d08b..928a97dd 100644
--- a/accord-core/src/main/java/accord/messages/Propagate.java
+++ b/accord-core/src/main/java/accord/messages/Propagate.java
@@ -55,7 +55,6 @@ import static accord.coordinate.Infer.InvalidIf.NotKnownToBeInvalid;
import static accord.local.Cleanup.ERASE;
import static accord.local.Cleanup.VESTIGIAL;
import static accord.local.Commands.purge;
-import static accord.local.RedundantStatus.LOCALLY_REDUNDANT;
import static accord.primitives.SaveStatus.Stable;
import static accord.primitives.Status.NotDefined;
import static accord.primitives.Status.PreApplied;
@@ -136,7 +135,7 @@ public class Propagate implements PreLoadContext, MapReduceConsume<SafeCommandSt
propagate(node, txnId, sourceEpoch, txnId.epoch(), sourceEpoch, withQuorum, route, propagateTo, target, full, callback);
}
- public static void propagate(Node node, TxnId txnId, long sourceEpoch, long lowEpoch, long highEpoch, WithQuorum withQuorum, Route<?> route, Unseekables<?> propagateTo, @Nullable Known target, CheckStatusOkFull full, BiConsumer<? super FetchResult, Throwable> callback)
+ public static void propagate(Node node, TxnId txnId, long sourceEpoch, long lowEpoch, long highEpoch, WithQuorum withQuorum, Route<?> queried, Unseekables<?> propagateTo, @Nullable Known target, CheckStatusOkFull full, BiConsumer<? super FetchResult, Throwable> callback)
{
if (full.maxKnowledgeSaveStatus.status == NotDefined && full.invalidIf == NotKnownToBeInvalid)
{
@@ -146,9 +145,10 @@ public class Propagate implements PreLoadContext, MapReduceConsume<SafeCommandSt
Invariants.checkState(sourceEpoch == txnId.epoch() || (full.executeAt != null && sourceEpoch == full.executeAt.epoch()) || full.maxKnowledgeSaveStatus == SaveStatus.Erased || full.maxKnowledgeSaveStatus == SaveStatus.ErasedOrVestigial);
- route = route.with((Unseekables) propagateTo);
- full = full.finish(route, withQuorum);
- route = Invariants.nonNull(full.route);
+ // TODO (required): consider and document whether it is safe to infer that we are stale if we have not received responses from all shards we know of
+ // (in principle, we should at least require responses from our own shard, and the home shard if we know it); if we only hear from a remote shard it may have fully Erased
+ full = full.finish(queried, queried.with((Unseekables) propagateTo), withQuorum);
+ Route<?> route = Invariants.nonNull(full.route);
Propagate propagate =
new Propagate(node, txnId, route, propagateTo, target, full.maxKnowledgeSaveStatus, full.maxSaveStatus, full.acceptedOrCommitted, full.durability, full.homeKey, full.map, full.partialTxn, full.stableDeps, lowEpoch, highEpoch, full.executeAtIfKnown(), full.writes, full.result, callback);
@@ -157,7 +157,7 @@ public class Propagate implements PreLoadContext, MapReduceConsume<SafeCommandSt
highEpoch = full.executeAt.epoch();
long untilEpoch = full.executeAt == null ? highEpoch : Math.max(highEpoch, full.executeAt.epoch());
- Route<?> finalRoute = route;
+ Route<?> finalRoute = queried;
node.withEpoch(highEpoch, propagate, () -> node.mapReduceConsumeLocal(propagate, finalRoute, lowEpoch, untilEpoch, propagate));
}
@@ -341,34 +341,26 @@ public class Propagate implements PreLoadContext, MapReduceConsume<SafeCommandSt
return known.knownForAny();
RedundantStatus status = safeStore.redundantBefore().status(txnId, participants.owns());
+ // try to see if we can safely purge the full command
+ if (tryPurge(safeStore, safeCommand, status))
+ return null;
// if our peers have truncated this command, then either:
// 1) we have already applied it locally; 2) the command doesn't apply locally; 3) we are stale; or 4) the command is invalidated
if (executeAtIfKnown == null)
{
- if (status.compareTo(LOCALLY_REDUNDANT) >= 0)
- {
- purge(safeStore, safeCommand, null, ERASE, true);
- return null;
- }
-
Ranges ranges = safeStore.ranges().allSince(txnId.epoch());
ranges = safeStore.redundantBefore().everExpectToExecute(txnId, ranges);
if (!ranges.isEmpty())
{
- // even though command stores only lose ranges, we still adopt ranges as of some epoch, and re-bootstrap.
- // however, this eventuality should be exceptionally rare
// we don't even know the execution time, so we cannot possibly proceed besides erasing the command state and marking ourselves stale
- // TODO (required): do not setErased until markShardStale is complete
+ // (even though command stores only lose ranges, we still adopt ranges as of some epoch, and re-bootstrap.)
safeStore.commandStore().markShardStale(safeStore, txnId, route.slice(ranges, Minimal).toRanges(), false);
}
Commands.setErased(safeStore, safeCommand);
return null;
}
- if (tryPurge(safeStore, safeCommand, status))
- return null;
-
Participants<?> executes = participants.executes(safeStore, txnId, executeAtIfKnown);
status = safeStore.redundantBefore().status(txnId, executes);
if (tryPurge(safeStore, safeCommand, status))
@@ -417,17 +409,21 @@ public class Propagate implements PreLoadContext, MapReduceConsume<SafeCommandSt
{
default: throw new AssertionError("Unhandled RedundantStatus: " + status);
case NOT_OWNED:
- case WAS_OWNED:
+ case WAS_OWNED_RETIRED:
+ case GC_BEFORE:
case SHARD_REDUNDANT:
case LOCALLY_REDUNDANT:
+ case REDUNDANT_PRE_BOOTSTRAP_OR_STALE:
+ Invariants.checkState(!known.knownForAny().outcome.isOrWasApply());
purge(safeStore, safeCommand, null, ERASE, true);
return true;
case PRE_BOOTSTRAP_OR_STALE:
- case REDUNDANT_PRE_BOOTSTRAP_OR_STALE:
purge(safeStore, safeCommand, null, VESTIGIAL, true);
return true;
case PARTIALLY_PRE_BOOTSTRAP_OR_STALE:
case LIVE:
+ case WAS_OWNED:
+ case WAS_OWNED_CLOSED:
return false;
}
}
diff --git a/accord-core/src/main/java/accord/messages/ReadData.java b/accord-core/src/main/java/accord/messages/ReadData.java
index 5e1d4635..cb5ae063 100644
--- a/accord-core/src/main/java/accord/messages/ReadData.java
+++ b/accord-core/src/main/java/accord/messages/ReadData.java
@@ -63,7 +63,7 @@ public abstract class ReadData implements PreLoadContext, Request, MapReduceCons
{
private static final Logger logger = LoggerFactory.getLogger(ReadData.class);
- private enum State { PENDING, RETURNED, OBSOLETE }
+ private enum State { PENDING, PENDING_OBSOLETE, RETURNED, OBSOLETE }
protected enum StoreAction { WAIT, EXECUTE, OBSOLETE }
public enum ReadType
@@ -157,6 +157,11 @@ public abstract class ReadData implements PreLoadContext, Request, MapReduceCons
return txnId;
}
+ protected long minEpoch()
+ {
+ return executeAtEpoch;
+ }
+
@Override
public final void process(Node on, Node.Id replyTo, ReplyContext replyContext)
{
@@ -165,7 +170,7 @@ public abstract class ReadData implements PreLoadContext, Request, MapReduceCons
this.replyContext = replyContext;
waitingOn = new IntHashSet();
reading = new IntHashSet();
- Cancellable cancel = node.mapReduceConsumeLocal(this, readScope, executeAtEpoch, executeAtEpoch, this);
+ Cancellable cancel = node.mapReduceConsumeLocal(this, readScope, minEpoch(), executeAtEpoch, this);
long expiresAt = node.agent().expiresAt(replyContext, MICROSECONDS);
RegisteredTimeout timeout = expiresAt <= 0 ? null : node.timeouts().registerWithDelay(this, expiresAt, MICROSECONDS);
synchronized (this)
@@ -176,6 +181,7 @@ public abstract class ReadData implements PreLoadContext, Request, MapReduceCons
this.cancel = cancel;
this.timeout = timeout;
return;
+ case PENDING_OBSOLETE:
case OBSOLETE:
timeout = null;
case RETURNED:
@@ -197,7 +203,7 @@ public abstract class ReadData implements PreLoadContext, Request, MapReduceCons
@Override
public CommitOrReadNack apply(SafeCommandStore safeStore)
{
- StoreParticipants participants = StoreParticipants.execute(safeStore, readScope, txnId, executeAtEpoch);
+ StoreParticipants participants = StoreParticipants.execute(safeStore, readScope, txnId, minEpoch(), executeAtEpoch);
SafeCommand safeCommand = safeStore.get(txnId, participants);
return apply(safeStore, safeCommand, participants);
}
@@ -223,12 +229,12 @@ public abstract class ReadData implements PreLoadContext, Request, MapReduceCons
++waitingOnCount;
int c = status.compareTo(SaveStatus.Stable);
- if (c < 0) safeStore.progressLog().waiting(HasStableDeps, safeStore, safeCommand, participants.route(), participants.owns());
- else if (c > 0 && status.compareTo(executeOn().min) >= 0 && status.compareTo(SaveStatus.PreApplied) < 0) safeStore.progressLog().waiting(CanApply, safeStore, safeCommand, null, readScope);
+ if (c < 0) safeStore.progressLog().waiting(HasStableDeps, safeStore, safeCommand, null, null, participants);
+ else if (c > 0 && status.compareTo(executeOn().min) >= 0 && status.compareTo(SaveStatus.PreApplied) < 0) safeStore.progressLog().waiting(CanApply, safeStore, safeCommand, null, readScope, null);
return status.compareTo(SaveStatus.Stable) >= 0 ? null : Insufficient;
case OBSOLETE:
- state = State.OBSOLETE;
+ state = State.PENDING_OBSOLETE;
return Redundant;
case EXECUTE:
@@ -400,6 +406,7 @@ public abstract class ReadData implements PreLoadContext, Request, MapReduceCons
case RETURNED:
throw illegalState("ReadOk was sent, yet ack called again");
+ case PENDING_OBSOLETE:
case OBSOLETE:
logger.debug("Before the read completed successfully for txn {}, the result was marked obsolete", txnId);
return null;
@@ -418,7 +425,7 @@ public abstract class ReadData implements PreLoadContext, Request, MapReduceCons
Cancellable clear;
synchronized (this)
{
- if (state != State.PENDING)
+ if (state.compareTo(State.PENDING_OBSOLETE) > 0)
return false;
state = State.OBSOLETE;
diff --git a/accord-core/src/main/java/accord/messages/WaitUntilApplied.java b/accord-core/src/main/java/accord/messages/WaitUntilApplied.java
index 2d2e1d96..22906967 100644
--- a/accord-core/src/main/java/accord/messages/WaitUntilApplied.java
+++ b/accord-core/src/main/java/accord/messages/WaitUntilApplied.java
@@ -37,23 +37,32 @@ public class WaitUntilApplied extends ReadData
{
public static class SerializerSupport
{
- public static WaitUntilApplied create(TxnId txnId, Participants<?> scope, long executeAtEpoch)
+ public static WaitUntilApplied create(TxnId txnId, Participants<?> scope, long minEpoch, long executeAtEpoch)
{
- return new WaitUntilApplied(txnId, scope, executeAtEpoch);
+ return new WaitUntilApplied(txnId, scope, minEpoch, executeAtEpoch);
}
}
private static final ExecuteOn EXECUTE_ON = new ExecuteOn(Applied, TruncatedApply);
+ private final long minEpoch;
private long retryInLaterEpoch;
public WaitUntilApplied(Node.Id to, Topologies topologies, TxnId txnId, Participants<?> readScope, long executeAtEpoch)
{
super(to, topologies, txnId, readScope, executeAtEpoch);
+ this.minEpoch = topologies.oldestEpoch();
}
- protected WaitUntilApplied(TxnId txnId, Participants<?> readScope, long executeAtEpoch)
+ protected WaitUntilApplied(TxnId txnId, Participants<?> readScope, long minEpoch, long executeAtEpoch)
{
super(txnId, readScope, executeAtEpoch);
+ this.minEpoch = minEpoch;
+ }
+
+ @Override
+ public long minEpoch()
+ {
+ return minEpoch;
}
@Override
diff --git a/accord-core/src/main/java/accord/primitives/AbstractRanges.java b/accord-core/src/main/java/accord/primitives/AbstractRanges.java
index 141bded3..4812fed0 100644
--- a/accord-core/src/main/java/accord/primitives/AbstractRanges.java
+++ b/accord-core/src/main/java/accord/primitives/AbstractRanges.java
@@ -18,6 +18,13 @@
package accord.primitives;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.function.Function;
+import javax.annotation.Nonnull;
+
+import com.google.common.collect.Iterators;
+
import accord.api.RoutingKey;
import accord.utils.ArrayBuffers.ObjectBuffers;
import accord.utils.IndexedFoldToLong;
@@ -26,15 +33,9 @@ import accord.utils.Invariants;
import accord.utils.SortedArrays;
import net.nicoulaj.compilecommand.annotations.Inline;
-import com.google.common.collect.Iterators;
-
-import javax.annotation.Nonnull;
-import java.util.*;
-import java.util.function.Function;
-
import static accord.primitives.Ranges.EMPTY;
-import static accord.primitives.Ranges.ofSortedAndDeoverlappedUnchecked;
import static accord.utils.ArrayBuffers.cachedRanges;
+import static accord.utils.Invariants.illegalArgument;
import static accord.utils.SortedArrays.Search.CEIL;
import static accord.utils.SortedArrays.Search.FAST;
import static accord.utils.SortedArrays.isSorted;
@@ -181,13 +182,6 @@ public abstract class AbstractRanges implements Iterable<Range>, Routables<Range
}
}
- public Ranges without(Unseekables<?> keysOrRanges)
- {
- if (keysOrRanges.domain() == Routable.Domain.Key)
- keysOrRanges = ((AbstractUnseekableKeys)keysOrRanges).toRanges();
- return without((AbstractRanges) keysOrRanges);
- }
-
// returns ri in low 32 bits, ki in top, or -1 if no match found
@Override
public final long findNextIntersection(int ri, AbstractKeys<?> keys, int ki)
@@ -240,21 +234,13 @@ public abstract class AbstractRanges implements Iterable<Range>, Routables<Range
/**
* Subtracts the given set of ranges from this
*/
- public Ranges without(Ranges that)
- {
- return without((AbstractRanges) that);
- }
-
- /**
- * Subtracts the given set of ranges from this
- */
- private Ranges without(AbstractRanges that)
+ Range[] withoutInternal(AbstractRanges that)
{
if (that.isEmpty())
- return this instanceof Ranges ? (Ranges)this : ofSortedAndDeoverlappedUnchecked(ranges);
+ return ranges;
if (isEmpty() || that == this)
- return EMPTY;
+ return EMPTY.ranges;
ObjectBuffers<Range> cachedRanges = cachedRanges();
Range[] result = null;
@@ -310,9 +296,9 @@ public abstract class AbstractRanges implements Iterable<Range>, Routables<Range
}
if (count == 0)
- return EMPTY;
+ return EMPTY.ranges;
- return ofSortedAndDeoverlappedUnchecked(cachedRanges.completeAndDiscard(result, count));
+ return cachedRanges.completeAndDiscard(result, count);
}
/**
@@ -802,7 +788,7 @@ public abstract class AbstractRanges implements Iterable<Range>, Routables<Range
for (int i = 1 ; i < ranges.length ; ++i)
{
if (ranges[i - 1].end().compareTo(ranges[i].start()) > 0)
- throw new IllegalArgumentException(Arrays.toString(ranges) + " is not correctly sorted or deoverlapped");
+ throw illegalArgument(Arrays.toString(ranges) + " is not correctly sorted or deoverlapped");
}
return constructor.apply(ranges);
diff --git a/accord-core/src/main/java/accord/primitives/KeyRoute.java b/accord-core/src/main/java/accord/primitives/KeyRoute.java
index 7970ccad..b03b64c2 100644
--- a/accord-core/src/main/java/accord/primitives/KeyRoute.java
+++ b/accord-core/src/main/java/accord/primitives/KeyRoute.java
@@ -57,6 +57,37 @@ public abstract class KeyRoute extends AbstractUnseekableKeys implements Route<R
return new PartialKeyRoute(homeKey, mergedKeys);
}
+ @Override
+ public KeyRoute without(Unseekables<?> keysOrRanges)
+ {
+ switch (keysOrRanges.domain())
+ {
+ default: throw new AssertionError("Unhandled domain: " + keysOrRanges.domain());
+ case Key:
+ {
+ AbstractUnseekableKeys that = (AbstractUnseekableKeys) keysOrRanges;
+ RoutingKey[] newKeys = SortedArrays.linearSubtract(this.keys, that.keys, cachedRoutingKeys());
+ return newKeys == keys ? this : new PartialKeyRoute(homeKey, newKeys);
+ }
+ case Range:
+ {
+ return without((AbstractRanges)keysOrRanges);
+ }
+ }
+ }
+
+ @Override
+ public KeyRoute without(Ranges ranges)
+ {
+ return without((AbstractRanges) ranges);
+ }
+
+ private KeyRoute without(AbstractRanges ranges)
+ {
+ RoutingKey[] newKeys = subtract(ranges, keys, RoutingKey[]::new);
+ return newKeys == keys ? this : new PartialKeyRoute(homeKey, newKeys);
+ }
+
@SuppressWarnings("unchecked")
@Override
public Participants<RoutingKey> participants()
diff --git a/accord-core/src/main/java/accord/primitives/Known.java b/accord-core/src/main/java/accord/primitives/Known.java
index 4b5fc6c5..0b934fa0 100644
--- a/accord-core/src/main/java/accord/primitives/Known.java
+++ b/accord-core/src/main/java/accord/primitives/Known.java
@@ -96,6 +96,10 @@ public class Known
return selectOrCreate(that, maxRoute, maxDefinition, maxExecuteAt, maxDeps, maxOutcome);
}
+ public static Known nonNullOrMin(Known a, Known b)
+ {
+ return a == null | b == null ? a == null ? b : a : a.min(b);
+ }
public Known min(Known that)
{
Invariants.checkArgument(compatibleWith(that));
@@ -613,7 +617,7 @@ public class Known
public boolean isOrWasKnown()
{
- return this == DefinitionKnown;
+ return this != DefinitionUnknown;
}
public Definition atLeast(Definition that)
diff --git a/accord-core/src/main/java/accord/primitives/LatestDeps.java b/accord-core/src/main/java/accord/primitives/LatestDeps.java
index 313eddb3..adae0e37 100644
--- a/accord-core/src/main/java/accord/primitives/LatestDeps.java
+++ b/accord-core/src/main/java/accord/primitives/LatestDeps.java
@@ -381,7 +381,6 @@ public class LatestDeps extends ReducingRangeMap<LatestDeps.LatestEntry>
};
}
- // TODO (now): override tryMergeEqual
static class MergeBuilder extends AbstractIntervalBuilder<RoutingKey, MergeEntry, Merge>
{
protected MergeBuilder(boolean inclusiveEnds, int capacity)
diff --git a/accord-core/src/main/java/accord/primitives/RangeDeps.java b/accord-core/src/main/java/accord/primitives/RangeDeps.java
index 4f8a8811..2ea91e46 100644
--- a/accord-core/src/main/java/accord/primitives/RangeDeps.java
+++ b/accord-core/src/main/java/accord/primitives/RangeDeps.java
@@ -80,7 +80,7 @@ import static accord.utils.SortedArrays.Search.FAST;
* TODO (expected): keep only the latest exclusive sync point for a given range
* TODO (expected): permit building out-of-order
* TODO (expected): currently permitting duplicates
- * TODO (required): randomised testing of all iteration methods (just found a double increment bug)
+ * TODO (testing): randomised testing of all iteration methods
*/
public class RangeDeps implements Iterable<Map.Entry<Range, TxnId>>
{
diff --git a/accord-core/src/main/java/accord/primitives/RangeRoute.java b/accord-core/src/main/java/accord/primitives/RangeRoute.java
index 808461fc..cbc74d3e 100644
--- a/accord-core/src/main/java/accord/primitives/RangeRoute.java
+++ b/accord-core/src/main/java/accord/primitives/RangeRoute.java
@@ -19,6 +19,7 @@
package accord.primitives;
import accord.api.RoutingKey;
+import accord.primitives.Routable.Domain;
import accord.utils.Invariants;
import javax.annotation.Nonnull;
@@ -59,6 +60,24 @@ public abstract class RangeRoute extends AbstractRanges implements Route<Range>,
return with((Unseekables<Range>) Ranges.of(withKey.asRange()));
}
+ @Override
+ public RangeRoute without(Unseekables<?> keysOrRanges)
+ {
+ return without(keysOrRanges.domain() == Domain.Range ? (AbstractRanges) keysOrRanges : ((AbstractUnseekableKeys)keysOrRanges).toRanges());
+ }
+
+ @Override
+ public RangeRoute without(Ranges that)
+ {
+ return without((AbstractRanges) that);
+ }
+
+ private RangeRoute without(AbstractRanges that)
+ {
+ Range[] newRanges = withoutInternal(that);
+ return newRanges == ranges ? this : new PartialRangeRoute(homeKey, newRanges);
+ }
+
@Override
public Route<Range> withHomeKey()
{
diff --git a/accord-core/src/main/java/accord/primitives/Ranges.java b/accord-core/src/main/java/accord/primitives/Ranges.java
index ed536abb..ad99b11c 100644
--- a/accord-core/src/main/java/accord/primitives/Ranges.java
+++ b/accord-core/src/main/java/accord/primitives/Ranges.java
@@ -19,6 +19,7 @@
package accord.primitives;
import accord.api.RoutingKey;
+import accord.utils.Invariants;
import javax.annotation.Nonnull;
import java.util.ArrayList;
@@ -73,6 +74,12 @@ public class Ranges extends AbstractRanges implements Iterable<Range>, Seekables
public Ranges select(int[] indexes)
{
+ if (indexes.length == size())
+ {
+ for (int i = 0 ; i < indexes.length ; ++i)
+ Invariants.checkState(i == indexes[i]);
+ return this;
+ }
Range[] selection = new Range[indexes.length];
for (int i=0; i<indexes.length; i++)
selection[i] = ranges[indexes[i]];
@@ -175,6 +182,21 @@ public class Ranges extends AbstractRanges implements Iterable<Range>, Seekables
return with(Ranges.of(withKey.asRange()));
}
+ public Ranges without(Unseekables<?> keysOrRanges)
+ {
+ if (keysOrRanges.domain() == Routable.Domain.Key)
+ keysOrRanges = ((AbstractUnseekableKeys)keysOrRanges).toRanges();
+ return ofSortedAndDeoverlappedUnchecked(withoutInternal((AbstractRanges) keysOrRanges));
+ }
+
+ /**
+ * Subtracts the given set of ranges from this
+ */
+ public Ranges without(Ranges that)
+ {
+ return ofSortedAndDeoverlappedUnchecked(withoutInternal(that));
+ }
+
@Override
public UnseekablesKind kind()
{
diff --git a/accord-core/src/main/java/accord/primitives/Route.java b/accord-core/src/main/java/accord/primitives/Route.java
index 63d85808..fb75ad4a 100644
--- a/accord-core/src/main/java/accord/primitives/Route.java
+++ b/accord-core/src/main/java/accord/primitives/Route.java
@@ -38,6 +38,8 @@ public interface Route<K extends Unseekable> extends Participants<K>
*/
@Override
Route<K> with(Participants<K> participants);
+ @Override Route<K> without(Ranges ranges);
+ @Override Route<K> without(Unseekables<?> without);
@Override
Route<K> slice(int from, int to);
diff --git a/accord-core/src/main/java/accord/primitives/Status.java b/accord-core/src/main/java/accord/primitives/Status.java
index 7ee27701..4deee9b3 100644
--- a/accord-core/src/main/java/accord/primitives/Status.java
+++ b/accord-core/src/main/java/accord/primitives/Status.java
@@ -167,6 +167,11 @@ public enum Status
return a;
}
+ public static Durability max(Durability a, Durability b)
+ {
+ return a.compareTo(b) >= 0 ? a : b;
+ }
+
public static Durability forOrdinal(int ordinal)
{
if (ordinal < 0 || ordinal > lookup.length)
diff --git a/accord-core/src/main/java/accord/primitives/Timestamp.java b/accord-core/src/main/java/accord/primitives/Timestamp.java
index da8cb733..0a9de552 100644
--- a/accord-core/src/main/java/accord/primitives/Timestamp.java
+++ b/accord-core/src/main/java/accord/primitives/Timestamp.java
@@ -37,7 +37,7 @@ public class Timestamp implements Comparable<Timestamp>, EpochSupplier
* which we may also want to retain when merging in other contexts (such as in Deps).
*/
private static final int MERGE_FLAGS = 0x8000;
- // TODO (required): is this the correct set of identity bits?
+ // TODO (testing): is this the correct set of identity bits?
private static final long IDENTITY_LSB = 0xFFFFFFFFFFFF001EL;
public static final int IDENTITY_FLAGS = 0x001E;
public static final long MAX_EPOCH = (1L << 48) - 1;
diff --git a/accord-core/src/main/java/accord/primitives/Txn.java b/accord-core/src/main/java/accord/primitives/Txn.java
index 99e18979..3771357f 100644
--- a/accord-core/src/main/java/accord/primitives/Txn.java
+++ b/accord-core/src/main/java/accord/primitives/Txn.java
@@ -103,12 +103,7 @@ public interface Txn
// TODO (expected): introduce a special kind of visible ExclusiveSyncPoint that creates a precise moment,
// and is therefore visible to all transactions.
ExclusiveSyncPoint('X', true, true, true, true),
-
- /**
- * Used for local book-keeping only, not visible to any other replica or directly to other transactions.
- * This is used to create pseudo transactions that take the place of dependencies that will be fulfilled by a bootstrap.
- */
- LocalOnly('L', false, false, true, false);
+ ;
public enum Kinds
{
@@ -198,11 +193,6 @@ public interface Txn
return this == Read;
}
- public boolean isLocal()
- {
- return this == LocalOnly;
- }
-
public boolean isDurable()
{
return this != EphemeralRead;
@@ -256,6 +246,11 @@ public interface Txn
return 0 != (ENCODED_ORDINAL_INFO & (1L << (AWAITS_ONLY_DEPS_ORDINAL_INFO_OFFSET + ordinal)));
}
+ public static boolean awaitsPreviouslyOwned(int ordinal)
+ {
+ return ordinal == ExclusiveSyncPoint.ordinal();
+ }
+
public Kinds witnesses()
{
switch (this)
diff --git a/accord-core/src/main/java/accord/primitives/TxnId.java b/accord-core/src/main/java/accord/primitives/TxnId.java
index 3f5fa5cb..186e8252 100644
--- a/accord-core/src/main/java/accord/primitives/TxnId.java
+++ b/accord-core/src/main/java/accord/primitives/TxnId.java
@@ -127,6 +127,11 @@ public class TxnId extends Timestamp
return Kind.awaitsOnlyDeps(kindOrdinal(flags()));
}
+ public boolean awaitsPreviouslyOwned()
+ {
+ return Kind.awaitsPreviouslyOwned(kindOrdinal(flags()));
+ }
+
public boolean is(Domain domain)
{
return domainOrdinal(flags()) == domain.ordinal();
diff --git a/accord-core/src/main/java/accord/topology/Topologies.java b/accord-core/src/main/java/accord/topology/Topologies.java
index c64f7e9d..5e7e144b 100644
--- a/accord-core/src/main/java/accord/topology/Topologies.java
+++ b/accord-core/src/main/java/accord/topology/Topologies.java
@@ -23,6 +23,7 @@ import java.util.Arrays;
import accord.api.TopologySorter;
import accord.local.Node;
import accord.local.Node.Id;
+import accord.primitives.Participants;
import accord.primitives.Ranges;
import accord.utils.ArrayBuffers;
import accord.utils.ArrayBuffers.RecursiveObjectBuffers;
@@ -80,6 +81,8 @@ public interface Topologies extends TopologySorter
int maxShardsPerEpoch();
+ Topologies select(Participants<?> participants, long sinceEpoch);
+
default void forEach(IndexedConsumer<Topology> consumer)
{
for (int i=0, mi=size(); i<mi; i++)
@@ -190,6 +193,14 @@ public interface Topologies extends TopologySorter
return topology;
}
+ @Override
+ public Topologies select(Participants<?> participants, long sinceEpoch)
+ {
+ Invariants.checkState(sinceEpoch <= currentEpoch());
+ Topology subset = topology.select(participants);
+ return subset == topology ? this : new Single(sorter, subset);
+ }
+
@Override
public int size()
{
@@ -269,8 +280,8 @@ public interface Topologies extends TopologySorter
{
this.topologies = Invariants.checkArgument(topologies, isSortedUnique(topologies, (a, b) -> Long.compare(b.epoch, a.epoch)));
int maxShardsPerEpoch = 0;
- for (int i = 0 ; i < topologies.length ; ++i)
- maxShardsPerEpoch = Math.max(maxShardsPerEpoch, topologies[i].size());
+ for (Topology topology : topologies)
+ maxShardsPerEpoch = Math.max(maxShardsPerEpoch, topology.size());
this.maxShardsPerEpoch = maxShardsPerEpoch;
this.supplier = sorter;
this.nodes = nodes(topologies);
@@ -434,6 +445,31 @@ public interface Topologies extends TopologySorter
{
return sorter.isFaulty(node);
}
+
+ @Override
+ public Topologies select(Participants<?> participants, long sinceEpoch)
+ {
+ Topology[] subsets = null;
+ int limit = topologies.length;
+ if (sinceEpoch > oldestEpoch())
+ limit = indexForEpoch(sinceEpoch) + 1;
+ for (int i = 0 ; i < limit ; ++i)
+ {
+ Topology superset = topologies[i];
+ Topology subset = superset.select(participants);
+ if (subset != superset && subsets == null)
+ {
+ subsets = new Topology[limit];
+ System.arraycopy(topologies, 0, subsets, 0, i);
+ }
+ if (subsets != null)
+ subsets[i] = subset;
+ }
+
+ if (subsets == null)
+ return this;
+ return new Multi(supplier, subsets);
+ }
}
class Builder
diff --git a/accord-core/src/main/java/accord/topology/Topology.java b/accord-core/src/main/java/accord/topology/Topology.java
index a427901f..308828a4 100644
--- a/accord-core/src/main/java/accord/topology/Topology.java
+++ b/accord-core/src/main/java/accord/topology/Topology.java
@@ -269,12 +269,12 @@ public class Topology
return forSubset(EMPTY_SUBSET);
}
- public Topology forSelection(Routables<?> select)
+ public Topology select(Routables<?> select)
{
- return forSelection(select, false);
+ return select(select, false);
}
- public Topology forSelection(Routables<?> select, boolean permitMissing)
+ public Topology select(Routables<?> select, boolean permitMissing)
{
return forSubset(subsetFor(select, permitMissing));
}
@@ -283,6 +283,9 @@ public class Topology
Topology forSubset(int[] newSubset)
{
Ranges rangeSubset = ranges.select(newSubset);
+ if (rangeSubset == ranges)
+ return this;
+
SimpleBitSet nodes = new SimpleBitSet(nodeIds.size());
Int2ObjectHashMap<NodeInfo> nodeLookup = new Int2ObjectHashMap<>(nodes.size(), 0.8f);
for (int shardIndex : newSubset)
diff --git a/accord-core/src/main/java/accord/topology/TopologyManager.java b/accord-core/src/main/java/accord/topology/TopologyManager.java
index da310314..fbb2ebcb 100644
--- a/accord-core/src/main/java/accord/topology/TopologyManager.java
+++ b/accord-core/src/main/java/accord/topology/TopologyManager.java
@@ -638,7 +638,7 @@ public class TopologyManager
return new Single(sorter, epochs.get(epoch).global);
}
- // TODO (required): test all of these methods when asking for epochs that have been cleaned up (and other code paths)
+ // TODO (testing): test all of these methods when asking for epochs that have been cleaned up (and other code paths)
public Topologies withUnsyncedEpochs(Unseekables<?> select, Timestamp min, Timestamp max)
{
return withUnsyncedEpochs(select, min.epoch(), max.epoch());
@@ -721,7 +721,7 @@ public class TopologyManager
EpochState maxEpochState = nonNull(snapshot.get(maxEpoch));
if (minEpoch == maxEpoch && isSufficientFor.apply(maxEpochState).containsAll(select))
- return new Single(sorter, maxEpochState.global.forSelection(select, false));
+ return new Single(sorter, maxEpochState.global.select(select, false));
int i = (int)(snapshot.currentEpoch - maxEpoch);
int maxi = (int)(Math.min(1 + snapshot.currentEpoch - minEpoch, snapshot.epochs.length));
@@ -734,7 +734,7 @@ public class TopologyManager
while (i < maxi)
{
EpochState epochState = snapshot.epochs[i++];
- topologies.add(epochState.global.forSelection(select, false));
+ topologies.add(epochState.global.select(select, false));
select = select.without(epochState.addedRanges);
}
@@ -764,7 +764,7 @@ public class TopologyManager
return topologies.build(sorter);
EpochState next = snapshot.epochs[i++];
- topologies.add(next.global.forSelection(select, false));
+ topologies.add(next.global.select(select, false));
prev = next;
} while (i < snapshot.epochs.length);
// needd to remove sufficent / added else remaining may not be empty when the final matches are the last epoch
@@ -794,7 +794,7 @@ public class TopologyManager
{
EpochState prev = minEpoch == snapshot.minEpoch() ? null : nonNull(snapshot.get(minEpoch - 1));
if (prev == null || isSufficientFor.apply(prev, cur).containsAll(select))
- return new Single(sorter, cur.global.forSelection(select, true));
+ return new Single(sorter, cur.global.select(select, true));
}
int i = (int)(snapshot.currentEpoch - maxEpoch);
@@ -803,7 +803,7 @@ public class TopologyManager
while (!select.isEmpty())
{
- Topology topology = cur.global.forSelection(select, true);
+ Topology topology = cur.global.select(select, true);
if (!topology.isEmpty())
topologies.add(topology);
select = select.without(cur.addedRanges);
@@ -830,14 +830,14 @@ public class TopologyManager
throw tm;
if (minEpoch == maxEpoch)
- return new Single(sorter, snapshot.get(minEpoch).global.forSelection(select));
+ return new Single(sorter, snapshot.get(minEpoch).global.select(select));
int count = (int)(1 + maxEpoch - minEpoch);
Topologies.Builder topologies = new Topologies.Builder(count);
for (int i = count - 1 ; i >= 0 ; --i)
{
EpochState epochState = snapshot.get(minEpoch + i);
- topologies.add(epochState.global.forSelection(select));
+ topologies.add(epochState.global.select(select));
select = select.without(epochState.addedRanges);
}
checkState(!topologies.isEmpty(), "Unable to find an epoch that contained %s", select);
@@ -848,7 +848,7 @@ public class TopologyManager
public Topologies forEpoch(Unseekables<?> select, long epoch)
{
EpochState state = epochs.get(epoch);
- return new Single(sorter, state.global.forSelection(select));
+ return new Single(sorter, state.global.select(select));
}
public Shard forEpochIfKnown(RoutingKey key, long epoch)
@@ -906,4 +906,12 @@ public class TopologyManager
throw new IllegalArgumentException("Unknown epoch: " + epoch);
return epochState.global();
}
+
+ public Topology maybeGlobalForEpoch(long epoch)
+ {
+ EpochState epochState = epochs.get(epoch);
+ if (epochState == null)
+ return null;
+ return epochState.global();
+ }
}
diff --git a/accord-core/src/main/java/accord/utils/Base62.java b/accord-core/src/main/java/accord/utils/Base62.java
new file mode 100644
index 00000000..a1685cfb
--- /dev/null
+++ b/accord-core/src/main/java/accord/utils/Base62.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package accord.utils;
+
+public class Base62
+{
+ private static final long OVERFLOWL = Long.MAX_VALUE / 62;
+ public static String encode(long value)
+ {
+ if (value == 0) return "0";
+ StringBuilder sb = new StringBuilder();
+ while (value > 0)
+ {
+ int next = (int) (value % 62);
+ char c;
+ if (next < 10) c = (char) ('0' + next);
+ else if (next < 36) c = (char) ('a' + next);
+ else c = (char) ('A' + next);
+ sb.append(c);
+ value /= 62;
+ }
+ return sb.toString();
+ }
+
+ public static long decodeLong(String value)
+ {
+ long result = 0;
+ for (int i = 0; i < value.length(); i++)
+ {
+ if (result >= OVERFLOWL)
+ throw new IllegalArgumentException("base62 string is too large for long: " + value);
+ result *= 62;
+ char c = value.charAt(i);
+ if (c >= '0' && c <= '9') result += c - '0';
+ else if (c >= 'a' && c <= 'z') result += c - 'a' + 10;
+ else if (c >= 'A' && c <= 'Z') result += c - 'A' + 36;
+ else throw new IllegalArgumentException("Invalid base62 string: " + value);
+ }
+ return result;
+ }
+}
diff --git a/accord-core/src/main/java/accord/utils/ThreadPoolScheduler.java b/accord-core/src/main/java/accord/utils/ThreadPoolScheduler.java
index f26a94fd..f2409f41 100644
--- a/accord-core/src/main/java/accord/utils/ThreadPoolScheduler.java
+++ b/accord-core/src/main/java/accord/utils/ThreadPoolScheduler.java
@@ -101,7 +101,7 @@ public class ThreadPoolScheduler implements Scheduler
public void stop()
{
- exec.shutdown();
+ exec.shutdownNow();
try
{
if (!exec.awaitTermination(1L, TimeUnit.MINUTES))
diff --git a/accord-core/src/main/java/accord/utils/Timestamped.java b/accord-core/src/main/java/accord/utils/Timestamped.java
index fad231f2..3b7f60a3 100644
--- a/accord-core/src/main/java/accord/utils/Timestamped.java
+++ b/accord-core/src/main/java/accord/utils/Timestamped.java
@@ -73,4 +73,9 @@ public class Timestamped<T>
{
return toString.apply(data);
}
+
+ public boolean equals(Timestamped<T> that, BiPredicate<T, T> equality)
+ {
+ return this.timestamp.equals(that.timestamp) && equality.test(this.data, that.data);
+ }
}
diff --git a/accord-core/src/test/java/accord/KeysTest.java b/accord-core/src/test/java/accord/KeysTest.java
index 67a329e3..3538e21a 100644
--- a/accord-core/src/test/java/accord/KeysTest.java
+++ b/accord-core/src/test/java/accord/KeysTest.java
@@ -32,10 +32,9 @@ import accord.api.RoutingKey;
import accord.impl.IntKey;
import accord.impl.IntKey.Raw;
import accord.primitives.AbstractKeys;
+import accord.primitives.Keys;
import accord.primitives.Range;
import accord.primitives.Ranges;
-import accord.primitives.Keys;
-
import accord.primitives.RoutableKey;
import accord.primitives.Routables;
import accord.primitives.RoutingKeys;
diff --git a/accord-core/src/test/java/accord/burn/BurnTest.java b/accord-core/src/test/java/accord/burn/BurnTest.java
index 4678cec3..69a7b683 100644
--- a/accord-core/src/test/java/accord/burn/BurnTest.java
+++ b/accord-core/src/test/java/accord/burn/BurnTest.java
@@ -187,7 +187,7 @@ public class BurnTest
return new Txn.InMemory(kind, new Keys(requestKeys), read, query, update);
};
}
- packets.add(new Packet(client, node, count, new ListRequest(description, txnGenerator, listener)));
+ packets.add(new Packet(client, node, Long.MAX_VALUE, count, new ListRequest(description, txnGenerator, listener)));
}
return packets;
@@ -287,7 +287,7 @@ public class BurnTest
}
@SuppressWarnings("unused")
- static void reconcile(long seed, TopologyFactory topologyFactory, List<Id> clients, List<Id> nodes, int keyCount, int operations, int concurrency) throws ExecutionException, InterruptedException
+ static void reconcile(long seed, TopologyFactory topologyFactory, List<Id> clients, List<Id> nodes, int keyCount, int prefixCount, int operations, int concurrency) throws ExecutionException, InterruptedException
{
RandomSource random1 = new DefaultRandom(), random2 = new DefaultRandom();
@@ -295,14 +295,14 @@ public class BurnTest
random2.setSeed(seed);
ExecutorService exec = Executors.newFixedThreadPool(2);
RandomDelayQueue.ReconcilingQueueFactory factory = new RandomDelayQueue.ReconcilingQueueFactory(seed);
- Future<?> f1 = exec.submit(() -> burn(random1, topologyFactory, clients, nodes, keyCount, operations, concurrency, factory.get(true)));
- Future<?> f2 = exec.submit(() -> burn(random2, topologyFactory, clients, nodes, keyCount, operations, concurrency, factory.get(false)));
+ Future<?> f1 = exec.submit(() -> burn(random1, topologyFactory, clients, nodes, keyCount, prefixCount, operations, concurrency, factory.get(true)));
+ Future<?> f2 = exec.submit(() -> burn(random2, topologyFactory, clients, nodes, keyCount, prefixCount, operations, concurrency, factory.get(false)));
exec.shutdown();
f1.get();
f2.get();
}
- static void burn(RandomSource random, TopologyFactory topologyFactory, List<Id> clients, List<Id> nodes, int keyCount, int operations, int concurrency, PendingQueue pendingQueue)
+ static void burn(RandomSource random, TopologyFactory topologyFactory, List<Id> clients, List<Id> nodes, int keyCount, int prefixCount, int operations, int concurrency, PendingQueue pendingQueue)
{
List<Throwable> failures = Collections.synchronizedList(new ArrayList<>());
AtomicLong progress = new AtomicLong();
@@ -314,7 +314,7 @@ public class BurnTest
RandomSource retryRandom = random.fork();
retryBootstrap = retry -> {
long delay = retryRandom.nextInt(1, 15);
- queue.add((PendingRunnable) retry::run, delay, TimeUnit.SECONDS);
+ queue.add(PendingRunnable.create(retry::run), delay, TimeUnit.SECONDS);
};
}
IntSupplier coordinationDelays, progressDelays, timeoutDelays;
@@ -324,7 +324,7 @@ public class BurnTest
progressDelays = delayGenerator(rnd, 1, 100, 100, 1000);
timeoutDelays = delayGenerator(rnd, 500, 800, 1000, 10000);
}
- Function<BiConsumer<Timestamp, Ranges>, ListAgent> agentSupplier = onStale -> new ListAgent(random.fork(), 1000L, failures::add, retryBootstrap, onStale, coordinationDelays, progressDelays, timeoutDelays);
+ Function<BiConsumer<Timestamp, Ranges>, ListAgent> agentSupplier = onStale -> new ListAgent(random.fork(), 1000L, failures::add, retryBootstrap, onStale, coordinationDelays, progressDelays, timeoutDelays, pendingQueue::nowInMillis);
Supplier<LongSupplier> nowSupplier = () -> {
RandomSource forked = random.fork();
@@ -340,12 +340,16 @@ public class BurnTest
SimulatedDelayedExecutorService globalExecutor = new SimulatedDelayedExecutorService(queue, new ListAgent(random.fork(), 1000L, failures::add, retryBootstrap, (i1, i2) -> {
throw new IllegalAccessError("Global executor should enver get a stale event");
- }, coordinationDelays, progressDelays, timeoutDelays));
- Int2ObjectHashMap<Verifier> validators = new Int2ObjectHashMap<>();
+ }, coordinationDelays, progressDelays, timeoutDelays, queue::nowInMillis));
+ Verifier verifier = createVerifier(keyCount * prefixCount);
Function<CommandStore, AsyncExecutor> executor = ignore -> globalExecutor;
MessageListener listener = MessageListener.get();
-
+
+ int[] prefixes = IntStream.concat(IntStream.of(0), IntStream.generate(() -> random.nextInt(1024))).distinct().limit(prefixCount).toArray();
+ int[] newPrefixes = Arrays.copyOfRange(prefixes, 1, prefixes.length);
+ Arrays.sort(prefixes);
+
int[] keys = IntStream.range(0, keyCount).toArray();
Packet[] requests = toArray(generate(random, listener, executor, clients, nodes, keys, operations), Packet[]::new);
int[] starts = new int[requests.length];
@@ -424,28 +428,25 @@ public class BurnTest
// the current logic for add keyspace only knows what is there, so a ABA problem exists where keyspaces
// may come back... logically this is a problem as the history doesn't get reset, but practically that
// is fine as the backing map and the validator are consistent
- Int2ObjectHashMap<Verifier.Checker> seen = new Int2ObjectHashMap<>();
+ Verifier.Checker check = verifier.witness(start, end);
for (int i = 0 ; i < reply.read.length ; ++i)
{
Key key = reply.responseKeys.get(i);
int prefix = prefix(key);
int keyValue = key(key);
int k = Arrays.binarySearch(keys, keyValue);
- Verifier verifier = validators.computeIfAbsent(prefix, ignore -> createVerifier(Integer.toString(prefix), keyCount));
- Verifier.Checker check = seen.computeIfAbsent(prefix, ignore -> verifier.witness(start, end));
int[] read = reply.read[i];
int write = reply.update == null ? -1 : reply.update.getOrDefault(key, -1);
+ int prefixIndex = Arrays.binarySearch(prefixes, prefix);
+ int index = keyCount * prefixIndex + k;
if (read != null)
- check.read(k, read);
+ check.read(index, read);
if (write >= 0)
- check.write(k, write);
- }
- for (Verifier.Checker check : seen.values())
- {
- check.close();
+ check.write(index, write);
}
+ check.close();
}
catch (Throwable t)
{
@@ -456,7 +457,7 @@ public class BurnTest
Map<MessageType, Stats> messageStatsMap;
try
{
- messageStatsMap = Cluster.run(toArray(nodes, Id[]::new), listener, () -> queue,
+ messageStatsMap = Cluster.run(toArray(nodes, Id[]::new), newPrefixes, listener, () -> queue,
(id, onStale) -> globalExecutor.withAgent(agentSupplier.apply(onStale)),
queue::checkFailures,
responseSink, random::fork, nowSupplier,
@@ -464,11 +465,12 @@ public class BurnTest
onSubmitted::set,
ignore -> {}
);
- for (Verifier verifier : validators.values())
- verifier.close();
+ verifier.close();
}
catch (Throwable t)
{
+ logger.info("Keys: " + Arrays.toString(keys));
+ logger.info("Prefixes: " + Arrays.toString(prefixes));
for (int i = 0 ; i < requests.length ; ++i)
{
logger.info("{}", requests[i]);
@@ -518,11 +520,11 @@ public class BurnTest
return stats.toString();
}
- private static Verifier createVerifier(String prefix, int keyCount)
+ private static Verifier createVerifier(int keyCount)
{
if (!ElleVerifier.Support.allowed())
- return new StrictSerializabilityVerifier(prefix, keyCount);
- return CompositeVerifier.create(new StrictSerializabilityVerifier(prefix, keyCount),
+ return new StrictSerializabilityVerifier("", keyCount);
+ return CompositeVerifier.create(new StrictSerializabilityVerifier("", keyCount),
new ElleVerifier());
}
@@ -625,6 +627,7 @@ public class BurnTest
clients,
nodes,
5 + random.nextInt(15),
+ 5 + random.nextInt(15),
operations,
10 + random.nextInt(30),
new Factory(random).get());
@@ -658,6 +661,7 @@ public class BurnTest
clients,
nodes,
5 + random.nextInt(15),
+ 5 + random.nextInt(15),
operations,
10 + random.nextInt(30));
}
diff --git a/accord-core/src/test/java/accord/burn/BurnTestConfigurationService.java b/accord-core/src/test/java/accord/burn/BurnTestConfigurationService.java
index 01ea96d9..f1f2af43 100644
--- a/accord-core/src/test/java/accord/burn/BurnTestConfigurationService.java
+++ b/accord-core/src/test/java/accord/burn/BurnTestConfigurationService.java
@@ -192,7 +192,8 @@ public class BurnTestConfigurationService extends AbstractConfigurationService.M
@Override
public void reportEpochRedundant(Ranges ranges, long epoch)
{
- Topology topology = lookup.apply(localId).topology().globalForEpoch(epoch);
- topologyUpdates.epochRedundant(lookup.apply(localId), topology.nodes(), ranges, epoch);
+ Topology topology = lookup.apply(localId).topology().maybeGlobalForEpoch(epoch);
+ if (topology != null)
+ topologyUpdates.epochRedundant(lookup.apply(localId), topology.nodes(), ranges, epoch);
}
}
diff --git a/accord-core/src/test/java/accord/coordinate/PreAcceptTrackerTest.java b/accord-core/src/test/java/accord/coordinate/PreAcceptTrackerTest.java
index b528742c..179e0a72 100644
--- a/accord-core/src/test/java/accord/coordinate/PreAcceptTrackerTest.java
+++ b/accord-core/src/test/java/accord/coordinate/PreAcceptTrackerTest.java
@@ -43,13 +43,11 @@ public class PreAcceptTrackerTest
private static void assertResponseState(FastPathTracker responses,
boolean quorumReached,
boolean fastPathAccepted,
- boolean failed,
- boolean hasOutstandingResponses)
+ boolean failed)
{
Assertions.assertEquals(quorumReached, responses.hasReachedQuorum());
Assertions.assertEquals(fastPathAccepted, responses.hasFastPathAccepted());
Assertions.assertEquals(failed, responses.hasFailed());
- Assertions.assertEquals(hasOutstandingResponses, responses.hasInFlight());
}
@Test
@@ -59,13 +57,13 @@ public class PreAcceptTrackerTest
FastPathTracker responses = new FastPathTracker(topologies(subTopology));
responses.recordSuccess(ids[0], false);
- assertResponseState(responses, false, false, false, true);
+ assertResponseState(responses, false, false, false);
responses.recordSuccess(ids[1], false);
- assertResponseState(responses, true, false, false, true);
+ assertResponseState(responses, true, false, false);
responses.recordSuccess(ids[2], false);
- assertResponseState(responses, true, false, false, false);
+ assertResponseState(responses, true, false, false);
}
@Test
@@ -75,13 +73,13 @@ public class PreAcceptTrackerTest
FastPathTracker responses = new FastPathTracker(topologies(subTopology));
responses.recordSuccess(ids[0], true);
- assertResponseState(responses, false, false, false, true);
+ assertResponseState(responses, false, false, false);
responses.recordSuccess(ids[1], true);
- assertResponseState(responses, true, false, false, true);
+ assertResponseState(responses, true, false, false);
responses.recordSuccess(ids[2], true);
- assertResponseState(responses, true, true, false, false);
+ assertResponseState(responses, true, true, false);
}
/**
@@ -94,14 +92,14 @@ public class PreAcceptTrackerTest
FastPathTracker responses = new FastPathTracker(topologies(subTopology));
responses.recordSuccess(ids[0], false);
- assertResponseState(responses, false, false, false, true);
+ assertResponseState(responses, false, false, false);
responses.recordSuccess(ids[1], false);
- assertResponseState(responses, true, false, false, true);
+ assertResponseState(responses, true, false, false);
Assertions.assertFalse(subTopology.get(0).nodes.contains(ids[4]));
responses.recordSuccess(ids[4], false);
- assertResponseState(responses, true, false, false, true);
+ assertResponseState(responses, true, false, false);
}
@Test
@@ -111,13 +109,13 @@ public class PreAcceptTrackerTest
FastPathTracker responses = new FastPathTracker(topologies(subTopology));
responses.recordSuccess(ids[0], true);
- assertResponseState(responses, false, false, false, true);
+ assertResponseState(responses, false, false, false);
responses.recordFailure(ids[1]);
- assertResponseState(responses, false, false, false, true);
+ assertResponseState(responses, false, false, false);
responses.recordFailure(ids[2]);
- assertResponseState(responses, false, false, true, false);
+ assertResponseState(responses, false, false, true);
}
@Test
@@ -135,19 +133,19 @@ public class PreAcceptTrackerTest
Assertions.assertSame(subTopology.get(2), responses.get(2).shard);
responses.recordSuccess(ids[1], true);
- assertResponseState(responses, false, false, false, true);
+ assertResponseState(responses, false, false, false);
responses.recordSuccess(ids[2], true);
- assertResponseState(responses, false, false, false, true);
+ assertResponseState(responses, false, false, false);
responses.recordSuccess(ids[3], true);
// the middle shard will have reached fast path
Assertions.assertTrue(responses.get(1).hasMetFastPathCriteria());
// but since the others haven't, it won't report it as accepted
- assertResponseState(responses, true, false, false, true);
+ assertResponseState(responses, true, false, false);
responses.recordSuccess(ids[0], true);
responses.recordSuccess(ids[4], true);
- assertResponseState(responses, true, true, false, false);
+ assertResponseState(responses, true, true, false);
}
}
diff --git a/accord-core/src/test/java/accord/coordinate/tracking/FastPathTrackerReconciler.java b/accord-core/src/test/java/accord/coordinate/tracking/FastPathTrackerReconciler.java
index 95517ae0..ab39e5ad 100644
--- a/accord-core/src/test/java/accord/coordinate/tracking/FastPathTrackerReconciler.java
+++ b/accord-core/src/test/java/accord/coordinate/tracking/FastPathTrackerReconciler.java
@@ -28,7 +28,7 @@ import java.util.ArrayList;
public class FastPathTrackerReconciler extends TrackerReconciler<FastPathShardTracker, FastPathTracker, FastPathTrackerReconciler.Rsp>
{
- enum Rsp { FAST, SLOW, FAIL }
+ enum Rsp { FAST, SLOW, DELAY, FAIL }
FastPathTrackerReconciler(RandomSource random, Topologies topologies)
{
@@ -48,6 +48,7 @@ public class FastPathTrackerReconciler extends TrackerReconciler<FastPathShardTr
default: throw new AssertionError();
case FAST: inflight.remove(from); return tracker.recordSuccess(from, true);
case SLOW: inflight.remove(from); return tracker.recordSuccess(from, false);
+ case DELAY: return tracker.recordDelayed(from);
case FAIL: inflight.remove(from); return tracker.recordFailure(from);
}
}
@@ -64,7 +65,7 @@ public class FastPathTrackerReconciler extends TrackerReconciler<FastPathShardTr
case Success:
Assertions.assertTrue(tracker.all(FastPathShardTracker::hasReachedQuorum));
- Assertions.assertTrue(tracker.all(shard -> shard.hasRejectedFastPath() || shard.hasMetFastPathCriteria()));
+ Assertions.assertTrue(tracker.all(shard -> shard.hasRejectedFastPath() || shard.hasMetFastPathCriteria() || shard.isFastPathDelayed()));
Assertions.assertFalse(tracker.any(FastPathShardTracker::hasFailed));
break;
diff --git a/accord-core/src/test/java/accord/coordinate/tracking/ReadTrackerReconciler.java b/accord-core/src/test/java/accord/coordinate/tracking/ReadTrackerReconciler.java
index 0f53bd48..13810a8e 100644
--- a/accord-core/src/test/java/accord/coordinate/tracking/ReadTrackerReconciler.java
+++ b/accord-core/src/test/java/accord/coordinate/tracking/ReadTrackerReconciler.java
@@ -37,6 +37,7 @@ public class ReadTrackerReconciler extends TrackerReconciler<ReadShardTracker, R
public InFlightCapturingReadTracker(Topologies topologies)
{
super(topologies);
+ initialise();
}
@Override
diff --git a/accord-core/src/test/java/accord/coordinate/tracking/ReadTrackerTest.java b/accord-core/src/test/java/accord/coordinate/tracking/ReadTrackerTest.java
index 04eecf14..348cc6eb 100644
--- a/accord-core/src/test/java/accord/coordinate/tracking/ReadTrackerTest.java
+++ b/accord-core/src/test/java/accord/coordinate/tracking/ReadTrackerTest.java
@@ -62,6 +62,7 @@ public class ReadTrackerTest
public TestReadTracker(Topologies topologies)
{
super(topologies);
+ Assertions.assertTrue(initialise());
}
@Override
diff --git a/accord-core/src/test/java/accord/coordinate/tracking/TrackerReconciler.java b/accord-core/src/test/java/accord/coordinate/tracking/TrackerReconciler.java
index 6aae00d1..e329b6b8 100644
--- a/accord-core/src/test/java/accord/coordinate/tracking/TrackerReconciler.java
+++ b/accord-core/src/test/java/accord/coordinate/tracking/TrackerReconciler.java
@@ -124,7 +124,7 @@ public abstract class TrackerReconciler<ST extends ShardTracker, T extends Abstr
Deque<Topology> topologies = new ArrayDeque<>();
topologies.add(topology);
TopologyUpdates topologyUpdates = new TopologyUpdates(ignore -> executor);
- TopologyRandomizer configRandomizer = new TopologyRandomizer(() -> random, topology, topologyUpdates, null, TopologyRandomizer.Listeners.NOOP);
+ TopologyRandomizer configRandomizer = new TopologyRandomizer(() -> random, new int[] {1, 2, 3, 4, 5}, topology, topologyUpdates, null, TopologyRandomizer.Listeners.NOOP);
while (--count > 0)
{
Topology next = configRandomizer.updateTopology();
diff --git a/accord-core/src/test/java/accord/impl/RemoteListenersTest.java b/accord-core/src/test/java/accord/impl/RemoteListenersTest.java
index 2b12039c..7f1f81b8 100644
--- a/accord-core/src/test/java/accord/impl/RemoteListenersTest.java
+++ b/accord-core/src/test/java/accord/impl/RemoteListenersTest.java
@@ -414,7 +414,7 @@ public class RemoteListenersTest
@Override public CommandStore commandStore() { return commandStore; }
@Override protected SafeCommand getInternal(TxnId txnId) { return null; }
- @Override protected SafeCommand ifLoadedAndInitialisedAndNotErasedInternal(TxnId txnId) { return null; }
+ @Override protected SafeCommand ifLoadedInternal(TxnId txnId) { return null; }
@Override protected SafeCommandsForKey getInternal(RoutingKey key) { return null;}
@Override protected SafeCommandsForKey ifLoadedInternal(RoutingKey key) { return null;}
@Override public SafeTimestampsForKey timestampsForKey(RoutingKey key) { return null; }
diff --git a/accord-core/src/test/java/accord/impl/basic/Cluster.java b/accord-core/src/test/java/accord/impl/basic/Cluster.java
index 56b7a970..a0db0f7e 100644
--- a/accord-core/src/test/java/accord/impl/basic/Cluster.java
+++ b/accord-core/src/test/java/accord/impl/basic/Cluster.java
@@ -29,7 +29,9 @@ import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
+import java.util.NavigableMap;
import java.util.Set;
+import java.util.TreeMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
@@ -53,6 +55,7 @@ import accord.api.LocalConfig;
import accord.api.MessageSink;
import accord.api.RoutingKey;
import accord.api.Scheduler;
+import accord.api.Scheduler.Scheduled;
import accord.burn.BurnTestConfigurationService;
import accord.burn.TopologyUpdates;
import accord.burn.random.FrequentLargeRange;
@@ -63,10 +66,10 @@ import accord.coordinate.Invalidated;
import accord.coordinate.Preempted;
import accord.coordinate.Timeout;
import accord.coordinate.Truncated;
-import accord.impl.DurabilityScheduling;
import accord.impl.DefaultLocalListeners;
import accord.impl.DefaultRemoteListeners;
import accord.impl.DefaultTimeouts;
+import accord.impl.DurabilityScheduling;
import accord.impl.InMemoryCommandStore.GlobalCommand;
import accord.impl.MessageListener;
import accord.impl.PrefixedIntHashKey;
@@ -82,11 +85,10 @@ import accord.local.CommandStore;
import accord.local.DurableBefore;
import accord.local.Node;
import accord.local.Node.Id;
-import accord.local.TimeService;
-import accord.primitives.SaveStatus;
+import accord.local.RedundantBefore;
import accord.local.ShardDistributor;
-import accord.primitives.Seekables;
-import accord.primitives.Status;
+import accord.local.StoreParticipants;
+import accord.local.TimeService;
import accord.local.cfk.CommandsForKey;
import accord.messages.Message;
import accord.messages.MessageType;
@@ -97,6 +99,10 @@ import accord.primitives.FullRoute;
import accord.primitives.Keys;
import accord.primitives.Range;
import accord.primitives.Ranges;
+import accord.primitives.RoutableKey;
+import accord.primitives.SaveStatus;
+import accord.primitives.Seekables;
+import accord.primitives.Status;
import accord.primitives.Timestamp;
import accord.primitives.Txn;
import accord.primitives.TxnId;
@@ -105,8 +111,10 @@ import accord.topology.TopologyRandomizer;
import accord.utils.Gens;
import accord.utils.Invariants;
import accord.utils.RandomSource;
+import accord.utils.Timestamped;
import accord.utils.async.AsyncChains;
import accord.utils.async.AsyncResult;
+import org.agrona.collections.Int2ObjectHashMap;
import static accord.impl.basic.Cluster.OverrideLinksKind.NONE;
import static accord.impl.basic.Cluster.OverrideLinksKind.RANDOM_BIDIRECTIONAL;
@@ -122,7 +130,7 @@ import static java.util.concurrent.TimeUnit.NANOSECONDS;
import static java.util.concurrent.TimeUnit.SECONDS;
import static java.util.stream.Collectors.toList;
-public class Cluster implements Scheduler
+public class Cluster
{
public static final Logger trace = LoggerFactory.getLogger("accord.impl.basic.Trace");
@@ -164,6 +172,56 @@ public class Cluster implements Scheduler
}
}
+ public class ClusterScheduler implements Scheduler
+ {
+ final int source;
+
+ ClusterScheduler(int source)
+ {
+ this.source = source;
+ }
+
+ @Override
+ public void now(Runnable run)
+ {
+ run.run();
+ }
+
+ @Override
+ public Scheduled recurring(Runnable run, long delay, TimeUnit units)
+ {
+ return recurring(run, () -> delay, units);
+ }
+
+ @Override
+ public Scheduled once(Runnable run, long delay, TimeUnit units)
+ {
+ RecurringPendingRunnable result = new RecurringPendingRunnable(source, null, run, () -> delay, units, false);
+ pending.add(result, delay, units);
+ return result;
+ }
+
+ @Override
+ public Scheduled selfRecurring(Runnable run, long delay, TimeUnit units)
+ {
+ RecurringPendingRunnable result = new RecurringPendingRunnable(source, null, run, () -> delay, units, true);
+ pending.add(result, delay, units);
+ return result;
+ }
+
+ public Scheduled recurring(Runnable run, LongSupplier delay, TimeUnit units)
+ {
+ RecurringPendingRunnable result = new RecurringPendingRunnable(source, pending, run, delay, units, true);
+ pending.add(result, delay.getAsLong(), units);
+ return result;
+ }
+
+ public void onDone(Runnable run)
+ {
+ Cluster.this.onDone(run);
+ }
+ }
+
final Map<MessageType, Stats> statsMap = new HashMap<>();
final RandomSource random;
@@ -177,7 +235,6 @@ public class Cluster implements Scheduler
final Map<Id, NodeSink> sinks = new HashMap<>();
final MessageListener messageListener;
int clock;
- int recurring;
BiFunction<Id, Id, Link> links;
public Cluster(RandomSource random, MessageListener messageListener, Supplier<PendingQueue> queueSupplier, Runnable checkFailures, Function<Id, Node> lookup, Function<Id, Journal> journalLookup, IntSupplier rf, Consumer<Packet> responseSink)
@@ -193,9 +250,9 @@ public class Cluster implements Scheduler
this.links = linkConfig.defaultLinks;
}
- NodeSink create(Id self, RandomSource random)
+ NodeSink create(Id self, NodeSink.TimeoutSupplier timeouts)
{
- NodeSink sink = new NodeSink(self, lookup, this, random);
+ NodeSink sink = new NodeSink(self, lookup, this, timeouts);
sinks.put(self, sink);
return sink;
}
@@ -224,34 +281,36 @@ public class Cluster implements Scheduler
boolean hasNonRecurring()
{
- boolean hasNonRecurring = false;
+ if (pending.hasNonRecurring())
+ return true;
+
for (Pending p : pending)
{
if (!(p instanceof RecurringPendingRunnable))
continue;
- RecurringPendingRunnable r = (RecurringPendingRunnable) p;
- if (r.requeue.hasNonRecurring())
- {
- hasNonRecurring = true;
- break;
- }
+
+ RecurringPendingRunnable r = (RecurringPendingRunnable) p.origin();
+ if (r.requeue != null && r.requeue.hasNonRecurring())
+ return true;
}
- return hasNonRecurring;
+ return false;
}
public boolean processPending()
{
checkFailures.run();
// All remaining tasks are recurring
- if (recurring > 0 && pending.size() == recurring && !hasNonRecurring())
+ if (!hasNonRecurring())
return false;
Pending next = pending.poll();
if (next == null)
return false;
+ Pending.Global.setActiveOrigin(next);
processNext(next);
+ Pending.Global.clearActiveOrigin();
checkFailures.run();
return true;
@@ -308,51 +367,14 @@ public class Cluster implements Scheduler
trace.trace("{} DROP[{}] (from:{}, to:{}, {}:{}, body:{})", clock++, lookup.apply(to).epoch(), from, to, message instanceof Reply ? "replyTo" : "id", id, message);
}
- @Override
- public Scheduled recurring(Runnable run, long delay, TimeUnit units)
- {
- return recurring(run, () -> delay, units);
- }
-
- @Override
- public Scheduled once(Runnable run, long delay, TimeUnit units)
- {
- RecurringPendingRunnable result = new RecurringPendingRunnable(null, run, () -> delay, units, false);
- pending.add(result, delay, units);
- return result;
- }
-
- @Override
- public Scheduled selfRecurring(Runnable run, long delay, TimeUnit units)
- {
- RecurringPendingRunnable result = new RecurringPendingRunnable(null, run, () -> delay, units, true);
- pending.add(result, delay, units);
- return result;
- }
-
- public Scheduled recurring(Runnable run, LongSupplier delay, TimeUnit units)
- {
- RecurringPendingRunnable result = new RecurringPendingRunnable(pending, run, delay, units, true);
- ++recurring;
- result.onCancellation(() -> --recurring);
- pending.add(result, delay.getAsLong(), units);
- return result;
-
- }
-
public void onDone(Runnable run)
{
onDone.add(run);
}
- @Override
- public void now(Runnable run)
- {
- run.run();
- }
-
// TODO (expected): merge with BurnTest.burn
public static Map<MessageType, Cluster.Stats> run(Supplier<RandomSource> randomSupplier,
+ int[] prefixes,
List<Node.Id> nodes,
Topology initialTopology,
Function<Map<Node.Id, Node>, Request> init)
@@ -364,7 +386,7 @@ public class Cluster implements Scheduler
RandomSource rnd = randomSupplier.get();
retryBootstrap = retry -> {
long delay = rnd.nextInt(1, 15);
- queue.add((PendingRunnable) retry::run, delay, TimeUnit.SECONDS);
+ queue.add(PendingRunnable.create(retry::run), delay, TimeUnit.SECONDS);
};
}
IntSupplier coordinationDelays, progressDelays, timeoutDelays;
@@ -372,11 +394,11 @@ public class Cluster implements Scheduler
RandomSource rnd = randomSupplier.get();
timeoutDelays = progressDelays = coordinationDelays = () -> rnd.nextInt(100, 1000);
}
- Function<BiConsumer<Timestamp, Ranges>, ListAgent> agentSupplier = onStale -> new ListAgent(randomSupplier.get(), 1000L, failures::add, retryBootstrap, onStale, coordinationDelays, progressDelays, timeoutDelays);
+ Function<BiConsumer<Timestamp, Ranges>, ListAgent> agentSupplier = onStale -> new ListAgent(randomSupplier.get(), 1000L, failures::add, retryBootstrap, onStale, coordinationDelays, progressDelays, timeoutDelays, queue::nowInMillis);
RandomSource nowRandom = randomSupplier.get();
Supplier<LongSupplier> nowSupplier = () -> {
RandomSource forked = nowRandom.fork();
- // TODO (now): meta-randomise scale of clock drift
+ // TODO (testing): meta-randomise scale of clock drift
return FrequentLargeRange.builder(forked)
.ratio(1, 5)
.small(50, 5000, TimeUnit.MICROSECONDS)
@@ -387,7 +409,7 @@ public class Cluster implements Scheduler
};
SimulatedDelayedExecutorService globalExecutor = new SimulatedDelayedExecutorService(queue, new ListAgent(randomSupplier.get(), 1000L, failures::add, retryBootstrap, (i1, i2) -> {
throw new IllegalAccessError("Global executor should never get a stale event");
- }, () -> { throw new UnsupportedOperationException(); }, () -> { throw new UnsupportedOperationException(); }, timeoutDelays));
+ }, () -> { throw new UnsupportedOperationException(); }, () -> { throw new UnsupportedOperationException(); }, timeoutDelays, queue::nowInMillis));
TopologyFactory topologyFactory = new TopologyFactory(initialTopology.maxRf(), initialTopology.ranges().stream().toArray(Range[]::new))
{
@Override
@@ -399,6 +421,7 @@ public class Cluster implements Scheduler
AtomicInteger counter = new AtomicInteger();
AtomicReference<Map<Id, Node>> nodeMap = new AtomicReference<>();
Map<MessageType, Cluster.Stats> stats = Cluster.run(nodes.toArray(Node.Id[]::new),
+ prefixes,
MessageListener.get(),
() -> queue,
(id, onStale) -> globalExecutor.withAgent(agentSupplier.apply(onStale)),
@@ -415,7 +438,6 @@ public class Cluster implements Scheduler
@Override
public Packet get()
{
- // ((Cluster) node.scheduler()).onDone(() -> checkOnResult(homeKey, txnId, 0, null));
if (requestIterator == null)
{
Map<Node.Id, Node> nodes = nodeMap.get();
@@ -424,7 +446,7 @@ public class Cluster implements Scheduler
if (!requestIterator.hasNext())
return null;
Node.Id id = rs.pick(nodes);
- return new Packet(id, id, counter.incrementAndGet(), requestIterator.next());
+ return new Packet(id, id, Long.MAX_VALUE, counter.incrementAndGet(), requestIterator.next());
}
},
Runnable::run,
@@ -438,7 +460,7 @@ public class Cluster implements Scheduler
return stats;
}
- public static Map<MessageType, Stats> run(Id[] nodes, MessageListener messageListener, Supplier<PendingQueue> queueSupplier,
+ public static Map<MessageType, Stats> run(Id[] nodes, int[] prefixes, MessageListener messageListener, Supplier<PendingQueue> queueSupplier,
BiFunction<Id, BiConsumer<Timestamp, Ranges>, AgentExecutor> nodeExecutorSupplier,
Runnable checkFailures, Consumer<Packet> responseSink,
Supplier<RandomSource> randomSupplier, Supplier<LongSupplier> nowSupplierSupplier,
@@ -462,12 +484,40 @@ public class Cluster implements Scheduler
}
messageListener.onTopologyChange(t);
};
- TopologyRandomizer configRandomizer = new TopologyRandomizer(randomSupplier, topology, topologyUpdates, nodeMap::get, schemaApply);
+ NodeSink.TimeoutSupplier timeouts = new NodeSink.TimeoutSupplier()
+ {
+ final RandomSource random = randomSupplier.get();
+ final LongSupplier slowAt, expiresAt, failsAt;
+ {
+ int medianSlowAt = random.nextInt(100, 200);
+ int medianExpiresAt = random.nextInt(1000, 2000);
+ int medianFailsAt = random.nextInt(1000, 2000);
+
+ int minSlowAt = random.nextInt(0, 100);
+ int minExpiresAt = random.nextBiasedInt(500, 800, 1000);
+ int minFailsAt = random.nextBiasedInt(500, 800, 1000);
+
+ int maxSlowAt = random.nextBiasedInt(medianSlowAt + 100, medianSlowAt + 200, 1000);
+ int maxExpiresAt = random.nextBiasedInt(medianExpiresAt + 500, 3000, 10000);
+ int maxFailsAt = random.nextBiasedInt(medianFailsAt + 500, 3000, 10000);
+
+ slowAt = random.biasedUniformLongs(minSlowAt, medianSlowAt, maxSlowAt);
+ expiresAt = random.biasedUniformLongs(minExpiresAt, medianExpiresAt, maxExpiresAt);
+ failsAt = random.biasedUniformLongs(minFailsAt, medianFailsAt, maxFailsAt);
+ }
+ @Override public long slowAt() { return now() + slowAt.getAsLong();}
+ @Override public long expiresAt() { return now() + expiresAt.getAsLong(); }
+ @Override public long failsAt() { return now() + failsAt.getAsLong(); }
+ @Override public long now() { return sinks.pending.nowInMillis(); }
+ @Override public TimeUnit units() { return MILLISECONDS; }
+ };
+ TopologyRandomizer configRandomizer = new TopologyRandomizer(randomSupplier, prefixes, topology, topologyUpdates, nodeMap::get, schemaApply);
List<DurabilityScheduling> durabilityScheduling = new ArrayList<>();
List<Service> services = new ArrayList<>();
for (Id id : nodes)
{
- MessageSink messageSink = sinks.create(id, randomSupplier.get());
+ ClusterScheduler scheduler = sinks.new ClusterScheduler(id.id);
+ MessageSink messageSink = sinks.create(id, timeouts);
LongSupplier nowSupplier = nowSupplierSupplier.get();
LocalConfig localConfig = LocalConfig.DEFAULT;
BiConsumer<Timestamp, Ranges> onStale = (sinceAtLeast, ranges) -> configRandomizer.onStale(id, sinceAtLeast, ranges);
@@ -476,11 +526,43 @@ public class Cluster implements Scheduler
Journal journal = new Journal(id);
journalMap.put(id, journal);
BurnTestConfigurationService configService = new BurnTestConfigurationService(id, nodeExecutor, randomSupplier, topology, nodeMap::get, topologyUpdates);
- BooleanSupplier isLoadedCheck = Gens.supplier(Gens.bools().mixedDistribution().next(random), random);
+ DelayedCommandStores.CacheLoadingChance isLoadedCheck = new DelayedCommandStores.CacheLoadingChance()
+ {
+ final float presentChance = random.nextBoolean() ? 1.0f : random.nextFloat();
+ private final BooleanSupplier cacheEmptyChance = Gens.supplier(Gens.bools().mixedDistribution().next(random), random);
+ public boolean cacheEmpty()
+ {
+ return cacheEmptyChance.getAsBoolean();
+ }
+
+ private final BooleanSupplier cacheFullChance = Gens.supplier(Gens.bools().mixedDistribution().next(random), random);
+ public boolean cacheFull()
+ {
+ return cacheFullChance.getAsBoolean();
+ }
+
+ private final BooleanSupplier commandLoadedChance = random.biasedUniformBools(presentChance);
+ public boolean commandLoaded()
+ {
+ return commandLoadedChance.getAsBoolean();
+ }
+
+ private final BooleanSupplier cfkLoadedChance = random.biasedUniformBools(presentChance);
+ public boolean cfkLoaded()
+ {
+ return cfkLoadedChance.getAsBoolean();
+ }
+
+ private final BooleanSupplier tfkLoadedChance = random.biasedUniformBools(presentChance);
+ public boolean tfkLoaded()
+ {
+ return tfkLoadedChance.getAsBoolean();
+ }
+ };
Node node = new Node(id, messageSink, configService, TimeService.ofNonMonotonic(nowSupplier, MILLISECONDS),
- () -> new ListStore(sinks, random, id), new ShardDistributor.EvenSplit<>(8, ignore -> new PrefixedIntHashKey.Splitter()),
+ () -> new ListStore(scheduler, random, id), new ShardDistributor.EvenSplit<>(8, ignore -> new PrefixedIntHashKey.Splitter()),
nodeExecutor.agent(),
- randomSupplier.get(), sinks, SizeOfIntersectionSorter.SUPPLIER, DefaultRemoteListeners::new, DefaultTimeouts::new,
+ randomSupplier.get(), scheduler, SizeOfIntersectionSorter.SUPPLIER, DefaultRemoteListeners::new, DefaultTimeouts::new,
DefaultProgressLogs::new, DefaultLocalListeners.Factory::new, DelayedCommandStores.factory(sinks.pending, isLoadedCheck, journal), new CoordinationAdapter.DefaultFactory(),
DurableBefore.NOOP_PERSISTER, localConfig);
DurabilityScheduling durability = node.durabilityScheduling();
@@ -516,44 +598,36 @@ public class Cluster implements Scheduler
while (sinks.processPending());
Assertions.assertTrue(startup.isDone());
+ ClusterScheduler clusterScheduler = sinks.new ClusterScheduler(-1);
List<Id> nodesList = new ArrayList<>(Arrays.asList(nodes));
- Scheduled chaos = sinks.recurring(() -> {
+ Scheduled chaos = clusterScheduler.recurring(() -> {
sinks.links = sinks.linkConfig.overrideLinks.apply(nodesList);
if (random.decide(0.1f))
updateDurabilityRate.run();
}, 5L, SECONDS);
- Scheduled reconfigure = sinks.recurring(configRandomizer::maybeUpdateTopology, 1, SECONDS);
+ Scheduled reconfigure = clusterScheduler.recurring(configRandomizer::maybeUpdateTopology, 1, SECONDS);
- Scheduled purge = sinks.recurring(() -> {
- trace.debug("Triggering purge.");
- int numNodes = random.nextInt(1, nodeMap.size());
- for (int i = 0; i < numNodes; i++)
- {
- Id id = random.pick(nodes);
- Node node = nodeMap.get(id);
+ Purge purge = new Purge(clusterScheduler, random, nodesList, nodeMap, journalMap);
- Journal journal = journalMap.get(node.id());
- CommandStore[] stores = nodeMap.get(node.id()).commandStores().all();
- journal.purge((j) -> stores[j]);
- }
- }, () -> random.nextInt(1, 10), SECONDS);
-
- Scheduled restart = sinks.recurring(() -> {
+ Scheduled restart = clusterScheduler.recurring(() -> {
Id id = random.pick(nodes);
CommandStore[] stores = nodeMap.get(id).commandStores().all();
((DelayedCommandStore)stores[0]).unsafeRunIn(() -> {
- Predicate<Pending> pred = getPendingPredicate(stores);
+ Predicate<Pending> pred = getPendingPredicate(id.id, stores);
while (sinks.drain(pred));
// Journal cleanup is a rough equivalent of a node restart.
trace.debug("Triggering journal cleanup for node " + id);
CommandsForKey.disableLinearizabilityViolationsReporting();
ListStore listStore = (ListStore) nodeMap.get(id).commandStores().dataStore();
+ NavigableMap<RoutableKey, Timestamped<int[]>> prevData = listStore.copyOfCurrentData();
listStore.clear();
listStore.restoreFromSnapshot();
-
+ // we are simulating node restart, so its remote listeners will also be gone
+ ((DefaultRemoteListeners)nodeMap.get(id).remoteListeners()).clear();
Journal journal = journalMap.get(id);
+ Int2ObjectHashMap<NavigableMap<TxnId, Command>> beforeStores = copyCommands(stores);
for (CommandStore s : stores)
{
DelayedCommandStores.DelayedCommandStore store = (DelayedCommandStores.DelayedCommandStore) s;
@@ -562,9 +636,11 @@ public class Cluster implements Scheduler
}
while (sinks.drain(pred));
CommandsForKey.enableLinearizabilityViolationsReporting();
+ Invariants.checkState(listStore.equals(prevData));
+ verifyConsistentRestore(beforeStores, stores);
trace.debug("Done with cleanup.");
});
- }, () -> random.nextInt(1, 10), SECONDS);
+ }, () -> random.nextInt(10, 30), SECONDS);
durabilityScheduling.forEach(DurabilityScheduling::start);
services.forEach(Service::start);
@@ -612,7 +688,134 @@ public class Cluster implements Scheduler
}
}
- private static Predicate<Pending> getPendingPredicate(CommandStore[] stores)
+ private static class Purge
+ {
+ Scheduled scheduled;
+
+ Purge(Scheduler clusterScheduler, RandomSource rs, List<Id> nodes, Map<Id, Node> nodeMap, Map<Id, Journal> journalMap)
+ {
+ schedule(clusterScheduler, rs, nodes, nodeMap, journalMap);
+ }
+
+ void cancel()
+ {
+ scheduled.cancel();
+ }
+
+ private void schedule(Scheduler clusterScheduler, RandomSource rs, List<Id> nodes, Map<Id, Node> nodeMap, Map<Id, Journal> journalMap)
+ {
+ scheduled = clusterScheduler.selfRecurring(() -> run(clusterScheduler, rs, nodes, nodeMap, journalMap), rs.nextInt(1, 30), SECONDS);
+ }
+
+ private void run(Scheduler clusterScheduler, RandomSource rs, List<Id> nodes, Map<Id, Node> nodeMap, Map<Id, Journal> journalMap)
+ {
+ Id id = rs.pick(nodes);
+ Node node = nodeMap.get(id);
+
+ Journal journal = journalMap.get(node.id());
+ CommandStore[] stores = nodeMap.get(node.id()).commandStores().all();
+ // run on node scheduler so doesn't run during replay
+ scheduled = node.scheduler().selfRecurring(() -> {
+ journal.purge(j -> j < stores.length ? stores[j] : null);
+ schedule(clusterScheduler, rs, nodes, nodeMap, journalMap);
+ }, 0, SECONDS);
+ }
+
+ }
+
+ private static Int2ObjectHashMap<NavigableMap<TxnId, Command>> copyCommands(CommandStore[] stores)
+ {
+ Int2ObjectHashMap<NavigableMap<TxnId, Command>> result = new Int2ObjectHashMap<>();
+ for (CommandStore s : stores)
+ {
+ DelayedCommandStores.DelayedCommandStore store = (DelayedCommandStores.DelayedCommandStore) s;
+ NavigableMap<TxnId, Command> commands = new TreeMap<>();
+ result.put(store.id(), commands);
+ for (Map.Entry<TxnId, GlobalCommand> e : store.unsafeCommands().entrySet())
+ commands.put(e.getKey(), e.getValue().value());
+ }
+ return result;
+ }
+
+ private static void verifyConsistentRestore(Int2ObjectHashMap<NavigableMap<TxnId, Command>> beforeStores, CommandStore[] stores)
+ {
+ for (CommandStore s : stores)
+ {
+ DelayedCommandStores.DelayedCommandStore store = (DelayedCommandStores.DelayedCommandStore) s;
+ NavigableMap<TxnId, Command> before = beforeStores.get(store.id());
+ for (Map.Entry<TxnId, GlobalCommand> e : store.unsafeCommands().entrySet())
+ {
+ Command beforeCommand = before.get(e.getKey());
+ Command afterCommand = e.getValue().value();
+ if (beforeCommand == null)
+ {
+ Invariants.checkArgument(afterCommand.is(Status.NotDefined));
+ continue;
+ }
+ if (afterCommand.hasBeen(Status.Truncated))
+ {
+ if (afterCommand.is(Status.Invalidated))
+ Invariants.checkState(beforeCommand.hasBeen(Status.Truncated) || !beforeCommand.hasBeen(Status.PreCommitted)
+ && store.unsafeGetRedundantBefore().max(beforeCommand.participants().touches(), RedundantBefore.Entry::shardRedundantBefore).compareTo(beforeCommand.txnId()) >= 0);
+ continue;
+ }
+ if (beforeCommand.hasBeen(Status.Truncated))
+ {
+ Invariants.checkState(!beforeCommand.is(Status.Invalidated) || afterCommand.is(Status.Invalidated));
+ Invariants.checkState(beforeCommand.is(Status.Invalidated) || afterCommand.is(Status.Truncated) || afterCommand.is(Status.Applied));
+ continue;
+ }
+ Invariants.checkState(isConsistent(beforeCommand.saveStatus(), afterCommand.saveStatus())
+ && beforeCommand.executeAtOrTxnId().equals(afterCommand.executeAtOrTxnId())
+ && beforeCommand.acceptedOrCommitted().equals(afterCommand.acceptedOrCommitted())
+ && beforeCommand.promised().equals(afterCommand.promised())
+ && beforeCommand.durability().equals(afterCommand.durability()));
+ }
+ if (before.size() > store.unsafeCommands().size())
+ {
+ for (TxnId txnId : before.keySet())
+ {
+ if (!store.unsafeCommands().containsKey(txnId))
+ {
+ Command beforeCommand = before.get(txnId);
+ if (beforeCommand.saveStatus() == SaveStatus.Erased)
+ continue;
+
+ if (store.unsafeGetRedundantBefore().min(beforeCommand.participants().owns(), RedundantBefore.Entry::shardRedundantBefore).compareTo(txnId) > 0)
+ continue;
+
+ if (beforeCommand.participants().owns().isEmpty() && store.durableBefore().min(txnId).compareTo(Status.Durability.MajorityOrInvalidated) >= 0)
+ continue;
+
+ if (!beforeCommand.saveStatus().hasBeen(Status.PreCommitted) && store.unsafeGetRedundantBefore().min(beforeCommand.participants().owns(), RedundantBefore.Entry::locallyRedundantBefore).compareTo(txnId) > 0)
+ continue;
+
+ Invariants.checkState(false);
+ }
+ }
+ }
+ }
+ }
+
+ private static boolean isConsistent(SaveStatus before, SaveStatus after)
+ {
+ if (before == after)
+ return true;
+
+ if (before == SaveStatus.Uninitialised || before == SaveStatus.NotDefined)
+ return after == SaveStatus.Uninitialised || after == SaveStatus.NotDefined;
+
+ // depending on arrival order, an unmanaged txn may be ready to execute immediately or have to wait for another transaction to commit
+ if (before == SaveStatus.Stable || before == SaveStatus.ReadyToExecute)
+ return after == SaveStatus.Stable || after == SaveStatus.ReadyToExecute;
+
+ if (before == SaveStatus.PreApplied || before == SaveStatus.Applying || before == SaveStatus.Applied)
+ return after == SaveStatus.PreApplied || after == SaveStatus.Applying || after == SaveStatus.Applied;
+
+ return false;
+ }
+
+ private static Predicate<Pending> getPendingPredicate(int nodeId, CommandStore[] stores)
{
Set<CommandStore> nodeStores = new HashSet<>(Arrays.asList(stores));
return item -> {
@@ -621,6 +824,12 @@ public class Cluster implements Scheduler
DelayedCommandStore.DelayedTask<?> task = (DelayedCommandStore.DelayedTask<?>) item;
if (nodeStores.contains(task.parent()))
return true;
+ item = item.origin();
+ }
+ if (item instanceof RecurringPendingRunnable)
+ {
+ RecurringPendingRunnable recurring = (RecurringPendingRunnable) item;
+ return recurring.source == nodeId && (!recurring.isRecurring || recurring.origin() != recurring);
}
return false;
};
@@ -952,9 +1161,19 @@ public class Cluster implements Scheduler
return findMin(minSaveStatus, maxSaveStatus, txnId::equals);
}
+ public BlockingTransaction find(boolean onlyIfOwned, TxnId txnId, @Nullable SaveStatus minSaveStatus, @Nullable SaveStatus maxSaveStatus)
+ {
+ return findMin(onlyIfOwned, minSaveStatus, maxSaveStatus, txnId::equals);
+ }
+
public List<BlockingTransaction> findTransitivelyBlocking(TxnId txnId)
{
- BlockingTransaction txn = find(txnId, null, null);
+ return findTransitivelyBlocking(true, txnId);
+ }
+
+ public List<BlockingTransaction> findTransitivelyBlocking(boolean onlyIfOwned, TxnId txnId)
+ {
+ BlockingTransaction txn = find(onlyIfOwned, txnId, null, null);
if (txn == null)
return null;
@@ -986,7 +1205,12 @@ public class Cluster implements Scheduler
public BlockingTransaction findMin(@Nullable SaveStatus minSaveStatus, @Nullable SaveStatus maxSaveStatus, Predicate<TxnId> testTxnId)
{
- return find(minSaveStatus, maxSaveStatus, testTxnId, (min, test) -> {
+ return findMin(false, minSaveStatus, maxSaveStatus, testTxnId);
+ }
+
+ public BlockingTransaction findMin(boolean onlyIfOwned, @Nullable SaveStatus minSaveStatus, @Nullable SaveStatus maxSaveStatus, Predicate<TxnId> testTxnId)
+ {
+ return find(onlyIfOwned, minSaveStatus, maxSaveStatus, testTxnId, (min, test) -> {
int c = -1;
if (min == null || (c = test.txnId.compareTo(min.txnId)) <= 0 && (c < 0 || test.command.saveStatus().compareTo(min.command.saveStatus()) < 0))
min = test;
@@ -1002,11 +1226,11 @@ public class Cluster implements Scheduler
public List<BlockingTransaction> findAll(@Nullable SaveStatus minSaveStatus, @Nullable SaveStatus maxSaveStatus, Predicate<TxnId> testTxnId)
{
List<BlockingTransaction> result = new ArrayList<>();
- find(minSaveStatus, maxSaveStatus, testTxnId, (r, c) -> { r.add(c); return r; }, result);
+ find(false, minSaveStatus, maxSaveStatus, testTxnId, (r, c) -> { r.add(c); return r; }, result);
return result;
}
- public <T> T find(@Nullable SaveStatus minSaveStatus, @Nullable SaveStatus maxSaveStatus, Predicate<TxnId> testTxnId, BiFunction<T, BlockingTransaction, T> fold, T accumulate)
+ public <T> T find(boolean onlyIfOwned, @Nullable SaveStatus minSaveStatus, @Nullable SaveStatus maxSaveStatus, Predicate<TxnId> testTxnId, BiFunction<T, BlockingTransaction, T> fold, T accumulate)
{
for (Node.Id id : sinks.keySet())
{
@@ -1018,7 +1242,8 @@ public class Cluster implements Scheduler
for (Map.Entry<TxnId, GlobalCommand> e : store.unsafeCommands().entrySet())
{
Command command = e.getValue().value();
- if ((minSaveStatus == null || command.saveStatus().compareTo(minSaveStatus) >= 0) &&
+ if ((!onlyIfOwned || owns(command, store)) &&
+ (minSaveStatus == null || command.saveStatus().compareTo(minSaveStatus) >= 0) &&
(maxSaveStatus == null || command.saveStatus().compareTo(maxSaveStatus) <= 0) &&
(testTxnId == null || testTxnId.test(command.txnId())))
{
@@ -1031,6 +1256,15 @@ public class Cluster implements Scheduler
return accumulate;
}
+ private boolean owns(Command command, CommandStore commandStore)
+ {
+ StoreParticipants participants = command.participants();
+ if (participants == null)
+ return false;
+
+ return participants.owns().intersects(commandStore.unsafeRangesForEpoch().allBetween(command.txnId().epoch(), command.executeAtIfKnownElseTxnId()));
+ }
+
private BlockingTransaction toBlocking(Command command, DelayedCommandStore store)
{
Object blockedVia = null;
diff --git a/accord-core/src/test/java/accord/impl/basic/DelayedCommandStores.java b/accord-core/src/test/java/accord/impl/basic/DelayedCommandStores.java
index e5cf16d5..e3f7d8be 100644
--- a/accord-core/src/test/java/accord/impl/basic/DelayedCommandStores.java
+++ b/accord-core/src/test/java/accord/impl/basic/DelayedCommandStores.java
@@ -26,20 +26,16 @@ import java.util.Objects;
import java.util.Queue;
import java.util.concurrent.Callable;
import java.util.function.BiConsumer;
-import java.util.function.BooleanSupplier;
import java.util.function.Consumer;
import java.util.function.Function;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import com.google.common.collect.Iterables;
import accord.api.Agent;
import accord.api.DataStore;
import accord.api.LocalListeners;
import accord.api.ProgressLog;
+import accord.api.RoutingKey;
import accord.impl.InMemoryCommandStore;
import accord.impl.InMemoryCommandStores;
import accord.impl.InMemorySafeCommand;
@@ -74,46 +70,12 @@ import static accord.utils.Invariants.ParanoiaCostFactor.HIGH;
public class DelayedCommandStores extends InMemoryCommandStores.SingleThread
{
- private static final Logger logger = LoggerFactory.getLogger(DelayedCommandStores.class);
-
- //TODO (correctness): remove once we have a Journal integration that does not change the values of a command...
- // There are known issues due to Journal, so exclude known problamtic fields
- private static class ValidationHack
- {
- private static Pattern field(String path)
- {
- path = path.replace(".", "\\.");
- path += ".*";
- return Pattern.compile(path);
- }
-
- private static final List<Pattern> KNOWN_ISSUES = List.of(
- // when a new epoch is detected and the execute ranges have more than the coordinating ranges,
- // and the coordinating ranges doesn't include the home key... we drop the query...
- // The logic to stitch messages together is not able to handle this as it doesn't know the original Topologies
- // TODO (required): test int if this is still true
-// field(".partialTxn.query."),
- // cmd.mutable().build() != cmd. This is due to Command.durability changing NotDurable to Local depending on the status
- field(".durability."));
- }
-
- private static boolean hasKnownIssue(String path)
- {
- for (Pattern p : ValidationHack.KNOWN_ISSUES)
- {
- Matcher m = p.matcher(path);
- if (m.find())
- return true;
- }
- return false;
- }
-
- private DelayedCommandStores(NodeCommandStoreService time, Agent agent, DataStore store, RandomSource random, ShardDistributor shardDistributor, ProgressLog.Factory progressLogFactory, LocalListeners.Factory listenersFactory, SimulatedDelayedExecutorService executorService, BooleanSupplier isLoadedCheck, Journal journal)
+ private DelayedCommandStores(NodeCommandStoreService time, Agent agent, DataStore store, RandomSource random, ShardDistributor shardDistributor, ProgressLog.Factory progressLogFactory, LocalListeners.Factory listenersFactory, SimulatedDelayedExecutorService executorService, CacheLoadingChance isLoadedCheck, Journal journal)
{
super(time, agent, store, random, shardDistributor, progressLogFactory, listenersFactory, DelayedCommandStore.factory(executorService, isLoadedCheck, journal));
}
- public static CommandStores.Factory factory(PendingQueue pending, BooleanSupplier isLoadedCheck, Journal journal)
+ public static CommandStores.Factory factory(PendingQueue pending, CacheLoadingChance isLoadedCheck, Journal journal)
{
return (time, agent, store, random, shardDistributor, progressLogFactory, listenersFactory) ->
new DelayedCommandStores(time, agent, store, random, shardDistributor, progressLogFactory, listenersFactory, new SimulatedDelayedExecutorService(pending, agent), isLoadedCheck, journal);
@@ -169,14 +131,14 @@ public class DelayedCommandStores extends InMemoryCommandStores.SingleThread
private final SimulatedDelayedExecutorService executor;
private final Queue<Task<?>> pending = new LinkedList<>();
- private final BooleanSupplier isLoadedCheck;
+ private final CacheLoadingChance cacheLoadingChance;
private final Journal journal;
- public DelayedCommandStore(int id, NodeCommandStoreService time, Agent agent, DataStore store, ProgressLog.Factory progressLogFactory, LocalListeners.Factory listenersFactory, EpochUpdateHolder epochUpdateHolder, SimulatedDelayedExecutorService executor, BooleanSupplier isLoadedCheck, Journal journal)
+ public DelayedCommandStore(int id, NodeCommandStoreService time, Agent agent, DataStore store, ProgressLog.Factory progressLogFactory, LocalListeners.Factory listenersFactory, EpochUpdateHolder epochUpdateHolder, SimulatedDelayedExecutorService executor, CacheLoadingChance cacheLoadingChance, Journal journal)
{
super(id, time, agent, store, progressLogFactory, listenersFactory, epochUpdateHolder);
this.executor = executor;
- this.isLoadedCheck = isLoadedCheck;
+ this.cacheLoadingChance = cacheLoadingChance;
this.journal = journal;
}
@@ -193,17 +155,16 @@ public class DelayedCommandStores extends InMemoryCommandStores.SingleThread
// Journal will not have result persisted. This part is here for test purposes and ensuring that we have strict object equality.
Command reconstructed = journal.reconstruct(id, current.txnId());
List<Difference<?>> diff = ReflectionUtils.recursiveEquals(current, reconstructed);
- List<String> filteredDiff = diff.stream().filter(d -> !DelayedCommandStores.hasKnownIssue(d.path)).map(Object::toString).collect(Collectors.toList());
- Invariants.checkState(filteredDiff.isEmpty(), "Commands did not match: expected %s, given %s, node %s, store %d, diff %s", current, reconstructed, node, id(), new LazyToString(() -> String.join("\n", filteredDiff)));
+ Invariants.checkState(diff.isEmpty(), "Commands did not match: expected %s, given %s, node %s, store %d, diff %s", current, reconstructed, node, id(), new LazyToString(() -> String.join("\n", Iterables.transform(diff, Object::toString))));
}
@Override
protected boolean canExposeUnloaded()
{
- return isLoadedCheck.getAsBoolean();
+ return !cacheLoadingChance.cacheEmpty();
}
- private static CommandStore.Factory factory(SimulatedDelayedExecutorService executor, BooleanSupplier isLoadedCheck, Journal journal)
+ private static CommandStore.Factory factory(SimulatedDelayedExecutorService executor, CacheLoadingChance isLoadedCheck, Journal journal)
{
return (id, node, agent, store, progressLogFactory, listenersFactory, rangesForEpoch) -> new DelayedCommandStore(id, node, agent, store, progressLogFactory, listenersFactory, rangesForEpoch, executor, isLoadedCheck, journal);
}
@@ -286,7 +247,7 @@ public class DelayedCommandStores extends InMemoryCommandStores.SingleThread
@Override
protected InMemorySafeStore createSafeStore(PreLoadContext context, RangesForEpoch ranges, Map<TxnId, InMemorySafeCommand> commands, Map<RoutableKey, InMemorySafeTimestampsForKey> timestampsForKey, Map<RoutableKey, InMemorySafeCommandsForKey> commandsForKeys)
{
- return new DelayedSafeStore(this, ranges, context, commands, timestampsForKey, commandsForKeys);
+ return new DelayedSafeStore(this, ranges, context, commands, timestampsForKey, commandsForKeys, cacheLoadingChance);
}
@Override
@@ -299,10 +260,18 @@ public class DelayedCommandStores extends InMemoryCommandStores.SingleThread
public static class DelayedSafeStore extends InMemoryCommandStore.InMemorySafeStore
{
private final DelayedCommandStore commandStore;
- public DelayedSafeStore(DelayedCommandStore commandStore, RangesForEpoch ranges, PreLoadContext context, Map<TxnId, InMemorySafeCommand> commands, Map<RoutableKey, InMemorySafeTimestampsForKey> timestampsForKey, Map<RoutableKey, InMemorySafeCommandsForKey> commandsForKey)
+ private final CacheLoadingChance cacheLoadingChance;
+ public DelayedSafeStore(DelayedCommandStore commandStore,
+ RangesForEpoch ranges,
+ PreLoadContext context,
+ Map<TxnId, InMemorySafeCommand> commands,
+ Map<RoutableKey, InMemorySafeTimestampsForKey> timestampsForKey,
+ Map<RoutableKey, InMemorySafeCommandsForKey> commandsForKey,
+ CacheLoadingChance cacheLoadingChance)
{
super(commandStore, ranges, context, commands, timestampsForKey, commandsForKey);
this.commandStore = commandStore;
+ this.cacheLoadingChance = cacheLoadingChance;
}
@Override
@@ -319,6 +288,42 @@ public class DelayedCommandStores extends InMemoryCommandStores.SingleThread
});
super.postExecute();
}
+
+ @Override
+ protected InMemoryCommandStore.InMemoryCommandStoreCaches tryGetCaches()
+ {
+ if (!cacheLoadingChance.cacheEmpty())
+ {
+ boolean cacheFull = cacheLoadingChance.cacheFull();
+ return commandStore.new InMemoryCommandStoreCaches() {
+ @Override
+ public InMemorySafeCommand acquireIfLoaded(TxnId txnId)
+ {
+ if (cacheFull || cacheLoadingChance.commandLoaded())
+ return super.acquireIfLoaded(txnId);
+ return null;
+ }
+
+ @Override
+ public InMemorySafeTimestampsForKey acquireTfkIfLoaded(RoutingKey key)
+ {
+ if (cacheFull || cacheLoadingChance.tfkLoaded())
+ return super.acquireTfkIfLoaded(key);
+ return null;
+ }
+
+ @Override
+ public InMemorySafeCommandsForKey acquireIfLoaded(RoutingKey key)
+ {
+ if (cacheFull || cacheLoadingChance.cfkLoaded())
+ return super.acquireIfLoaded(key);
+ return null;
+ }
+ };
+ }
+ return null;
+
+ }
}
public List<DelayedCommandStore> unsafeStores()
@@ -328,4 +333,13 @@ public class DelayedCommandStores extends InMemoryCommandStores.SingleThread
stores.add((DelayedCommandStore) holder.store);
return stores;
}
+
+ public interface CacheLoadingChance
+ {
+ boolean cacheEmpty();
+ boolean cacheFull();
+ boolean commandLoaded();
+ boolean cfkLoaded();
+ boolean tfkLoaded();
+ }
}
diff --git a/accord-core/src/test/java/accord/impl/basic/Journal.java b/accord-core/src/test/java/accord/impl/basic/Journal.java
index 66052112..ac75af76 100644
--- a/accord-core/src/test/java/accord/impl/basic/Journal.java
+++ b/accord-core/src/test/java/accord/impl/basic/Journal.java
@@ -18,9 +18,9 @@
package accord.impl.basic;
+import java.util.AbstractList;
import java.util.ArrayList;
import java.util.Comparator;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -40,19 +40,20 @@ import accord.local.CommandStore;
import accord.local.Commands;
import accord.local.CommonAttributes;
import accord.local.Node;
-import accord.primitives.Known;
-import accord.primitives.SaveStatus;
-import accord.primitives.Status;
import accord.local.StoreParticipants;
import accord.primitives.Ballot;
+import accord.primitives.Known;
import accord.primitives.PartialDeps;
import accord.primitives.PartialTxn;
+import accord.primitives.SaveStatus;
+import accord.primitives.Status;
import accord.primitives.Timestamp;
import accord.primitives.TxnId;
import accord.primitives.Writes;
import accord.utils.Invariants;
import org.agrona.collections.Long2ObjectHashMap;
+import static accord.primitives.SaveStatus.NotDefined;
import static accord.primitives.SaveStatus.Stable;
import static accord.primitives.Status.Invalidated;
import static accord.primitives.Status.Truncated;
@@ -76,13 +77,13 @@ public class Journal
int commandStoreId = e.getKey().intValue();
Map<TxnId, List<Diff>> localJournal = e.getValue();
CommandStore store = storeSupplier.apply(commandStoreId);
+ if (store == null)
+ continue;
- Map<TxnId, List<Diff>> updates = new HashMap<>();
- List<TxnId> removals = new ArrayList<>();
for (Map.Entry<TxnId, List<Diff>> e2 : localJournal.entrySet())
{
- TxnId txnId = e2.getKey();
List<Diff> diffs = e2.getValue();
+ if (diffs.isEmpty()) continue;
Command command = reconstruct(diffs, Reconstruct.Last).get(0);
if (command.status() == Truncated || command.status() == Invalidated)
continue; // Already truncated
@@ -94,55 +95,112 @@ public class Journal
case INVALIDATE:
case TRUNCATE_WITH_OUTCOME:
case TRUNCATE:
+ case ERASE:
command = Commands.purge(command, command.participants(), cleanup);
Invariants.checkState(command.saveStatus() != SaveStatus.Uninitialised);
- List<Diff> arr = new ArrayList<>();
- arr.add(diff(null, command));
- updates.put(txnId, arr);
+ Diff diff = diff(null, command);
+ e2.setValue(cleanup == Cleanup.ERASE ? new ErasedList(diff) : new TruncatedList(diff));
break;
- case ERASE:
- removals.add(txnId);
+
+ case EXPUNGE:
+ e2.setValue(new PurgedList());
break;
}
}
-
- for (TxnId removal : removals)
- localJournal.remove(removal);
- localJournal.putAll(updates);
}
}
- // TODO (required): this might be a good first approximation, but maybe we need to make a better distinction between
- // when we want to produce side-effects.
- private boolean loading = false;
-
public void reconstructAll(InMemoryCommandStore.Loader loader, int commandStoreId)
{
- Map<TxnId, List<Diff>> diffs = diffsPerCommandStore.get(commandStoreId);
-
// Nothing to do here, journal is empty for this command store
- if (diffs == null)
+ if (!diffsPerCommandStore.containsKey(commandStoreId))
return;
- loading = true;
- try
+ // copy to avoid concurrent modification when appending to journal
+ Map<TxnId, List<Diff>> diffs = new TreeMap<>(diffsPerCommandStore.get(commandStoreId));
+ for (Map.Entry<TxnId, List<Diff>> e : diffs.entrySet())
+ e.setValue(new ArrayList<>(e.getValue()));
+
+ List<Command> toApply = new ArrayList<>();
+ for (Map.Entry<TxnId, List<Diff>> e : diffs.entrySet())
{
- List<Command> toApply = new ArrayList<>();
- for (TxnId txnId : diffs.keySet())
- {
- Command command = reconstruct(commandStoreId, txnId);
- if (command.saveStatus().compareTo(Stable) >= 0 && !command.hasBeen(Truncated))
- toApply.add(command);
- loader.load(command);
- }
+ if (e.getValue().isEmpty()) continue;
+ Command command = reconstruct(commandStoreId, e.getKey(), e.getValue());
+ if (command.saveStatus().compareTo(Stable) >= 0 && !command.hasBeen(Truncated))
+ toApply.add(command);
+ loader.load(command);
+ }
+
+ toApply.sort(Comparator.comparing(Command::executeAt));
+ for (Command command : toApply)
+ loader.apply(command);
+ }
- toApply.sort(Comparator.comparing(Command::executeAt));
- for (Command command : toApply)
- loader.apply(command);
+ static class ErasedList extends AbstractList<Diff>
+ {
+ final Diff erased;
+
+ ErasedList(Diff erased)
+ {
+ Invariants.checkArgument(erased.saveStatus.value == SaveStatus.Erased);
+ this.erased = erased;
}
- finally
+
+ @Override
+ public Diff get(int index)
+ {
+ if (index != 0)
+ throw new IndexOutOfBoundsException();
+ return erased;
+ }
+
+ @Override
+ public int size()
{
- loading = false;
+ return 1;
+ }
+
+ @Override
+ public boolean add(Diff diff)
+ {
+ if (diff.saveStatus != null && diff.saveStatus.value == SaveStatus.Erased)
+ return false;
+ throw illegalState();
+ }
+ }
+
+ static class TruncatedList extends ArrayList<Diff>
+ {
+ TruncatedList(Diff truncated)
+ {
+ add(truncated);
+ }
+ }
+
+ static class PurgedList extends AbstractList<Diff>
+ {
+ PurgedList()
+ {
+ }
+
+ @Override
+ public Diff get(int index)
+ {
+ throw new IndexOutOfBoundsException();
+ }
+
+ @Override
+ public int size()
+ {
+ return 0;
+ }
+
+ @Override
+ public boolean add(Diff diff)
+ {
+ if (diff.saveStatus != null && diff.saveStatus.value == SaveStatus.Erased)
+ return false;
+ throw illegalState();
}
}
@@ -154,7 +212,11 @@ public class Journal
public Command reconstruct(int commandStoreId, TxnId txnId)
{
- List<Diff> diffs = this.diffsPerCommandStore.get(commandStoreId).get(txnId);
+ return reconstruct(commandStoreId, txnId, diffsPerCommandStore.get(commandStoreId).get(txnId));
+ }
+
+ public Command reconstruct(int commandStoreId, TxnId txnId, List<Diff> diffs)
+ {
return reconstruct(diffs, Reconstruct.Last).get(0);
}
@@ -167,7 +229,7 @@ public class Journal
TxnId txnId = null;
Timestamp executeAt = null;
Timestamp executesAtLeast = null;
- SaveStatus saveStatus = null;
+ SaveStatus saveStatus = NotDefined;
Status.Durability durability = Status.Durability.NotDurable;
Ballot acceptedOrCommitted = Ballot.ZERO;
@@ -185,7 +247,7 @@ public class Journal
{
Diff diff = diffs.get(i);
if (diff.txnId != null)
- txnId = diff.txnId.get();
+ txnId = diff.txnId;
if (diff.executeAt != null)
executeAt = diff.executeAt.get();
if (diff.executesAtLeast != null)
@@ -229,6 +291,7 @@ public class Journal
{
t.printStackTrace();
}
+
switch (saveStatus.known.outcome)
{
case Erased:
@@ -329,15 +392,9 @@ public class Journal
public void onExecute(int commandStoreId, Command before, Command after, boolean isPrimary)
{
- if (loading || (before == null && after == null))
+ if (before == null && after == null)
return;
- if (after.saveStatus() == SaveStatus.Erased)
- {
- diffsPerCommandStore.computeIfAbsent(commandStoreId, (k) -> new TreeMap<>())
- .remove(after.txnId());
- return;
- }
Diff diff = diff(before, after);
if (!isPrimary)
diff = diff.asNonPrimary();
@@ -352,7 +409,7 @@ public class Journal
private static class Diff
{
- public final NewValue<TxnId> txnId;
+ public final TxnId txnId;
public final NewValue<Timestamp> executeAt;
public final NewValue<Timestamp> executesAtLeast;
@@ -371,7 +428,7 @@ public class Journal
public final NewValue<Result> result; // temporarily here for sakes for reloads
- public Diff(NewValue<TxnId> txnId,
+ public Diff(TxnId txnId,
NewValue<Timestamp> executeAt,
NewValue<Timestamp> executesAtLeast,
NewValue<SaveStatus> saveStatus,
@@ -470,7 +527,7 @@ public class Journal
if (Objects.equals(before, after))
return null;
- Diff diff = new Diff(ifNotEqual(before, after, Command::txnId, false),
+ Diff diff = new Diff(after.txnId(),
ifNotEqual(before, after, Command::executeAt, true),
ifNotEqual(before, after, Command::executesAtLeast, true),
ifNotEqual(before, after, Command::saveStatus, false),
@@ -543,6 +600,21 @@ public class Journal
{
return "" + value;
}
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ NewValue<?> newValue = (NewValue<?>) o;
+ return Objects.equals(value, newValue.value);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ throw new UnsupportedOperationException();
+ }
}
}
\ No newline at end of file
diff --git a/accord-core/src/test/java/accord/impl/basic/NodeSink.java b/accord-core/src/test/java/accord/impl/basic/NodeSink.java
index bfa4b3c9..be929e7d 100644
--- a/accord-core/src/test/java/accord/impl/basic/NodeSink.java
+++ b/accord-core/src/test/java/accord/impl/basic/NodeSink.java
@@ -20,11 +20,9 @@ package accord.impl.basic;
import java.util.LinkedHashMap;
import java.util.Map;
+import java.util.concurrent.TimeUnit;
import java.util.function.Function;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import accord.api.MessageSink;
import accord.impl.basic.Cluster.Link;
import accord.local.AgentExecutor;
@@ -37,38 +35,42 @@ import accord.messages.Reply.FailureReply;
import accord.messages.ReplyContext;
import accord.messages.Request;
import accord.messages.SafeCallback;
-import accord.utils.RandomSource;
import static java.util.concurrent.TimeUnit.MICROSECONDS;
-import static java.util.concurrent.TimeUnit.MILLISECONDS;
public class NodeSink implements MessageSink
{
- private static final Logger logger = LoggerFactory.getLogger(NodeSink.class);
-
- private static final boolean DEBUG = false;
public enum Action { DELIVER, DROP, DELIVER_WITH_FAILURE, FAILURE }
+ public interface TimeoutSupplier
+ {
+ long slowAt();
+ long expiresAt();
+ long failsAt();
+ long now();
+ TimeUnit units();
+ }
+
final Id self;
final Function<Id, Node> lookup;
final Cluster parent;
- final RandomSource random;
+ final TimeoutSupplier timeouts;
int nextMessageId = 0;
Map<Long, SafeCallback> callbacks = new LinkedHashMap<>();
- public NodeSink(Id self, Function<Id, Node> lookup, Cluster parent, RandomSource random)
+ public NodeSink(Id self, Function<Id, Node> lookup, Cluster parent, TimeoutSupplier timeouts)
{
this.self = self;
this.lookup = lookup;
this.parent = parent;
- this.random = random;
+ this.timeouts = timeouts;
}
@Override
public void send(Id to, Request send)
{
- maybeEnqueue(to, nextMessageId++, send, null);
+ maybeEnqueue(to, nextMessageId++, timeouts.expiresAt(), send, null);
}
@Override
@@ -77,32 +79,38 @@ public class NodeSink implements MessageSink
long messageId = nextMessageId++;
SafeCallback sc = new SafeCallback(executor, callback);
callbacks.put(messageId, sc);
- if (maybeEnqueue(to, messageId, send, sc))
+ TimeUnit units = timeouts.units();
+ long now = timeouts.now();
+ long expiresAt = timeouts.expiresAt();
+ long slowAt = timeouts.slowAt();
+ if (maybeEnqueue(to, messageId, expiresAt, send, sc))
{
- parent.pending.add((PendingRunnable) () -> {
+ parent.pending.add(PendingRunnable.create(() -> {
if (sc == callbacks.get(messageId))
sc.slowResponse(to);
- }, 100 + random.nextInt(200), MILLISECONDS);
- parent.pending.add((PendingRunnable) () -> {
+ }), slowAt - now, units);
+ parent.pending.add(PendingRunnable.create(() -> {
if (sc == callbacks.remove(messageId))
sc.timeout(to);
- }, 1000 + random.nextInt(1000), MILLISECONDS);
+ }), expiresAt - now, units);
}
}
@Override
public void reply(Id replyToNode, ReplyContext replyContext, Reply reply)
{
- maybeEnqueue(replyToNode, Packet.getMessageId(replyContext), reply, null);
+ long expiresAt = Packet.getExpiresAt(replyContext);
+ if (expiresAt < 0) expiresAt = timeouts.expiresAt();
+ maybeEnqueue(replyToNode, Packet.getMessageId(replyContext), expiresAt, reply, null);
}
- private boolean maybeEnqueue(Node.Id to, long id, Message message, SafeCallback callback)
+ private boolean maybeEnqueue(Node.Id to, long id, long expiresAt, Message message, SafeCallback callback)
{
Link link = parent.links.apply(self, to);
if (to.equals(self) || lookup.apply(to) == null /* client */)
{
parent.messageListener.onMessage(Action.DELIVER, self, to, id, message);
- deliver(to, id, message, link);
+ deliver(to, id, expiresAt, message, link);
return true;
}
@@ -111,21 +119,21 @@ public class NodeSink implements MessageSink
switch (action)
{
case DELIVER:
- deliver(to, id, message, link);
+ deliver(to, id, expiresAt, message, link);
return true;
case DELIVER_WITH_FAILURE:
- deliver(to, id, message, link);
+ deliver(to, id, expiresAt, message, link);
case FAILURE:
if (action == Action.FAILURE)
parent.notifyDropped(self, to, id, message);
if (callback != null)
{
- parent.pending.add((PendingRunnable) () -> {
+ long failesAt = timeouts.failsAt();
+ parent.pending.add(PendingRunnable.create(() -> {
if (callback == callbacks.remove(id))
{
try
{
- // TODO (now): we MUST drop any ACTUAL reply now
callback.failure(to, new SimulatedFault("Simulation Failure; src=" + self + ", to=" + to + ", id=" + id + ", message=" + message));
}
catch (Throwable t)
@@ -134,11 +142,11 @@ public class NodeSink implements MessageSink
lookup.apply(self).agent().onUncaughtException(t);
}
}
- }, 1000 + random.nextInt(1000), MILLISECONDS);
+ }), failesAt - timeouts.now(), timeouts.units());
}
return false;
case DROP:
- // TODO (consistency): parent.notifyDropped is a trace logger that is very similar in spirit to MessageListener; can we unify?
+ // TODO (desired): parent.notifyDropped is a trace logger that is very similar in spirit to MessageListener; can we unify?
parent.notifyDropped(self, to, id, message);
return true;
default:
@@ -146,11 +154,11 @@ public class NodeSink implements MessageSink
}
}
- private void deliver(Node.Id to, long id, Message message, Link link)
+ private void deliver(Node.Id to, long id, long expiresAt, Message message, Link link)
{
Packet packet;
- if (message instanceof Reply) packet = new Packet(self, to, id, (Reply) message);
- else packet = new Packet(self, to, id, (Request) message);
+ if (message instanceof Reply) packet = new Packet(self, to, expiresAt, id, (Reply) message);
+ else packet = new Packet(self, to, expiresAt, id, (Request) message);
parent.add(packet, link.latencyMicros.getAsLong(), MICROSECONDS);
}
diff --git a/accord-core/src/test/java/accord/impl/basic/Packet.java b/accord-core/src/test/java/accord/impl/basic/Packet.java
index dc7fd567..5afe91f7 100644
--- a/accord-core/src/test/java/accord/impl/basic/Packet.java
+++ b/accord-core/src/test/java/accord/impl/basic/Packet.java
@@ -29,30 +29,40 @@ public class Packet implements Pending, ReplyContext
{
static final int SENTINEL_MESSAGE_ID = Integer.MIN_VALUE;
+ final Pending origin = Pending.Global.activeOrigin();
public final Id src;
public final Id dst;
+ public final long expiresAt;
public final long requestId; // if message is Reply, this is the id of the message we are replying to
public final long replyId; // if message is Reply, this is the id of the message we are replying to
public final Message message;
- public Packet(Id src, Id dst, long requestId, Request request)
+ public Packet(Id src, Id dst, long expiresAt, long requestId, Request request)
{
this.src = src;
this.dst = dst;
+ this.expiresAt = expiresAt;
this.requestId = requestId;
this.replyId = SENTINEL_MESSAGE_ID;
this.message = request;
}
- public Packet(Id src, Id dst, long replyId, Reply reply)
+ public Packet(Id src, Id dst, long expiresAt, long replyId, Reply reply)
{
this.src = src;
this.dst = dst;
+ this.expiresAt = expiresAt;
this.requestId = SENTINEL_MESSAGE_ID;
this.replyId = replyId;
this.message = reply;
}
+ @Override
+ public Pending origin()
+ {
+ return origin;
+ }
+
@Override
public String toString()
{
@@ -69,4 +79,11 @@ public class Packet implements Pending, ReplyContext
return ((Network.MessageId) context).msgId;
return ((Packet) context).requestId;
}
+
+ public static long getExpiresAt(ReplyContext context)
+ {
+ if (context instanceof Network.MessageId)
+ return -1;
+ return ((Packet) context).expiresAt;
+ }
}
diff --git a/accord-core/src/test/java/accord/impl/basic/Pending.java b/accord-core/src/test/java/accord/impl/basic/Pending.java
index 7aadb5c9..8acfd5b4 100644
--- a/accord-core/src/test/java/accord/impl/basic/Pending.java
+++ b/accord-core/src/test/java/accord/impl/basic/Pending.java
@@ -18,6 +18,28 @@
package accord.impl.basic;
+import accord.utils.Invariants;
+
public interface Pending
{
+ Pending origin();
+
+ // a slightly hacky but easy way to track transitive recurring tasks
+ class Global
+ {
+ private static Pending activeOrigin;
+ public static void setActiveOrigin(Pending newActive)
+ {
+ Invariants.checkState(activeOrigin == null);
+ activeOrigin = newActive.origin();
+ }
+ public static void clearActiveOrigin()
+ {
+ activeOrigin = null;
+ }
+ public static Pending activeOrigin()
+ {
+ return activeOrigin;
+ }
+ }
}
diff --git a/accord-core/src/test/java/accord/impl/basic/PendingRunnable.java b/accord-core/src/test/java/accord/impl/basic/PendingRunnable.java
index 0ef0c989..15a0eb7f 100644
--- a/accord-core/src/test/java/accord/impl/basic/PendingRunnable.java
+++ b/accord-core/src/test/java/accord/impl/basic/PendingRunnable.java
@@ -20,4 +20,22 @@ package accord.impl.basic;
public interface PendingRunnable extends Pending, Runnable
{
+ static PendingRunnable create(Runnable run)
+ {
+ Pending origin = Global.activeOrigin();
+ return new PendingRunnable()
+ {
+ @Override
+ public Pending origin()
+ {
+ return origin;
+ }
+
+ @Override
+ public void run()
+ {
+ run.run();
+ }
+ };
+ }
}
diff --git a/accord-core/src/test/java/accord/impl/basic/RandomDelayQueue.java b/accord-core/src/test/java/accord/impl/basic/RandomDelayQueue.java
index a90f150a..3708c8cf 100644
--- a/accord-core/src/test/java/accord/impl/basic/RandomDelayQueue.java
+++ b/accord-core/src/test/java/accord/impl/basic/RandomDelayQueue.java
@@ -32,6 +32,7 @@ import accord.impl.basic.DelayedCommandStores.DelayedCommandStore.DelayedTask;
import accord.utils.DefaultRandom;
import accord.utils.RandomSource;
+import static accord.impl.basic.RecurringPendingRunnable.isRecurring;
import static accord.utils.Invariants.illegalArgument;
import static accord.utils.Invariants.illegalState;
@@ -169,7 +170,9 @@ public class RandomDelayQueue implements PendingQueue
for (Item item : items)
{
if (toDrain.test(item.item))
+ {
ret.add(item.item);
+ }
else
{
queue.add(item);
@@ -301,9 +304,4 @@ public class RandomDelayQueue implements PendingQueue
this.seed = seed;
}
}
-
- private static boolean isRecurring(Pending pending)
- {
- return pending instanceof RecurringPendingRunnable && ((RecurringPendingRunnable) pending).isRecurring;
- }
}
diff --git a/accord-core/src/test/java/accord/impl/basic/RecurringPendingRunnable.java b/accord-core/src/test/java/accord/impl/basic/RecurringPendingRunnable.java
index 6dbfd754..257f6387 100644
--- a/accord-core/src/test/java/accord/impl/basic/RecurringPendingRunnable.java
+++ b/accord-core/src/test/java/accord/impl/basic/RecurringPendingRunnable.java
@@ -25,6 +25,7 @@ import accord.api.Scheduler.Scheduled;
class RecurringPendingRunnable implements PendingRunnable, Scheduled
{
+ final int source;
final PendingQueue requeue;
final LongSupplier delay;
final TimeUnit units;
@@ -32,8 +33,9 @@ class RecurringPendingRunnable implements PendingRunnable, Scheduled
Runnable run;
Runnable onCancellation;
- RecurringPendingRunnable(PendingQueue requeue, Runnable run, LongSupplier delay, TimeUnit units, boolean isRecurring)
+ RecurringPendingRunnable(int source, PendingQueue requeue, Runnable run, LongSupplier delay, TimeUnit units, boolean isRecurring)
{
+ this.source = source;
this.requeue = requeue;
this.run = run;
this.delay = delay;
@@ -41,6 +43,12 @@ class RecurringPendingRunnable implements PendingRunnable, Scheduled
this.isRecurring = isRecurring;
}
+ @Override
+ public Pending origin()
+ {
+ return this;
+ }
+
@Override
public void run()
{
@@ -87,4 +95,10 @@ class RecurringPendingRunnable implements PendingRunnable, Scheduled
return run + " with " + delay + " " + units + " delay";
}
+
+ public static boolean isRecurring(Pending pending)
+ {
+ Pending origin = pending.origin();
+ return origin instanceof RecurringPendingRunnable && ((RecurringPendingRunnable) origin).isRecurring;
+ }
}
diff --git a/accord-core/src/test/java/accord/impl/basic/TaskExecutorService.java b/accord-core/src/test/java/accord/impl/basic/TaskExecutorService.java
index 0e733095..286d34da 100644
--- a/accord-core/src/test/java/accord/impl/basic/TaskExecutorService.java
+++ b/accord-core/src/test/java/accord/impl/basic/TaskExecutorService.java
@@ -34,11 +34,17 @@ public abstract class TaskExecutorService extends AbstractExecutorService implem
{
public static class Task<T> extends AsyncResults.RunnableResult<T> implements Pending, RunnableFuture<T>
{
+ final Pending origin = Pending.Global.activeOrigin();
public Task(Callable<T> fn)
{
super(fn);
}
+ public Pending origin()
+ {
+ return origin;
+ }
+
@Override
public boolean cancel(boolean mayInterruptIfRunning)
{
diff --git a/accord-core/src/test/java/accord/impl/list/ListAgent.java b/accord-core/src/test/java/accord/impl/list/ListAgent.java
index bb1b04c8..eb3e602d 100644
--- a/accord-core/src/test/java/accord/impl/list/ListAgent.java
+++ b/accord-core/src/test/java/accord/impl/list/ListAgent.java
@@ -22,10 +22,13 @@ import java.util.concurrent.TimeUnit;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
import java.util.function.IntSupplier;
+import java.util.function.LongSupplier;
import accord.api.Agent;
import accord.api.ProgressLog;
import accord.api.Result;
+import accord.coordinate.EpochTimeout;
+import accord.impl.basic.Packet;
import accord.impl.mock.Network;
import accord.local.Command;
import accord.local.Node;
@@ -55,8 +58,9 @@ public class ListAgent implements Agent
final IntSupplier coordinationDelays;
final IntSupplier progressDelays;
final IntSupplier timeoutDelays;
+ final LongSupplier nowExact;
- public ListAgent(RandomSource rnd, long timeout, Consumer<Throwable> onFailure, Consumer<Runnable> retryBootstrap, BiConsumer<Timestamp, Ranges> onStale, IntSupplier coordinationDelays, IntSupplier progressDelays, IntSupplier timeoutDelays)
+ public ListAgent(RandomSource rnd, long timeout, Consumer<Throwable> onFailure, Consumer<Runnable> retryBootstrap, BiConsumer<Timestamp, Ranges> onStale, IntSupplier coordinationDelays, IntSupplier progressDelays, IntSupplier timeoutDelays, LongSupplier nowExact)
{
this.rnd = rnd;
this.timeout = timeout;
@@ -66,6 +70,7 @@ public class ListAgent implements Agent
this.coordinationDelays = coordinationDelays;
this.progressDelays = progressDelays;
this.timeoutDelays = timeoutDelays;
+ this.nowExact = nowExact;
}
@Override
@@ -105,7 +110,8 @@ public class ListAgent implements Agent
@Override
public void onUncaughtException(Throwable t)
{
- onFailure.accept(t);
+ if (!(t instanceof EpochTimeout))
+ onFailure.accept(t);
}
@Override
@@ -152,7 +158,7 @@ public class ListAgent implements Agent
@Override
public long attemptCoordinationDelay(Node node, SafeCommandStore safeStore, TxnId txnId, TimeUnit units, int retryCount)
{
- // TODO (required): meta randomise
+ // TODO (testing): meta randomise
return units.convert(rnd.nextInt(100, 1000), MILLISECONDS);
}
@@ -172,8 +178,12 @@ public class ListAgent implements Agent
@Override
public long expiresAt(ReplyContext replyContext, TimeUnit unit)
{
- // TODO (required): implement this (add it to the replyContext)
- return -1;
+ long expiresAt = ((Packet)replyContext).expiresAt;
+ long now = nowExact.getAsLong();
+ expiresAt -= now;
+ expiresAt *= 1.8f - rnd.nextFloat();
+ expiresAt += now;
+ return unit.convert(expiresAt, MILLISECONDS);
}
diff --git a/accord-core/src/test/java/accord/impl/list/ListFetchCoordinator.java b/accord-core/src/test/java/accord/impl/list/ListFetchCoordinator.java
index c7ffc2c6..340b6077 100644
--- a/accord-core/src/test/java/accord/impl/list/ListFetchCoordinator.java
+++ b/accord-core/src/test/java/accord/impl/list/ListFetchCoordinator.java
@@ -65,7 +65,7 @@ public class ListFetchCoordinator extends AbstractFetchCoordinator
ListData listData = (ListData) data;
persisting.add(commandStore.execute(PreLoadContext.empty(), safeStore -> {
listData.forEach((key, value) -> listStore.data.merge(key, value, Timestamped::merge));
- }).flatMap(ignore -> listStore.snapshot(received, syncPoint.syncId)).addCallback((success, fail) -> {
+ }).flatMap(ignore -> listStore.snapshot(true, received, syncPoint.syncId)).addCallback((success, fail) -> {
if (fail == null) success(from, received);
else fail(from, received, fail);
}).beginAsResult());
diff --git a/accord-core/src/test/java/accord/impl/list/ListRequest.java b/accord-core/src/test/java/accord/impl/list/ListRequest.java
index 46ac4c2b..acb9270e 100644
--- a/accord-core/src/test/java/accord/impl/list/ListRequest.java
+++ b/accord-core/src/test/java/accord/impl/list/ListRequest.java
@@ -226,7 +226,15 @@ public class ListRequest implements Request
node.reply(client, replyContext, ListResult.lost(client, ((Packet)replyContext).requestId, txnId), null);
break;
case Other:
- node.reply(client, replyContext, ListResult.other(client, ((Packet)replyContext).requestId, txnId), null);
+ if (attempt < 100)
+ {
+ node.reply(client, replyContext, ListResult.heartBeat(client, ((Packet)replyContext).requestId, txnId), null);
+ node.scheduler().once(() -> checkOnResult(finalHomeKey, txnId, attempt + 1, null), 5L, TimeUnit.MINUTES);
+ }
+ else
+ {
+ node.reply(client, replyContext, ListResult.other(client, ((Packet)replyContext).requestId, txnId), null);
+ }
break;
default:
node.agent().onUncaughtException(new AssertionError("Unknown outcome: " + s));
diff --git a/accord-core/src/test/java/accord/impl/list/ListResult.java b/accord-core/src/test/java/accord/impl/list/ListResult.java
index c9da1719..adfa1fe3 100644
--- a/accord-core/src/test/java/accord/impl/list/ListResult.java
+++ b/accord-core/src/test/java/accord/impl/list/ListResult.java
@@ -155,7 +155,7 @@ public class ListResult implements Result, Reply
+ "requestId:" + requestId + ", "
+ "txnId:" + txnId + ", "
+ (responseKeys == null
- ? (read == null ? "invalidated!}" : read.length == 0 ? "unknown}" : "lost}")
+ ? status.toString() + '}'
: "reads:" + IntStream.range(0, responseKeys.size())
.mapToObj(i -> responseKeys.get(i) + ":" + Arrays.toString(read[i]))
.collect(Collectors.joining(", ", "{", "}")) + ", "
diff --git a/accord-core/src/test/java/accord/impl/list/ListStore.java b/accord-core/src/test/java/accord/impl/list/ListStore.java
index 3671e3b6..022b74f4 100644
--- a/accord-core/src/test/java/accord/impl/list/ListStore.java
+++ b/accord-core/src/test/java/accord/impl/list/ListStore.java
@@ -22,6 +22,7 @@ import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Deque;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
@@ -179,27 +180,44 @@ public class ListStore implements DataStore
private static final class PendingSnapshot
{
+ // we don't restart bootstraps after clearing journal state and replaying, so must finish snapshots
+ final boolean runBeforeRestart;
final long delay;
final Consumer<Boolean> onCompletion;
- private PendingSnapshot(long delay, Consumer<Boolean> onCompletion)
+ private PendingSnapshot(boolean runBeforeRestart, long delay, Consumer<Boolean> onCompletion)
{
+ this.runBeforeRestart = runBeforeRestart;
this.delay = delay;
this.onCompletion = onCompletion;
}
}
private Snapshot snapshot;
+ private Scheduler.Scheduled scheduled;
private final Deque<PendingSnapshot> pendingSnapshots = new ArrayDeque<>();
private long pendingDelay = 0;
+ @Override
public AsyncResult<Void> snapshot(Ranges ranges, TxnId before)
+ {
+ return snapshot(false, ranges, before);
+ }
+
+ public AsyncResult<Void> snapshot(boolean runBeforeRestart, Ranges ranges, TxnId before)
{
Snapshot snapshot = new Snapshot(data, addedAts, removedAts, purgedAts, fetchCompletes, pendingRemoves);
AsyncResult.Settable<Void> result = new AsyncResults.SettableResult<>();
long delay = Math.max(1, random.nextBiasedLong(100, 1000, 5000) - pendingDelay);
pendingDelay += delay;
- pendingSnapshots.add(new PendingSnapshot(delay, success -> {
+
+ if (scheduled != null && runBeforeRestart && !pendingSnapshots.stream().anyMatch(p -> p.runBeforeRestart))
+ {
+ scheduled.cancel();
+ scheduled = null;
+ }
+
+ pendingSnapshots.add(new PendingSnapshot(runBeforeRestart, delay, success -> {
if (success)
{
this.snapshot = snapshot;
@@ -211,7 +229,7 @@ public class ListStore implements DataStore
}
}));
- if (pendingSnapshots.size() == 1)
+ if (scheduled == null)
scheduleRunSnapshot();
return result;
}
@@ -219,7 +237,9 @@ public class ListStore implements DataStore
private void scheduleRunSnapshot()
{
Invariants.checkState(!pendingSnapshots.isEmpty());
- scheduler.once(() -> {
+ // schedule as recurring so that we don't run them
+ Runnable run = () -> {
+ scheduled = null;
if (pendingSnapshots.isEmpty())
return;
@@ -228,7 +248,10 @@ public class ListStore implements DataStore
pendingDelay -= pendingSnapshot.delay;
if (!pendingSnapshots.isEmpty())
scheduleRunSnapshot();
- }, pendingSnapshots.peekFirst().delay, TimeUnit.MILLISECONDS);
+ };
+
+ if (pendingSnapshots.stream().anyMatch(p -> p.runBeforeRestart)) scheduled = scheduler.once(run, pendingSnapshots.peekFirst().delay, TimeUnit.MILLISECONDS);
+ else scheduled = scheduler.selfRecurring(run, pendingSnapshots.peekFirst().delay, TimeUnit.MILLISECONDS);
}
public void restoreFromSnapshot()
@@ -297,10 +320,16 @@ public class ListStore implements DataStore
private void checkAccess(Timestamp executeAt, Key key)
{
if (!allowed.contains(key))
- throw new IllegalStateException(String.format("Attempted to access key %s on node %s, which is not in the range %s;\nexecuteAt = %s\n%s",
- key, node, allowed,
- executeAt,
- history(key)));
+ {
+ // TODO (expected): improve this validation logic
+ // but in the meantime, whitelist valid things, e.g. ephemeral reads can access data that has been retired
+ if (executeAt instanceof TxnId && ((TxnId) executeAt).awaitsOnlyDeps()
+ && removedAts.stream().anyMatch(r -> r.ranges.contains(key) && r.epoch > executeAt.epoch()))
+ return;
+
+ throw illegalState("Attempted to access key %s on node %s, which is not in the range %s;\nexecuteAt = %s\n%s",
+ key, node, allowed, executeAt, history(key));
+ }
}
private void checkAccess(Timestamp executeAt, Range range)
@@ -462,6 +491,34 @@ public class ListStore implements DataStore
return Timestamped.mergeEqual(a, b, Arrays::equals);
}
+ public NavigableMap<RoutableKey, Timestamped<int[]>> copyOfCurrentData()
+ {
+ return new TreeMap<>(data);
+ }
+
+ public boolean equals(NavigableMap<RoutableKey, Timestamped<int[]>> a)
+ {
+ return equal(a, data);
+ }
+
+ public static boolean equal(NavigableMap<RoutableKey, Timestamped<int[]>> a, NavigableMap<RoutableKey, Timestamped<int[]>> b)
+ {
+ Iterator<Map.Entry<RoutableKey, Timestamped<int[]>>> aiter = a.entrySet().iterator();
+ Iterator<Map.Entry<RoutableKey, Timestamped<int[]>>> biter = b.entrySet().iterator();
+ while (aiter.hasNext() && biter.hasNext())
+ {
+ Map.Entry<RoutableKey, Timestamped<int[]>> av = aiter.next();
+ Map.Entry<RoutableKey, Timestamped<int[]>> bv = biter.next();
+ if (!av.getKey().equals(bv.getKey()))
+ return false;
+ if (!av.getValue().equals(bv.getValue(), Arrays::equals))
+ return false;
+ }
+ if (aiter.hasNext() || biter.hasNext())
+ return false;
+ return true;
+ }
+
private static boolean isStrictPrefix(int[] a, int[] b)
{
if (a.length >= b.length)
@@ -531,7 +588,7 @@ public class ListStore implements DataStore
private void runWhenReady(Node node, long epoch, Runnable whenKnown)
{
if (node.topology().epoch() >= epoch) whenKnown.run();
- else node.scheduler().once(() -> runWhenReady(node, epoch, whenKnown), 10, TimeUnit.SECONDS);
+ else node.scheduler().selfRecurring(() -> runWhenReady(node, epoch, whenKnown), 10, TimeUnit.SECONDS);
}
private void removeLocalWhenReady(Node node, long epoch, Ranges removed)
@@ -567,16 +624,9 @@ public class ListStore implements DataStore
pendingRemoves.remove(epoch);
this.allowed = this.allowed.without(removed);
purgedAts.add(new PurgeAt(s, epoch, removed));
- // C* encodes keyspace/table within a Range, so Ranges being added/removed to the cluster are expected behaviors and not just local range movements. To better simulate that
- // this logic attempts to purge the data once we know its "safe" (no read/writes pending).
- // The assumption in the C* case is that we will be able to do similar logic when integrating with Transactional Cluster Metadata (TCM). The case there is that Epoch is owned by TCM
- // and C* will start applying changes outside the context of Accord (hence why this logic uses callbacks before Accord sees the topology), so there exists an assumption: we will be able
- // to delay local removal of data (drop keyspace/table) on accord txn state!
- for (Range range : removed)
- {
- NavigableMap<RoutableKey, Timestamped<int[]>> historicData = data.subMap(range.start(), range.startInclusive(), range.end(), range.endInclusive());
- historicData.clear();
- }
+ // C* encodes keyspace/table within a Range, so Ranges being added/removed to the cluster are expected behaviors and not just local range movements.
+ // however, there is no reason to erase old data as it being used should be detected as a violation due to being stale,
+ // and clearing data unnecessarily complicates journal replay
List<Runnable> callbacks = new ArrayList<>(onRemovalDone);
onRemovalDone.clear();
@@ -607,7 +657,7 @@ public class ListStore implements DataStore
private static AsyncChain<SyncPoint<Range>> awaitSyncPoint(Node node, SyncPoint<Range> exclusiveSyncPoint)
{
Await e = new Await(node, exclusiveSyncPoint);
- e.addCallback(() -> node.configService().reportEpochRedundant(exclusiveSyncPoint.route.toRanges(), exclusiveSyncPoint.syncId.epoch()));
+ e.addCallback(() -> node.configService().reportEpochRedundant(exclusiveSyncPoint.route.toRanges(), exclusiveSyncPoint.syncId.epoch() - 1));
e.start();
return e.recover(t -> {
if (t.getClass() == SyncPointErased.class)
diff --git a/accord-core/src/test/java/accord/impl/list/ListWrite.java b/accord-core/src/test/java/accord/impl/list/ListWrite.java
index aced2e04..79e0de5a 100644
--- a/accord-core/src/test/java/accord/impl/list/ListWrite.java
+++ b/accord-core/src/test/java/accord/impl/list/ListWrite.java
@@ -56,7 +56,7 @@ public class ListWrite extends TreeMap<Key, int[]> implements Write
ListStore s = (ListStore) store;
if (!containsKey(key))
return Writes.SUCCESS;
- TimestampsForKeys.updateLastExecutionTimestamps((AbstractSafeCommandStore<?, ?, ?>) safeStore, ((Key)key).toUnseekable(), txnId, executeAt, true);
+ TimestampsForKeys.updateLastExecutionTimestamps(safeStore, ((Key)key).toUnseekable(), txnId, executeAt, true);
logger.trace("submitting WRITE on {} at {} key:{}", s.node, executeAt, key);
return executor.apply(safeStore.commandStore()).submit(() -> {
@@ -73,7 +73,7 @@ public class ListWrite extends TreeMap<Key, int[]> implements Write
if (!containsKey(key))
return;
- TimestampsForKeys.updateLastExecutionTimestamps((AbstractSafeCommandStore<?, ?, ?>) safeStore, ((Key)key).toUnseekable(), txnId, executeAt, true);
+ TimestampsForKeys.updateLastExecutionTimestamps(safeStore, ((Key)key).toUnseekable(), txnId, executeAt, true);
logger.trace("unsafe applying WRITE on {} at {} key:{}", s.node, executeAt, key);
int[] data = get(key);
s.data.merge((Key)key, new Timestamped<>(executeAt, data, Arrays::toString), ListStore::merge);
diff --git a/accord-core/src/test/java/accord/impl/mock/MockCluster.java b/accord-core/src/test/java/accord/impl/mock/MockCluster.java
index c125f9e1..7604548b 100644
--- a/accord-core/src/test/java/accord/impl/mock/MockCluster.java
+++ b/accord-core/src/test/java/accord/impl/mock/MockCluster.java
@@ -107,7 +107,11 @@ public class MockCluster implements Network, AutoCloseable, Iterable<Node>
@Override
public void close()
{
- nodes.values().forEach(Node::shutdown);
+ nodes.values().forEach(n -> {
+ n.shutdown();
+ ((ThreadPoolScheduler)n.scheduler()).stop();
+ });
+
}
private synchronized Id nextNodeId()
diff --git a/accord-core/src/test/java/accord/local/CommandsTest.java b/accord-core/src/test/java/accord/local/CommandsTest.java
index 073b2f7b..d6488e66 100644
--- a/accord-core/src/test/java/accord/local/CommandsTest.java
+++ b/accord-core/src/test/java/accord/local/CommandsTest.java
@@ -79,11 +79,11 @@ class CommandsTest
Topology initialTopology = TopologyUtils.topology(1, nodes, Ranges.of(allRanges), rf);
Topology updatedTopology = TopologyUtils.topology(2, nodes, Ranges.of(prefix0), rf); // drop prefix1
- Cluster.run(rs::fork, nodes, initialTopology, nodeMap -> new Request()
+ Cluster.run(rs::fork, new int[] {0, 1}, nodes, initialTopology, nodeMap -> new Request()
{
{
// the node selected does not matter and should not impact determinism, they all share the same scheduler
- ((Cluster) nodeMap.values().stream().findFirst().get().scheduler()).onDone(() -> checkState(0, nodeMap.values()));
+ ((Cluster.ClusterScheduler) nodeMap.values().stream().findFirst().get().scheduler()).onDone(() -> checkState(0, nodeMap.values()));
}
@Override
@@ -142,6 +142,9 @@ class CommandsTest
if (cmd.value() == null) // empty isn't public
continue;
Command command = cmd.value();
+ if (command.txnId().isSyncPoint())
+ continue;
+
if (command.status() == Status.Invalidated || command.status() == Status.AcceptedInvalidate)
continue;
// current limitation of SimpleProgressLog is that only the home shard will attempt to recover, so
diff --git a/accord-core/src/test/java/accord/local/ImmutableCommandTest.java b/accord-core/src/test/java/accord/local/ImmutableCommandTest.java
index b5958276..58cb6cbb 100644
--- a/accord-core/src/test/java/accord/local/ImmutableCommandTest.java
+++ b/accord-core/src/test/java/accord/local/ImmutableCommandTest.java
@@ -118,7 +118,7 @@ public class ImmutableCommandTest
void noConflictWitnessTest()
{
CommandStoreSupport support = new CommandStoreSupport();
- InMemoryCommandStore commands = createStore(support);
+ InMemoryCommandStore.Synchronized commands = (InMemoryCommandStore.Synchronized) createStore(support);
MockCluster.Clock clock = new MockCluster.Clock(100);
TxnId txnId = clock.idForNode(1, 1);
Keys keys = Keys.of(KEY);
@@ -130,13 +130,23 @@ public class ImmutableCommandTest
Assertions.assertEquals(Status.NotDefined, command.status());
Assertions.assertNull(command.executeAt());
}
- SafeCommandStore safeStore = commands.beginOperation(PreLoadContext.contextFor(txnId, keys.toParticipants()));
- StoreParticipants participants = StoreParticipants.update(safeStore, ROUTE, txnId.epoch(), txnId, txnId.epoch());
- SafeCommand safeCommand = safeStore.get(txnId, participants);
- Commands.preaccept(safeStore, safeCommand, participants, txnId, txnId.epoch(), txn.slice(FULL_RANGES, true), ROUTE);
- Command command = safeStore.get(txnId).current();
- Assertions.assertEquals(Status.PreAccepted, command.status());
- Assertions.assertEquals(txnId, command.executeAt());
+
+ commands.execute(() -> {
+ SafeCommandStore safeStore = commands.beginOperation(PreLoadContext.contextFor(txnId, keys.toParticipants()));
+ try
+ {
+ StoreParticipants participants = StoreParticipants.update(safeStore, ROUTE, txnId.epoch(), txnId, txnId.epoch());
+ SafeCommand safeCommand = safeStore.get(txnId, participants);
+ Commands.preaccept(safeStore, safeCommand, participants, txnId, txnId.epoch(), txn.slice(FULL_RANGES, true), ROUTE);
+ Command command = safeStore.get(txnId).current();
+ Assertions.assertEquals(Status.PreAccepted, command.status());
+ Assertions.assertEquals(txnId, command.executeAt());
+ }
+ finally
+ {
+ commands.completeOperation(safeStore);
+ }
+ });
}
@Test
diff --git a/accord-core/src/test/java/accord/local/cfk/CommandsForKeyTest.java b/accord-core/src/test/java/accord/local/cfk/CommandsForKeyTest.java
index dfe37867..50b2b97c 100644
--- a/accord-core/src/test/java/accord/local/cfk/CommandsForKeyTest.java
+++ b/accord-core/src/test/java/accord/local/cfk/CommandsForKeyTest.java
@@ -719,7 +719,7 @@ public class CommandsForKeyTest
}
@Override
- protected SafeCommand ifLoadedAndInitialisedAndNotErasedInternal(TxnId txnId)
+ protected SafeCommand ifLoadedInternal(TxnId txnId)
{
return getInternal(txnId);
}
@@ -757,7 +757,7 @@ public class CommandsForKeyTest
}
@Override
- public SafeCommand ifLoadedAndInitialisedAndNotErased(TxnId txnId)
+ public SafeCommand ifLoadedAndInitialised(TxnId txnId)
{
if (txnId.compareTo(cfk.current.prunedBefore()) < 0)
return null;
diff --git a/accord-core/src/test/java/accord/messages/PreAcceptTest.java b/accord-core/src/test/java/accord/messages/PreAcceptTest.java
index c07e8252..fb5e5667 100644
--- a/accord-core/src/test/java/accord/messages/PreAcceptTest.java
+++ b/accord-core/src/test/java/accord/messages/PreAcceptTest.java
@@ -165,7 +165,7 @@ public class PreAcceptTest
invalidate.process(node, ID2, REPLY_CONTEXT);
messageSink.assertHistorySizes(0, 1);
- assertThat(messageSink.responses.get(0).payload).isEqualTo(new BeginInvalidation.InvalidateReply(null, Ballot.ZERO, SaveStatus.NotDefined, false, null, null, null));
+ assertThat(messageSink.responses.get(0).payload).isEqualTo(new BeginInvalidation.InvalidateReply(null, Ballot.ZERO, SaveStatus.Uninitialised, false, null, null, null));
messageSink.clearHistory();
PreAccept preAccept = preAccept(txnId, txn, key.toUnseekable());
diff --git a/accord-core/src/test/java/accord/messages/ReadDataTest.java b/accord-core/src/test/java/accord/messages/ReadDataTest.java
index d118bf8e..c72022ff 100644
--- a/accord-core/src/test/java/accord/messages/ReadDataTest.java
+++ b/accord-core/src/test/java/accord/messages/ReadDataTest.java
@@ -185,8 +185,9 @@ class ReadDataTest
store = stores.get(1);
check(store.execute(PreLoadContext.contextFor(state.txnId, state.route), safeStore -> {
StoreParticipants participants = StoreParticipants.invalidate(safeStore, state.route, state.txnId);
- SafeCommand command = safeStore.get(state.txnId, participants);
- command.commitInvalidated(safeStore);
+ SafeCommand safeCommand = safeStore.get(state.txnId, participants);
+ Command prev = safeCommand.current();
+ safeCommand.commitInvalidated(safeStore);
}));
ReplyContext replyContext = state.process();
diff --git a/accord-core/src/test/java/accord/primitives/AbstractRangesTest.java b/accord-core/src/test/java/accord/primitives/AbstractRangesTest.java
index bb243399..68965df7 100644
--- a/accord-core/src/test/java/accord/primitives/AbstractRangesTest.java
+++ b/accord-core/src/test/java/accord/primitives/AbstractRangesTest.java
@@ -127,6 +127,12 @@ class AbstractRangesTest
throw new UnsupportedOperationException();
}
+ @Override
+ public Routables<Range> without(accord.primitives.Ranges ranges)
+ {
+ return new Ranges(withoutInternal(ranges));
+ }
+
@Override
public Ranges intersecting(Seekables<?, ?> intersecting)
{
diff --git a/accord-core/src/test/java/accord/primitives/RangeDepsTest.java b/accord-core/src/test/java/accord/primitives/RangeDepsTest.java
index cf03e41c..dd3ca9f9 100644
--- a/accord-core/src/test/java/accord/primitives/RangeDepsTest.java
+++ b/accord-core/src/test/java/accord/primitives/RangeDepsTest.java
@@ -237,7 +237,7 @@ public class RangeDepsTest
}
- // TODO (now): broaden patterns of random contents, sizes of collection etc.
+ // TODO (testing): broaden patterns of random contents, sizes of collection etc.
@Test
public void testRandom()
{
@@ -296,7 +296,7 @@ public class RangeDepsTest
{
for (int txnIdCount = 1; txnIdCount < 100 ; txnIdCount *= 2)
{
- // TODO (now): verify that we write only a small number of checkpoints
+ // TODO (testing): verify that we write only a small number of checkpoints
generateNemesisRanges(width, txnIdCount, 1000, nonNemesisEntryPerNemesisEntry)
.validate(random);
}
diff --git a/accord-core/src/test/java/accord/topology/TopologyManagerTest.java b/accord-core/src/test/java/accord/topology/TopologyManagerTest.java
index aaa313e7..864b5f8f 100644
--- a/accord-core/src/test/java/accord/topology/TopologyManagerTest.java
+++ b/accord-core/src/test/java/accord/topology/TopologyManagerTest.java
@@ -28,6 +28,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Supplier;
+import java.util.stream.IntStream;
import com.google.common.collect.AbstractIterator;
import com.google.common.collect.Iterators;
@@ -217,14 +218,14 @@ public class TopologyManagerTest
Assertions.assertFalse(service.getEpochStateUnsafe(2).syncComplete());
RoutingKeys keys = keys(150).toParticipants();
- Assertions.assertEquals(topologies(topology3.forSelection(keys), topology2.forSelection(keys), topology1.forSelection(keys)),
+ Assertions.assertEquals(topologies(topology3.select(keys), topology2.select(keys), topology1.select(keys)),
service.withUnsyncedEpochs(keys, 3, 3));
service.onEpochSyncComplete(id(2), 2);
service.onEpochSyncComplete(id(3), 2);
service.onEpochSyncComplete(id(2), 3);
service.onEpochSyncComplete(id(3), 3);
- Assertions.assertEquals(topologies(topology3.forSelection(keys)),
+ Assertions.assertEquals(topologies(topology3.select(keys)),
service.withUnsyncedEpochs(keys, 3, 3));
}
@@ -392,7 +393,8 @@ public class TopologyManagerTest
AgentExecutor executor = Mockito.mock(AgentExecutor.class, Mockito.withSettings().defaultAnswer(ignore -> { throw new IllegalStateException("Attempted to perform async operation"); }));
Mockito.doReturn(new TestAgent.RethrowAgent()).when(executor).agent();
qt().withExamples(20).check(rs -> {
- TopologyRandomizer randomizer = new TopologyRandomizer(() -> rs, firstTopology.next(rs), new TopologyUpdates(ignore -> executor), null, TopologyRandomizer.Listeners.NOOP);
+ int[] prefixes = IntStream.generate(rs::nextInt).limit(10).toArray();
+ TopologyRandomizer randomizer = new TopologyRandomizer(() -> rs, prefixes, firstTopology.next(rs), new TopologyUpdates(ignore -> executor), null, TopologyRandomizer.Listeners.NOOP);
Iterator<Topology> next = Iterators.limit(new AbstractIterator<Topology>()
{
@Override
diff --git a/accord-core/src/test/java/accord/topology/TopologyRandomizer.java b/accord-core/src/test/java/accord/topology/TopologyRandomizer.java
index 135eb91b..ab24cf72 100644
--- a/accord-core/src/test/java/accord/topology/TopologyRandomizer.java
+++ b/accord-core/src/test/java/accord/topology/TopologyRandomizer.java
@@ -24,9 +24,10 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Queue;
import java.util.Set;
import java.util.TreeSet;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.function.Supplier;
@@ -55,8 +56,7 @@ import org.agrona.collections.IntHashSet;
import static accord.burn.BurnTest.HASH_RANGE_END;
import static accord.burn.BurnTest.HASH_RANGE_START;
-
-// TODO (required, testing): add change replication factor
+// TODO (testing): add change replication factor
public class TopologyRandomizer
{
public interface Listener
@@ -78,7 +78,7 @@ public class TopologyRandomizer
private static class State
{
- AtomicInteger currentPrefix;
+ Queue<Integer> newPrefixes;
Shard[] shards;
}
@@ -87,17 +87,16 @@ public class TopologyRandomizer
private static final Shard[] EMPTY_SHARDS = new Shard[0];
private final RandomSource random;
- private final AtomicInteger currentPrefix;
private final List<Topology> epochs = new ArrayList<>();
private final Function<Id, Node> nodeLookup;
+ private final ConcurrentLinkedQueue<Integer> newPrefixes = new ConcurrentLinkedQueue<>();
private final Map<Id, Ranges> previouslyReplicated = new HashMap<>();
private final TopologyUpdates topologyUpdates;
private final Listener listener;
- public TopologyRandomizer(Supplier<RandomSource> randomSupplier, Topology initialTopology, TopologyUpdates topologyUpdates, @Nullable Function<Id, Node> nodeLookup, Listener listener)
+ public TopologyRandomizer(Supplier<RandomSource> randomSupplier, int[] prefixes, Topology initialTopology, TopologyUpdates topologyUpdates, @Nullable Function<Id, Node> nodeLookup, Listener listener)
{
this.random = randomSupplier.get();
- this.currentPrefix = new AtomicInteger(random.nextInt(0, 1024));
this.topologyUpdates = topologyUpdates;
this.epochs.add(Topology.EMPTY);
this.epochs.add(initialTopology);
@@ -105,6 +104,8 @@ public class TopologyRandomizer
previouslyReplicated.put(node, initialTopology.rangesForNode(node));
this.nodeLookup = nodeLookup;
this.listener = listener;
+ for (int prefix : prefixes)
+ newPrefixes.add(prefix);
}
@VisibleForTesting
@@ -318,13 +319,13 @@ public class TopologyRandomizer
// Future work will add a new "removePrefix" method, that will cause prefixes to be dropped over time, when that happens the ABA problem
// could pop up (add prefix=0, drop prefix=0, add prefix=0) which is not the focus of this logic, so attempt to also generate a higher
// prefix than seen before.
- int[] prefixes = prefixes(shards);
- if (prefixes[prefixes.length - 1] > state.currentPrefix.get())
- state.currentPrefix.set(prefixes[prefixes.length - 1]);
// TODO (coverage): add support for bringing prefixes back after removal
// In implementations (such as Apache Cassandra) its possible that a range exists, gets removed, then added back (CREATE KEYSPACE, DROP KEYSPACE, CREATE KEYSPACE),
// in this case the old prefix should be "cleared".
- int prefix = state.currentPrefix.incrementAndGet();
+ Integer prefix = state.newPrefixes.poll();
+ if (prefix == null)
+ return shards;
+
Set<Id> joining = new HashSet<>();
Id[] nodes;
{
@@ -446,14 +447,14 @@ public class TopologyRandomizer
logger.debug("Updating topology with {} mutations", remainingMutations);
Shard[] newShards = oldShards;
State state = new State();
- state.currentPrefix = currentPrefix;
+ state.newPrefixes = newPrefixes;
while (remainingMutations > 0 && rejectedMutations < 10)
{
UpdateType type = UpdateType.kind(random);
state.shards = newShards;
Shard[] testShards = type.apply(state, random);
+ Arrays.sort(testShards, (a, b) -> a.range.compareTo(b.range));
if (!everyShardHasOverlaps(current.epoch, oldShards, testShards)
- // TODO (now): I don't think it is necessary to prevent re-replicating ranges any longer
|| reassignsRanges(current, testShards, previouslyReplicated)
)
{
diff --git a/accord-core/src/test/java/accord/topology/TopologyTest.java b/accord-core/src/test/java/accord/topology/TopologyTest.java
index 76eaf215..b87fc123 100644
--- a/accord-core/src/test/java/accord/topology/TopologyTest.java
+++ b/accord-core/src/test/java/accord/topology/TopologyTest.java
@@ -62,7 +62,7 @@ public class TopologyTest
Assertions.assertTrue(shard.range.contains(expectedKey));
Assertions.assertEquals(expectedRange, shard.range);
- Topology subTopology = topology.forSelection(Keys.of(expectedKey).toParticipants());
+ Topology subTopology = topology.select(Keys.of(expectedKey).toParticipants());
shard = Iterables.getOnlyElement(subTopology.shards());
Assertions.assertTrue(shard.range.contains(expectedKey));
Assertions.assertEquals(expectedRange, shard.range);
@@ -202,12 +202,12 @@ public class TopologyTest
}
for (Range range : topology.ranges())
{
- Topology subset = topology.forSelection(Ranges.single(range));
+ Topology subset = topology.select(Ranges.single(range));
for (int i = 0; i < 10; i++)
{
RoutingKey key = routingKey(range, rs);
- assertThat(topology.forSelection(RoutingKeys.of(key))).isEqualTo(subset);
+ assertThat(topology.select(RoutingKeys.of(key))).isEqualTo(subset);
assertThat(topology.forKey(key))
.describedAs("forKey(key) != get(indexForKey(key)) for key %s", key)
@@ -225,6 +225,6 @@ public class TopologyTest
.hasMessage("Range not found for %s", outsideRange);
}
}
- assertThat(topology.forSelection(topology.ranges())).isEqualTo(topology);
+ assertThat(topology.select(topology.ranges())).isEqualTo(topology);
}
}
diff --git a/accord-core/src/test/java/accord/utils/AccordGens.java b/accord-core/src/test/java/accord/utils/AccordGens.java
index eda147a8..560caa11 100644
--- a/accord-core/src/test/java/accord/utils/AccordGens.java
+++ b/accord-core/src/test/java/accord/utils/AccordGens.java
@@ -585,7 +585,6 @@ public class AccordGens
break;
case ExclusiveSyncPoint:
case SyncPoint:
- case LocalOnly:
//TODO (coverage, now):
keyDepsGen = i -> KeyDeps.NONE;
rangeDepsGen = i -> RangeDeps.NONE;
diff --git a/accord-core/src/test/java/accord/utils/LogGroupTimersTest.java b/accord-core/src/test/java/accord/utils/LogGroupTimersTest.java
index bb09b65b..8fbadf04 100644
--- a/accord-core/src/test/java/accord/utils/LogGroupTimersTest.java
+++ b/accord-core/src/test/java/accord/utils/LogGroupTimersTest.java
@@ -38,7 +38,7 @@ public class LogGroupTimersTest
public void test()
{
Random random = new Random();
- for (int i = 0 ; i < 10000 ; ++i)
+ for (int i = 0 ; i < 1000 ; ++i)
testOne(random.nextLong(), 1000, 100);
}
@@ -95,6 +95,7 @@ public class LogGroupTimersTest
private void testOne(long seed, int rounds, int maxBatchSize)
{
+ System.out.printf("Seed %d\n", seed);
RandomTestRunner.test().withSeed(seed).check(rnd -> {
TimeUnit scale = rnd.randomWeightedPicker(new TimeUnit[] { MINUTES, HOURS, DAYS }).get();
long globalMaxDeadline = scale.toMicros(rnd.nextInt(1, 8));
diff --git a/accord-core/src/test/java/accord/utils/SortedArraysTest.java b/accord-core/src/test/java/accord/utils/SortedArraysTest.java
index 98ce2f59..07f14e6a 100644
--- a/accord-core/src/test/java/accord/utils/SortedArraysTest.java
+++ b/accord-core/src/test/java/accord/utils/SortedArraysTest.java
@@ -237,24 +237,35 @@ class SortedArraysTest
@Test
public void testLinearSubtract()
{
- Gen<Integer[]> gen = sortedUniqueIntegerArray(0);
- qt().forAll(gen, gen).check((a, b) -> {
- Set<Integer> left = new HashSet<>(Arrays.asList(a));
- Set<Integer> right = new HashSet<>(Arrays.asList(b));
+ int[][] quants = {{0, 1000, 0, 1000},
+ {0, 100, 0, 1000},
+ {0, 100, 0, 10},
+ {0, 10, 0, 10},
+ {0, 5, 0, 5},
+ {0, 5, 0, 5}};
+
+ for (int[] quant : quants)
+ {
+ Gen<Integer[]> gen1 = sortedUniqueIntegerArray(quant[1], quant[0]);
+ Gen<Integer[]> gen2 = sortedUniqueIntegerArray(quant[3], quant[2]);
+ qt().forAll(gen1, gen2).withExamples(1000).check((a, b) -> {
+ Set<Integer> left = new HashSet<>(Arrays.asList(a));
+ Set<Integer> right = new HashSet<>(Arrays.asList(b));
- {
- Set<Integer> difference = Sets.difference(left, right);
- Integer[] expected = toArray(difference, Integer[]::new);
- Arrays.sort(expected);
- assertArrayEquals(expected, SortedArrays.linearSubtract(a, b, uncached(Integer[]::new)));
- }
- {
- Set<Integer> difference = Sets.difference(right, left);
- Integer[] expected = toArray(difference, Integer[]::new);
- Arrays.sort(expected);
- assertArrayEquals(expected, SortedArrays.linearSubtract(b, a, uncached(Integer[]::new)));
- }
- });
+ {
+ Set<Integer> difference = Sets.difference(left, right);
+ Integer[] expected = toArray(difference, Integer[]::new);
+ Arrays.sort(expected);
+ assertArrayEquals(expected, SortedArrays.linearSubtract(a, b, uncached(Integer[]::new)));
+ }
+ {
+ Set<Integer> difference = Sets.difference(right, left);
+ Integer[] expected = toArray(difference, Integer[]::new);
+ Arrays.sort(expected);
+ assertArrayEquals(expected, SortedArrays.linearSubtract(b, a, uncached(Integer[]::new)));
+ }
+ });
+ }
}
@Test
diff --git a/accord-core/src/test/java/accord/verify/StrictSerializabilityVerifier.java b/accord-core/src/test/java/accord/verify/StrictSerializabilityVerifier.java
index c0cb6b95..a39c566d 100644
--- a/accord-core/src/test/java/accord/verify/StrictSerializabilityVerifier.java
+++ b/accord-core/src/test/java/accord/verify/StrictSerializabilityVerifier.java
@@ -53,7 +53,7 @@ import static java.util.stream.Collectors.joining;
* find a path of predecessors that would witness us.
* <p>
* TODO (low priority): find and report a path when we encounter a violation
- * TODO (required): validate we only witness values we may have written (this is a post-processing step to just validate all steps have an associated write that may have succeeded)
+ * TODO (testing): validate we only witness values we may have written (this is a post-processing step to just validate all steps have an associated write that may have succeeded)
*/
public class StrictSerializabilityVerifier implements Verifier
{
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org