You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by if...@apache.org on 2024/09/30 15:01:35 UTC
(cassandra-accord) 02/09: Revisit eppch interval handling and inference of invalidations
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch ex-sp-pl
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git
commit 4eddc493a648da55fe36d837f5bbfa3f9c7636bf
Author: Benedict Elliott Smith <be...@apache.org>
AuthorDate: Fri Jul 26 15:57:37 2024 +0100
Revisit eppch interval handling and inference of invalidations
---
accord-core/src/main/java/accord/api/Agent.java | 15 +-
.../src/main/java/accord/api/LocalListeners.java | 2 +-
.../src/main/java/accord/api/ProgressLog.java | 8 +-
.../main/java/accord/api/ProtocolModifiers.java | 143 +++
.../src/main/java/accord/api/RemoteListeners.java | 4 +-
accord-core/src/main/java/accord/api/Write.java | 3 +-
.../coordinate/AbstractCoordinatePreAccept.java | 25 +-
.../java/accord/coordinate/AsynchronousAwait.java | 9 +-
.../src/main/java/accord/coordinate/Barrier.java | 110 ++-
.../main/java/accord/coordinate/CheckShards.java | 6 +-
.../accord/coordinate/CollectCalculatedDeps.java | 15 +-
.../accord/coordinate/CoordinateEphemeralRead.java | 28 +-
.../accord/coordinate/CoordinatePreAccept.java | 18 +-
.../accord/coordinate/CoordinateShardDurable.java | 28 +-
.../accord/coordinate/CoordinateSyncPoint.java | 83 +-
.../accord/coordinate/CoordinateTransaction.java | 12 +-
.../accord/coordinate/CoordinationAdapter.java | 224 +++--
.../accord/coordinate/ExecuteEphemeralRead.java | 2 +-
.../java/accord/coordinate/ExecuteSyncPoint.java | 104 ++-
.../main/java/accord/coordinate/ExecuteTxn.java | 2 +-
.../java/accord/coordinate/FetchCoordinator.java | 2 +
.../src/main/java/accord/coordinate/FetchData.java | 174 ++--
.../java/accord/coordinate/FetchMaxConflict.java | 23 +-
.../main/java/accord/coordinate/FindSomeRoute.java | 23 +-
.../src/main/java/accord/coordinate/Infer.java | 320 +------
.../main/java/accord/coordinate/Invalidate.java | 54 +-
.../main/java/accord/coordinate/MaybeRecover.java | 29 +-
.../src/main/java/accord/coordinate/Persist.java | 9 +-
...liseTxn.java => PersistExclusiveSyncPoint.java} | 25 +-
.../src/main/java/accord/coordinate/Preempted.java | 2 +-
.../src/main/java/accord/coordinate/Propose.java | 30 +-
.../java/accord/coordinate/ProposeSyncPoint.java | 16 +-
.../main/java/accord/coordinate/ProposeTxn.java | 12 +-
.../java/accord/coordinate/ReadCoordinator.java | 11 +-
.../src/main/java/accord/coordinate/Recover.java | 155 ++--
.../java/accord/coordinate/RecoverWithRoute.java | 26 +-
.../accord/coordinate/RecoverWithSomeRoute.java | 111 ---
.../coordinate/{Preempted.java => Redundant.java} | 19 +-
.../src/main/java/accord/coordinate/Stabilise.java | 13 +-
.../java/accord/coordinate/StabiliseSyncPoint.java | 12 +-
.../main/java/accord/coordinate/StabiliseTxn.java | 4 +-
.../coordinate/tracking/InvalidationTracker.java | 40 +-
.../coordinate/tracking/RecoveryTracker.java | 13 +-
.../java/accord/impl/AbstractFetchCoordinator.java | 4 +-
.../java/accord/impl/AbstractSafeCommandStore.java | 24 +-
.../impl/CoordinateDurabilityScheduling.java | 24 +-
.../java/accord/impl/DefaultLocalListeners.java | 17 +-
.../java/accord/impl/DefaultRemoteListeners.java | 4 +-
.../main/java/accord/impl/ErasedSafeCommand.java | 15 +-
.../java/accord/impl/InMemoryCommandStore.java | 258 +++---
.../accord/impl/InMemorySafeCommandsForKey.java | 4 +-
.../accord/impl/InMemorySafeTimestampsForKey.java | 4 +-
.../java/accord/impl/SafeTimestampsForKey.java | 8 +-
.../main/java/accord/impl/TimestampsForKey.java | 12 +-
.../main/java/accord/impl/TimestampsForKeys.java | 19 +-
.../java/accord/impl/progresslog/BaseTxnState.java | 2 +-
.../impl/progresslog/DefaultProgressLog.java | 90 +-
.../java/accord/impl/progresslog/HomeState.java | 3 +-
.../java/accord/impl/progresslog/WaitingState.java | 63 +-
.../src/main/java/accord/local/Bootstrap.java | 2 +-
.../src/main/java/accord/local/Cleanup.java | 133 ++-
.../src/main/java/accord/local/Command.java | 200 ++---
.../src/main/java/accord/local/CommandStore.java | 94 +-
.../src/main/java/accord/local/CommandStores.java | 94 +-
.../src/main/java/accord/local/Commands.java | 442 +++++----
.../main/java/accord/local/CommonAttributes.java | 69 +-
.../src/main/java/accord/local/DurableBefore.java | 8 +-
.../src/main/java/accord/local/MaxConflicts.java | 8 +-
accord-core/src/main/java/accord/local/Node.java | 16 +-
.../src/main/java/accord/local/PreLoadContext.java | 47 +-
.../main/java/accord/local/RedundantBefore.java | 226 +++--
.../main/java/accord/local/RedundantStatus.java | 20 +
.../src/main/java/accord/local/RejectBefore.java | 95 ++
.../src/main/java/accord/local/SafeCommand.java | 26 +-
.../main/java/accord/local/SafeCommandStore.java | 121 ++-
accord-core/src/main/java/accord/local/Status.java | 995 ---------------------
.../main/java/accord/local/StoreParticipants.java | 278 ++++++
.../main/java/accord/local/cfk/CommandsForKey.java | 898 ++++++++++++-------
.../src/main/java/accord/local/cfk/NotifySink.java | 42 +-
.../main/java/accord/local/cfk/PostProcess.java | 61 +-
.../src/main/java/accord/local/cfk/Pruning.java | 213 +++--
.../java/accord/local/cfk/SafeCommandsForKey.java | 16 +-
.../src/main/java/accord/local/cfk/Updating.java | 158 ++--
.../src/main/java/accord/local/cfk/Utils.java | 53 +-
.../src/main/java/accord/messages/Accept.java | 84 +-
.../src/main/java/accord/messages/Apply.java | 70 +-
.../accord/messages/ApplyThenWaitUntilApplied.java | 31 +-
.../src/main/java/accord/messages/Await.java | 31 +-
.../java/accord/messages/BeginInvalidation.java | 85 +-
.../main/java/accord/messages/BeginRecovery.java | 88 +-
.../main/java/accord/messages/CalculateDeps.java | 41 +-
.../src/main/java/accord/messages/CheckStatus.java | 486 +++-------
.../src/main/java/accord/messages/Commit.java | 56 +-
.../java/accord/messages/GetEphemeralReadDeps.java | 33 +-
.../main/java/accord/messages/GetMaxConflict.java | 23 +-
.../main/java/accord/messages/InformDurable.java | 10 +-
.../src/main/java/accord/messages/PreAccept.java | 95 +-
.../src/main/java/accord/messages/Propagate.java | 285 +++---
.../src/main/java/accord/messages/ReadData.java | 39 +-
.../java/accord/messages/ReadEphemeralTxnData.java | 9 +-
.../src/main/java/accord/messages/ReadTxnData.java | 2 +-
.../main/java/accord/messages/SetShardDurable.java | 6 +-
.../src/main/java/accord/messages/TxnRequest.java | 4 +-
.../java/accord/messages/WaitUntilApplied.java | 4 +-
.../java/accord/primitives/AbstractRanges.java | 5 +
.../accord/primitives/AbstractUnseekableKeys.java | 9 +-
.../src/main/java/accord/primitives/Deps.java | 19 +-
.../main/java/accord/primitives/EpochSupplier.java | 1 +
.../java/accord/primitives/FullRangeRoute.java | 6 +
.../src/main/java/accord/primitives/KeyDeps.java | 96 +-
.../src/main/java/accord/primitives/Keys.java | 4 +-
.../src/main/java/accord/primitives/Known.java | 743 +++++++++++++++
.../src/main/java/accord/primitives/KnownMap.java | 189 ++++
.../main/java/accord/primitives/LatestDeps.java | 21 +-
.../main/java/accord/primitives/PartialDeps.java | 26 +-
.../java/accord/primitives/PartialRangeRoute.java | 6 +
.../main/java/accord/primitives/PartialTxn.java | 10 +-
.../main/java/accord/primitives/Participants.java | 18 +-
.../main/java/accord/primitives/ProgressToken.java | 7 +-
.../src/main/java/accord/primitives/Range.java | 6 +
.../src/main/java/accord/primitives/RangeDeps.java | 25 +-
.../main/java/accord/primitives/RangeRoute.java | 7 +-
.../src/main/java/accord/primitives/Routables.java | 11 +-
.../main/java/accord/primitives/RoutingKeys.java | 15 +-
.../accord/{local => primitives}/SaveStatus.java | 70 +-
.../src/main/java/accord/primitives/Seekables.java | 1 -
.../src/main/java/accord/primitives/Status.java | 243 +++++
.../src/main/java/accord/primitives/SyncPoint.java | 42 +-
.../src/main/java/accord/primitives/Txn.java | 144 +--
.../src/main/java/accord/primitives/TxnId.java | 57 +-
.../main/java/accord/primitives/Unseekable.java | 3 +
.../main/java/accord/primitives/Unseekables.java | 10 +-
.../Faults.java => primitives/WithQuorum.java} | 9 +-
.../src/main/java/accord/primitives/Writes.java | 3 +-
.../src/main/java/accord/topology/Topologies.java | 135 ++-
.../src/main/java/accord/topology/Topology.java | 40 +-
.../main/java/accord/topology/TopologyManager.java | 159 +++-
.../src/main/java/accord/utils/Invariants.java | 2 +-
.../src/main/java/accord/utils/LogGroupTimers.java | 15 +-
.../java/accord/utils/ReducingIntervalMap.java | 4 +-
.../main/java/accord/utils/ReducingRangeMap.java | 2 +-
.../src/main/java/accord/utils/SortedArrays.java | 94 +-
.../src/main/java/accord/utils/SortedList.java | 11 +
.../src/main/java/accord/utils/SortedListMap.java | 257 ++++++
accord-core/src/test/java/accord/Utils.java | 14 +-
.../src/test/java/accord/burn/BurnTest.java | 7 +-
.../accord/coordinate/CoordinateSyncPointTest.java | 157 ++++
.../coordinate/CoordinateTransactionTest.java | 97 +-
.../test/java/accord/coordinate/RecoverTest.java | 11 +-
.../java/accord/coordinate/TopologyChangeTest.java | 15 +-
.../tracking/InvalidationTrackerReconciler.java | 11 +-
.../src/test/java/accord/impl/IntHashKey.java | 6 +
accord-core/src/test/java/accord/impl/IntKey.java | 6 +
.../test/java/accord/impl/LocalListenersTest.java | 33 +-
.../test/java/accord/impl/PrefixedIntHashKey.java | 12 +
.../test/java/accord/impl/RemoteListenersTest.java | 24 +-
.../src/test/java/accord/impl/TestAgent.java | 18 +-
.../src/test/java/accord/impl/basic/Cluster.java | 39 +-
.../src/test/java/accord/impl/basic/Journal.java | 69 +-
...endingQueue.java => MonitoredPendingQueue.java} | 55 +-
.../src/test/java/accord/impl/list/ListAgent.java | 8 +-
.../test/java/accord/impl/list/ListRequest.java | 18 +-
.../src/test/java/accord/impl/list/ListStore.java | 146 +--
.../src/test/java/accord/impl/list/ListWrite.java | 4 +-
.../src/test/java/accord/impl/mock/MockStore.java | 2 +-
.../test/java/accord/local/CheckedCommands.java | 28 +-
.../src/test/java/accord/local/CommandsTest.java | 1 +
.../java/accord/local/ImmutableCommandTest.java | 17 +-
.../src/test/java/accord/local/WaitingOnTest.java | 6 +-
.../java/accord/local/cfk/CommandsForKeyTest.java | 62 +-
.../test/java/accord/messages/PreAcceptTest.java | 28 +-
.../test/java/accord/messages/ReadDataTest.java | 37 +-
.../java/accord/primitives/AbstractRangesTest.java | 17 +-
.../src/test/java/accord/primitives/DepsTest.java | 8 +-
.../test/java/accord/primitives/KeyDepsTest.java | 105 +--
.../test/java/accord/topology/TopologyUtils.java | 4 +-
.../src/test/java/accord/utils/AccordGens.java | 18 +-
.../test/java/accord/utils/LogGroupTimersTest.java | 5 +-
.../test/java/accord/utils/SortedArraysTest.java | 5 +-
.../src/main/java/accord/maelstrom/Json.java | 16 +-
.../main/java/accord/maelstrom/MaelstromAgent.java | 6 +-
.../main/java/accord/maelstrom/MaelstromKey.java | 6 +
.../main/java/accord/maelstrom/MaelstromWrite.java | 3 +-
.../java/accord/maelstrom/TopologyFactory.java | 4 +-
.../src/test/java/accord/maelstrom/JsonTest.java | 5 +-
185 files changed, 6597 insertions(+), 5373 deletions(-)
diff --git a/accord-core/src/main/java/accord/api/Agent.java b/accord-core/src/main/java/accord/api/Agent.java
index 116c8e4a..2f42da1d 100644
--- a/accord-core/src/main/java/accord/api/Agent.java
+++ b/accord-core/src/main/java/accord/api/Agent.java
@@ -18,8 +18,6 @@
package accord.api;
-import javax.annotation.Nonnull;
-
import java.util.concurrent.TimeUnit;
import accord.api.ProgressLog.BlockedUntil;
@@ -28,7 +26,7 @@ import accord.local.Node;
import accord.local.SafeCommandStore;
import accord.messages.ReplyContext;
import accord.primitives.Ranges;
-import accord.primitives.Seekables;
+import accord.primitives.Routable;
import accord.primitives.Timestamp;
import accord.primitives.Txn;
import accord.primitives.TxnId;
@@ -62,15 +60,6 @@ public interface Agent extends UncaughtExceptionListener
void onFailedBootstrap(String phase, Ranges ranges, Runnable retry, Throwable failure);
- /**
- * Invoked with the keys (but not ranges) that have all dependent transactions in the applied
- * state at this node as of some TxnId. No guarantees are made about other nodes.
- *
- * Useful for migrations to/from Accord where you want to know there are no in flight
- * transactions in Accord that might still execute, and that it is safe to read
- * outside of Accord.
- */
- default void onLocalBarrier(@Nonnull Seekables<?, ?> keysOrRanges, @Nonnull TxnId txnId) {}
void onStale(Timestamp staleSince, Ranges ranges);
@Override
@@ -102,7 +91,7 @@ public interface Agent extends UncaughtExceptionListener
/**
* Create an empty transaction that Accord can use for its own internal transactions.
*/
- Txn emptySystemTxn(Txn.Kind kind, Seekables<?, ?> keysOrRanges);
+ Txn emptySystemTxn(Txn.Kind kind, Routable.Domain domain);
default EventsListener metricsEventsListener()
{
diff --git a/accord-core/src/main/java/accord/api/LocalListeners.java b/accord-core/src/main/java/accord/api/LocalListeners.java
index 861d5553..0ec09e71 100644
--- a/accord-core/src/main/java/accord/api/LocalListeners.java
+++ b/accord-core/src/main/java/accord/api/LocalListeners.java
@@ -22,7 +22,7 @@ import accord.local.Command;
import accord.local.CommandStore;
import accord.local.SafeCommand;
import accord.local.SafeCommandStore;
-import accord.local.SaveStatus;
+import accord.primitives.SaveStatus;
import accord.primitives.TxnId;
public interface LocalListeners
diff --git a/accord-core/src/main/java/accord/api/ProgressLog.java b/accord-core/src/main/java/accord/api/ProgressLog.java
index 6bdcefae..5dd1c2c0 100644
--- a/accord-core/src/main/java/accord/api/ProgressLog.java
+++ b/accord-core/src/main/java/accord/api/ProgressLog.java
@@ -25,8 +25,8 @@ import accord.local.CommandStore;
import accord.local.Node;
import accord.local.SafeCommand;
import accord.local.SafeCommandStore;
-import accord.local.SaveStatus;
-import accord.local.Status.Durability;
+import accord.primitives.SaveStatus;
+import accord.primitives.Status.Durability;
import accord.primitives.Participants;
import accord.primitives.Route;
import accord.primitives.Timestamp;
@@ -35,8 +35,8 @@ import accord.utils.SortedArrays;
import static accord.api.ProgressLog.BlockedUntil.Query.HOME;
import static accord.api.ProgressLog.BlockedUntil.Query.SHARD;
-import static accord.local.Status.Durability.Majority;
-import static accord.local.Status.Durability.NotDurable;
+import static accord.primitives.Status.Durability.Majority;
+import static accord.primitives.Status.Durability.NotDurable;
import static accord.utils.SortedArrays.Search.FAST;
/**
diff --git a/accord-core/src/main/java/accord/api/ProtocolModifiers.java b/accord-core/src/main/java/accord/api/ProtocolModifiers.java
new file mode 100644
index 00000000..b8b0bcb4
--- /dev/null
+++ b/accord-core/src/main/java/accord/api/ProtocolModifiers.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package accord.api;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import accord.utils.Invariants;
+
+import static accord.api.ProtocolModifiers.Faults.initialiseFaults;
+import static accord.api.ProtocolModifiers.QuorumEpochIntersections.ChaseFixedPoint.Chase;
+import static accord.api.ProtocolModifiers.QuorumEpochIntersections.ChaseFixedPoint.DoNotChase;
+import static accord.api.ProtocolModifiers.QuorumEpochIntersections.Include.Owned;
+import static accord.api.ProtocolModifiers.QuorumEpochIntersections.Include.Unsynced;
+import static accord.api.ProtocolModifiers.QuorumEpochIntersections.initialiseQuorumEpochIntersections;
+
+public class ProtocolModifiers
+{
+ public static final ProtocolModifiers.QuorumEpochIntersections QuorumEpochIntersections = initialiseQuorumEpochIntersections();
+ public static final Faults Faults = initialiseFaults();
+
+ public static class QuorumEpochIntersections
+ {
+ private static final Pattern PARSE = Pattern.compile("(preaccept|accept|commit|stable|recover)=([+-]{0,2})");
+
+ static ProtocolModifiers.QuorumEpochIntersections initialiseQuorumEpochIntersections()
+ {
+ String description = System.getProperty("accord.quorums.epochs", "preaccept=-,accept=-,commit=-,stable=,recover=-");
+ Matcher m = PARSE.matcher(description);
+ ChaseAndInclude preaccept = null;
+ Include accept = null;
+ Include commit = null;
+ Include stable = null;
+ Include recover = null;
+ while (m.find())
+ {
+ ChaseFixedPoint cfp = DoNotChase;
+ Include include = Owned;
+ String str = m.group(2);
+ for (int i = 0 ; i < str.length() ; ++i)
+ {
+ switch (str.charAt(i))
+ {
+ default: throw new AssertionError("Unexpected char: '" + str.charAt(i) + "'");
+ case '+': cfp = Chase; break;
+ case '-': include = Unsynced; break;
+ }
+ }
+
+ switch (m.group(1))
+ {
+ default: throw new AssertionError("Unexpected phase: " + m.group(1));
+ case "preaccept": preaccept = new ChaseAndInclude(cfp, include); break;
+ case "accept": accept = include; Invariants.checkState(cfp == DoNotChase, "Invalid to specify ChaseFixedPoint.Chase for accept"); break;
+ case "commit": commit = include; Invariants.checkState(cfp == DoNotChase, "Invalid to specify ChaseFixedPoint.Chase for commit"); break;
+ case "stable": stable = include; Invariants.checkState(cfp == DoNotChase, "Invalid to specify ChaseFixedPoint.Chase for stable"); break;
+ case "recover": recover = include; Invariants.checkState(cfp == DoNotChase, "Invalid to specify ChaseFixedPoint.Chase for stable"); break;
+ }
+ }
+
+ Invariants.checkState(preaccept != null, "preaccept not specified for quorum epoch intersections: " + description);
+ Invariants.checkState(accept != null, "accept not specified for quorum epoch intersections: " + description);
+ Invariants.checkState(commit != null, "commit not specified for quorum epoch intersections: " + description);
+ Invariants.checkState(stable != null, "stable not specified for quorum epoch intersections: " + description);
+ Invariants.checkState(recover != null, "recover not specified for quorum epoch intersections: " + description);
+ return new QuorumEpochIntersections(preaccept, accept, commit, stable, recover);
+ }
+
+ public enum ChaseFixedPoint
+ {
+ Chase, DoNotChase;
+ }
+ public enum Include
+ {
+ Unsynced, Owned;
+ }
+
+ public static class ChaseAndInclude
+ {
+ public final ChaseFixedPoint chase;
+ public final Include include;
+
+ public ChaseAndInclude(ChaseFixedPoint chase, Include include)
+ {
+ this.chase = chase;
+ this.include = include;
+ }
+ }
+
+ // TODO (desired): support fixed-point chasing for recovery
+ public final ChaseAndInclude preaccept;
+ public final Include accept, commit, stable, recover;
+ public final Include preacceptOrRecover, preacceptOrCommit;
+
+ public QuorumEpochIntersections(ChaseAndInclude preaccept, Include accept, Include commit, Include stable, Include recover)
+ {
+ this.preaccept = preaccept;
+ this.accept = accept;
+ this.commit = commit;
+ this.stable = stable;
+ this.recover = recover;
+ this.preacceptOrRecover = preaccept.include == Owned || recover == Owned ? Owned : Unsynced;
+ this.preacceptOrCommit = preaccept.include == Owned || commit == Owned ? Owned : Unsynced;
+ }
+ }
+
+ public static class Faults
+ {
+ static Faults initialiseFaults()
+ {
+ // TODO (expected): configurable
+ return new Faults(false, false, false, false);
+ }
+
+ public final boolean txnInstability, txnDiscardPreAcceptDeps;
+ public final boolean syncPointInstability, syncPointDiscardPreAcceptDeps;
+
+ public Faults(boolean txnInstability, boolean txnDiscardPreAcceptDeps, boolean syncPointInstability, boolean syncPointDiscardPreAcceptDeps)
+ {
+ this.txnInstability = txnInstability;
+ this.txnDiscardPreAcceptDeps = txnDiscardPreAcceptDeps;
+ this.syncPointInstability = syncPointInstability;
+ this.syncPointDiscardPreAcceptDeps = syncPointDiscardPreAcceptDeps;
+ }
+ }
+
+}
diff --git a/accord-core/src/main/java/accord/api/RemoteListeners.java b/accord-core/src/main/java/accord/api/RemoteListeners.java
index 06fdad13..c6b4e867 100644
--- a/accord-core/src/main/java/accord/api/RemoteListeners.java
+++ b/accord-core/src/main/java/accord/api/RemoteListeners.java
@@ -22,8 +22,8 @@ import accord.local.Command;
import accord.local.Node;
import accord.local.SafeCommand;
import accord.local.SafeCommandStore;
-import accord.local.SaveStatus;
-import accord.local.Status.Durability;
+import accord.primitives.SaveStatus;
+import accord.primitives.Status.Durability;
import accord.primitives.TxnId;
/**
diff --git a/accord-core/src/main/java/accord/api/Write.java b/accord-core/src/main/java/accord/api/Write.java
index 62635379..7f3fad7d 100644
--- a/accord-core/src/main/java/accord/api/Write.java
+++ b/accord-core/src/main/java/accord/api/Write.java
@@ -22,6 +22,7 @@ import accord.local.SafeCommandStore;
import accord.primitives.PartialTxn;
import accord.primitives.Seekable;
import accord.primitives.Timestamp;
+import accord.primitives.TxnId;
import accord.utils.async.AsyncChain;
/**
@@ -31,5 +32,5 @@ import accord.utils.async.AsyncChain;
*/
public interface Write
{
- AsyncChain<Void> apply(Seekable key, SafeCommandStore safeStore, Timestamp executeAt, DataStore store, PartialTxn txn);
+ AsyncChain<Void> apply(Seekable key, SafeCommandStore safeStore, TxnId txnId, Timestamp executeAt, DataStore store, PartialTxn txn);
}
diff --git a/accord-core/src/main/java/accord/coordinate/AbstractCoordinatePreAccept.java b/accord-core/src/main/java/accord/coordinate/AbstractCoordinatePreAccept.java
index e9ea6c54..64d065c4 100644
--- a/accord-core/src/main/java/accord/coordinate/AbstractCoordinatePreAccept.java
+++ b/accord-core/src/main/java/accord/coordinate/AbstractCoordinatePreAccept.java
@@ -29,12 +29,13 @@ import accord.local.Node;
import accord.local.Node.Id;
import accord.messages.Callback;
import accord.primitives.FullRoute;
-import accord.primitives.Seekables;
import accord.primitives.TxnId;
import accord.topology.Topologies;
import accord.utils.Invariants;
import accord.utils.async.AsyncResults.SettableResult;
+import static accord.api.ProtocolModifiers.QuorumEpochIntersections.ChaseFixedPoint.DoNotChase;
+import static accord.api.ProtocolModifiers.QuorumEpochIntersections;
import static accord.coordinate.tracking.RequestStatus.Failed;
import static accord.coordinate.tracking.RequestStatus.Success;
@@ -102,11 +103,11 @@ abstract class AbstractCoordinatePreAccept<T, R> extends SettableResult<T> imple
private Topologies topologies;
private boolean initialRoundIsDone;
private ExtraEpochs extraEpochs;
- private Map<Id, Object> debug = Invariants.debug() ? new TreeMap<>() : null;
+ private final Map<Id, Object> debug = Invariants.debug() ? new TreeMap<>() : null;
AbstractCoordinatePreAccept(Node node, FullRoute<?> route, TxnId txnId)
{
- this(node, route, txnId, node.topology().withUnsyncedEpochs(route, txnId, txnId));
+ this(node, route, txnId, node.topology().select(route, txnId, txnId, QuorumEpochIntersections.preaccept.include));
}
AbstractCoordinatePreAccept(Node node, FullRoute<?> route, @Nullable TxnId txnId, Topologies topologies)
@@ -122,7 +123,6 @@ abstract class AbstractCoordinatePreAccept<T, R> extends SettableResult<T> imple
contact(topologies.nodes(), topologies, this);
}
- abstract Seekables<?, ?> keysOrRanges();
abstract void contact(Collection<Id> nodes, Topologies topologies, Callback<R> callback);
abstract void onSuccessInternal(Id from, R reply);
/**
@@ -194,6 +194,14 @@ abstract class AbstractCoordinatePreAccept<T, R> extends SettableResult<T> imple
{
Invariants.checkState(!initialRoundIsDone || (extraEpochs != null && !extraEpochs.extraRoundIsDone));
initialRoundIsDone = true;
+ if (QuorumEpochIntersections.preaccept.chase == DoNotChase)
+ {
+ long latestEpoch = executeAtEpoch();
+ if (latestEpoch > topologies.currentEpoch()) node.withEpoch(latestEpoch, node.agent(), () -> onPreAcceptedInNewEpoch(topologies, latestEpoch));
+ else onPreAcceptedInNewEpoch(topologies, latestEpoch);
+ return;
+ }
+
if (extraEpochs != null)
extraEpochs.extraRoundIsDone = true;
@@ -207,6 +215,13 @@ abstract class AbstractCoordinatePreAccept<T, R> extends SettableResult<T> imple
onNewEpoch(topologies, latestEpoch);
}
+ final void onPreAcceptedInNewEpoch(Topologies topologies, long latestEpoch)
+ {
+ TopologyMismatch mismatch = TopologyMismatch.checkForMismatch(node.topology().globalForEpoch(latestEpoch), txnId, route.homeKey(), route);
+ if (mismatch == null) onPreAccepted(topologies);
+ else onNewEpochTopologyMismatch(mismatch);
+ }
+
final void onNewEpoch(Topologies prevTopologies, long latestEpoch)
{
// TODO (desired, efficiency): check if we have already have a valid quorum for the future epoch
@@ -217,7 +232,7 @@ abstract class AbstractCoordinatePreAccept<T, R> extends SettableResult<T> imple
tryFailure(CoordinationFailed.wrap(withEpochFailure));
return;
}
- TopologyMismatch mismatch = TopologyMismatch.checkForMismatch(node.topology().globalForEpoch(latestEpoch), txnId, route.homeKey(), keysOrRanges());
+ TopologyMismatch mismatch = TopologyMismatch.checkForMismatch(node.topology().globalForEpoch(latestEpoch), txnId, route.homeKey(), route);
if (mismatch != null)
{
initialRoundIsDone = true;
diff --git a/accord-core/src/main/java/accord/coordinate/AsynchronousAwait.java b/accord-core/src/main/java/accord/coordinate/AsynchronousAwait.java
index 4d40a4cd..6a2a87e1 100644
--- a/accord-core/src/main/java/accord/coordinate/AsynchronousAwait.java
+++ b/accord-core/src/main/java/accord/coordinate/AsynchronousAwait.java
@@ -30,10 +30,12 @@ import accord.local.Node.Id;
import accord.messages.Await;
import accord.messages.Await.AwaitOk;
import accord.messages.Callback;
+import accord.primitives.Participants;
import accord.primitives.Route;
import accord.primitives.TxnId;
import accord.primitives.Unseekables;
import accord.topology.Topologies;
+import accord.utils.Invariants;
import static accord.coordinate.tracking.RequestStatus.Success;
@@ -67,14 +69,14 @@ public class AsynchronousAwait implements Callback<AwaitOk>
}
final TxnId txnId;
- final Unseekables<?> contact;
+ final Participants<?> contact;
final AwaitTracker tracker;
final int asynchronousCallbackId;
final BiConsumer<SynchronousResult, Throwable> synchronousCallback;
private boolean isDone;
private Throwable failure;
- public AsynchronousAwait(TxnId txnId, Unseekables<?> contact, AwaitTracker tracker, int asynchronousCallbackId, BiConsumer<SynchronousResult, Throwable> synchronousCallback)
+ public AsynchronousAwait(TxnId txnId, Participants<?> contact, AwaitTracker tracker, int asynchronousCallbackId, BiConsumer<SynchronousResult, Throwable> synchronousCallback)
{
this.txnId = txnId;
this.contact = contact;
@@ -84,11 +86,12 @@ public class AsynchronousAwait implements Callback<AwaitOk>
}
/**
- * we require a Route to contact so we can be sure a home shard recipient invokes {@link Commands#updateRouteOrParticipants},
+ * we require a Route to contact so we can be sure a home shard recipient invokes {@link Commands#updateParticipants},
* notifying the progress log of a Route to determine it is the home shard.
*/
public static AsynchronousAwait awaitAny(Node node, Topologies topologies, TxnId txnId, Route<?> contact, BlockedUntil awaiting, int asynchronousCallbackId, BiConsumer<SynchronousResult, Throwable> synchronousCallback)
{
+ Invariants.checkArgument(topologies.size() == 1);
AwaitTracker tracker = new AwaitTracker(topologies);
AsynchronousAwait result = new AsynchronousAwait(txnId, contact, tracker, asynchronousCallbackId, synchronousCallback);
result.start(node, topologies, contact, awaiting);
diff --git a/accord-core/src/main/java/accord/coordinate/Barrier.java b/accord-core/src/main/java/accord/coordinate/Barrier.java
index bd21ea4a..aa06015d 100644
--- a/accord-core/src/main/java/accord/coordinate/Barrier.java
+++ b/accord-core/src/main/java/accord/coordinate/Barrier.java
@@ -25,7 +25,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import accord.api.BarrierType;
-import accord.api.Key;
import accord.api.LocalListeners;
import accord.api.RoutingKey;
import accord.local.Command;
@@ -35,14 +34,18 @@ import accord.local.SafeCommand;
import accord.local.SafeCommandStore;
import accord.local.SafeCommandStore.TestDep;
import accord.local.SafeCommandStore.TestStartedAt;
-import accord.local.Status;
+import accord.primitives.Seekables;
+import accord.primitives.Status;
+import accord.primitives.FullRoute;
import accord.primitives.Routable.Domain;
import accord.primitives.RoutableKey;
-import accord.primitives.Seekables;
import accord.primitives.SyncPoint;
import accord.primitives.Timestamp;
+import accord.primitives.Txn;
import accord.primitives.TxnId;
+import accord.utils.Invariants;
import accord.utils.MapReduceConsume;
+import accord.utils.TriFunction;
import accord.utils.async.AsyncResult;
import accord.utils.async.AsyncResults;
import javax.annotation.Nonnull;
@@ -62,33 +65,47 @@ import static accord.utils.Invariants.illegalState;
*
* Note that reads might still order after, but side effect bearing transactions should not.
*/
-public class Barrier<S extends Seekables<?, ?>> extends AsyncResults.AbstractResult<TxnId>
+public class Barrier extends AsyncResults.AbstractResult<TxnId>
{
@SuppressWarnings("unused")
private static final Logger logger = LoggerFactory.getLogger(Barrier.class);
private final Node node;
- private final S seekables;
+ private final FullRoute<?> route;
+ private final Seekables<?, ?> keysOrRanges;
private final long minEpoch;
private final BarrierType barrierType;
@VisibleForTesting
- AsyncResult<SyncPoint<S>> coordinateSyncPoint;
+ AsyncResult<? extends SyncPoint<?>> coordinateSyncPoint;
@VisibleForTesting
ExistingTransactionCheck existingTransactionCheck;
- private final BiFunction<Node, S, AsyncResult<SyncPoint<S>>> syncPoint;
+ public static class AsyncSyncPoint
+ {
+ final TxnId txnId;
+ final AsyncResult<? extends SyncPoint<?>> async;
+
+ public AsyncSyncPoint(TxnId txnId, AsyncResult<? extends SyncPoint<?>> async)
+ {
+ this.txnId = txnId;
+ this.async = async;
+ }
+ }
+
+ private final BiFunction<Node, FullRoute<?>, AsyncSyncPoint> syncPoint;
- Barrier(Node node, S keysOrRanges, long minEpoch, BarrierType barrierType, BiFunction<Node, S, AsyncResult<SyncPoint<S>>> syncPoint)
+ Barrier(Node node, Seekables<?, ?> keysOrRanges, FullRoute<?> route, long minEpoch, BarrierType barrierType, BiFunction<Node, FullRoute<?>, AsyncSyncPoint> syncPoint)
{
this.syncPoint = syncPoint;
- checkArgument(keysOrRanges.domain() == Domain.Key || barrierType.global, "Ranges are only supported with global barriers");
- checkArgument(keysOrRanges.size() == 1 || barrierType.global, "Only a single key is supported with local barriers");
+ this.keysOrRanges = keysOrRanges;
+ checkArgument(route.domain() == Domain.Key || barrierType.global, "Ranges are only supported with global barriers");
+ checkArgument(route.size() == 1 || barrierType.global, "Only a single key is supported with local barriers");
this.node = node;
this.minEpoch = minEpoch;
- this.seekables = keysOrRanges;
+ this.route = route;
this.barrierType = barrierType;
}
@@ -108,9 +125,9 @@ public class Barrier<S extends Seekables<?, ?>> extends AsyncResults.AbstractRes
*
* Returns the Timestamp the barrier actually ended up occurring at. Keep in mind for local barriers it doesn't mean a new transaction was created.
*/
- public static <S extends Seekables<?, ?>> Barrier<S> barrier(Node node, S keysOrRanges, long minEpoch, BarrierType barrierType, BiFunction<Node, S, AsyncResult<SyncPoint<S>>> syncPoint)
+ public static Barrier barrier(Node node, Seekables<?, ?> keysOrRanges, FullRoute<?> route, long minEpoch, BarrierType barrierType, BiFunction<Node, FullRoute<?>, AsyncSyncPoint> syncPoint)
{
- Barrier<S> barrier = new Barrier(node, keysOrRanges, minEpoch, barrierType, syncPoint);
+ Barrier barrier = new Barrier(node, keysOrRanges, route, minEpoch, barrierType, syncPoint);
node.topology().awaitEpoch(minEpoch).begin((ignored, failure) -> {
if (failure != null)
{
@@ -122,9 +139,17 @@ public class Barrier<S extends Seekables<?, ?>> extends AsyncResults.AbstractRes
return barrier;
}
- public static <S extends Seekables<?, ?>> Barrier<S> barrier(Node node, S keysOrRanges, long minEpoch, BarrierType barrierType)
+ public static Barrier barrier(Node node, Seekables<?, ?> keysOrRanges, FullRoute route, long minEpoch, BarrierType barrierType)
+ {
+ return barrier(node, keysOrRanges, route, minEpoch, barrierType, wrap(barrierType.async ? CoordinateSyncPoint::inclusive : CoordinateSyncPoint::inclusiveAndAwaitQuorum));
+ }
+
+ private static BiFunction<Node, FullRoute<?>, AsyncSyncPoint> wrap(TriFunction<Node, TxnId, FullRoute<?>, AsyncResult<? extends SyncPoint<?>>> syncPointFactory)
{
- return barrier(node, keysOrRanges, minEpoch, barrierType, barrierType.async ? CoordinateSyncPoint::inclusive : CoordinateSyncPoint::inclusiveAndAwaitQuorum);
+ return (node, route) -> {
+ TxnId txnId = node.nextTxnId(Txn.Kind.SyncPoint, route.domain());
+ return new AsyncSyncPoint(txnId, syncPointFactory.apply(node, txnId, route));
+ };
}
private void start()
@@ -153,20 +178,20 @@ public class Barrier<S extends Seekables<?, ?>> extends AsyncResults.AbstractRes
}
}
- private void doBarrierSuccess(TxnId syncId)
- {
- // The transaction we wait on might have more keys, but it's not
- // guaranteed they were wanted and we don't want to force the agent to filter them
- // so provide the seekables we were given.
- // We also don't notify the agent for range barriers since that makes sense as a local concept
- // since ranges can easily span multiple nodes
- node.agent().onLocalBarrier(seekables, syncId);
- Barrier.this.trySuccess(syncId);
- }
-
private void createSyncPoint()
{
- coordinateSyncPoint = syncPoint.apply(node, seekables);
+ AsyncSyncPoint async = syncPoint.apply(node, route);
+ coordinateSyncPoint = async.async;
+ if (!barrierType.global)
+ {
+ Invariants.checkState(barrierType.async);
+ TxnId txnId = async.txnId;
+ long epoch = txnId.epoch();
+ RoutingKey homeKey = route.homeKey();
+ node.commandStores().ifLocal(contextFor(txnId), homeKey, epoch, epoch, safeStore -> register(safeStore, txnId, homeKey))
+ .begin(node.agent());
+ }
+
coordinateSyncPoint.addCallback((syncPoint, syncPointFailure) -> {
if (syncPointFailure != null)
{
@@ -174,20 +199,8 @@ public class Barrier<S extends Seekables<?, ?>> extends AsyncResults.AbstractRes
return;
}
- // Need to wait for the local transaction to finish since coordinate sync point won't wait on anything
- // if async was requested or there were no deps found
- if (barrierType.async)
- {
- TxnId txnId = syncPoint.syncId;
- long epoch = txnId.epoch();
- RoutingKey homeKey = syncPoint.homeKey;
- node.commandStores().ifLocal(contextFor(txnId), homeKey, epoch, epoch, safeStore -> register(safeStore, txnId, homeKey))
- .begin(node.agent());
- }
- else
- {
- doBarrierSuccess(syncPoint.syncId);
- }
+ if (!barrierType.async)
+ Barrier.this.trySuccess(syncPoint.syncId);
});
}
@@ -201,7 +214,7 @@ public class Barrier<S extends Seekables<?, ?>> extends AsyncResults.AbstractRes
{
// In all the cases where we add a listener (listening to existing command, async completion of CoordinateSyncPoint)
// we want to notify the agent
- doBarrierSuccess(command.txnId());
+ Barrier.this.trySuccess(command.txnId());
return false;
}
else if (command.hasBeen(Status.Truncated))
@@ -216,9 +229,9 @@ public class Barrier<S extends Seekables<?, ?>> extends AsyncResults.AbstractRes
private ExistingTransactionCheck checkForExistingTransaction()
{
- checkState(seekables.size() == 1 && seekables.domain() == Domain.Key);
+ checkState(route.size() == 1 && route.domain() == Domain.Key);
ExistingTransactionCheck check = new ExistingTransactionCheck();
- Key k = seekables.get(0).asKey();
+ RoutingKey k = route.get(0).asRoutingKey();
node.commandStores().mapReduceConsume(
contextFor(k, KeyHistory.COMMANDS),
k.toUnseekable(),
@@ -242,8 +255,8 @@ public class Barrier<S extends Seekables<?, ?>> extends AsyncResults.AbstractRes
@Nonnull
public final Timestamp executeAt;
@Nonnull
- public final Key key;
- public BarrierTxn(@Nonnull TxnId txnId, @Nonnull Timestamp executeAt, Key key)
+ public final RoutingKey key;
+ public BarrierTxn(@Nonnull TxnId txnId, @Nonnull Timestamp executeAt, RoutingKey key)
{
this.txnId = txnId;
this.executeAt = executeAt;
@@ -265,8 +278,7 @@ public class Barrier<S extends Seekables<?, ?>> extends AsyncResults.AbstractRes
{
// TODO (required, consider): consider these semantics
BarrierTxn found = safeStore.mapReduceFull(
- seekables,
- safeStore.ranges().allAfter(minEpoch),
+ route,
// Barriers are trying to establish that committed transactions are applied before the barrier (or in this case just minEpoch)
// so all existing transaction types should ensure that at this point. An earlier txnid may have an executeAt that is after
// this barrier or the transaction we listen on and that is fine
@@ -279,7 +291,7 @@ public class Barrier<S extends Seekables<?, ?>> extends AsyncResults.AbstractRes
if (barrierTxn != null)
return barrierTxn;
if (keyOrRange.domain() == Domain.Key)
- return new BarrierTxn(txnId, executeAt, keyOrRange.asKey());
+ return new BarrierTxn(txnId, executeAt, keyOrRange.asRoutingKey());
return null;
},
null,
diff --git a/accord-core/src/main/java/accord/coordinate/CheckShards.java b/accord-core/src/main/java/accord/coordinate/CheckShards.java
index a778325e..8fc91ca6 100644
--- a/accord-core/src/main/java/accord/coordinate/CheckShards.java
+++ b/accord-core/src/main/java/accord/coordinate/CheckShards.java
@@ -34,7 +34,7 @@ import static accord.utils.Invariants.illegalState;
* A result of null indicates the transaction is globally persistent
* A result of CheckStatusOk indicates the maximum status found for the transaction, which may be used to assess progress
*/
-public abstract class CheckShards<U extends Unseekables<?>> extends ReadCoordinator<CheckStatusReply>
+public abstract class CheckShards<U extends Participants<?>> extends ReadCoordinator<CheckStatusReply>
{
final U route;
@@ -52,7 +52,7 @@ public abstract class CheckShards<U extends Unseekables<?>> extends ReadCoordina
protected CheckShards(Node node, TxnId txnId, U route, IncludeInfo includeInfo)
{
this(node, txnId, route, txnId.epoch(), includeInfo);
- Invariants.checkState(txnId.kind().isGloballyVisible());
+ Invariants.checkState(txnId.isVisible());
}
protected CheckShards(Node node, TxnId txnId, U route, long srcEpoch, IncludeInfo includeInfo)
@@ -72,7 +72,7 @@ public abstract class CheckShards<U extends Unseekables<?>> extends ReadCoordina
@Override
protected void contact(Id id)
{
- Unseekables<?> unseekables = route.slice(topologies().computeRangesForNode(id));
+ Participants<?> unseekables = route.slice(topologies().computeRangesForNode(id));
node.send(id, new CheckStatus(txnId, unseekables, sourceEpoch, includeInfo), this);
}
diff --git a/accord-core/src/main/java/accord/coordinate/CollectCalculatedDeps.java b/accord-core/src/main/java/accord/coordinate/CollectCalculatedDeps.java
index e715001a..7e48ccb7 100644
--- a/accord-core/src/main/java/accord/coordinate/CollectCalculatedDeps.java
+++ b/accord-core/src/main/java/accord/coordinate/CollectCalculatedDeps.java
@@ -18,8 +18,6 @@
package accord.coordinate;
-import java.util.ArrayList;
-import java.util.List;
import java.util.function.BiConsumer;
import accord.api.RoutingKey;
@@ -32,6 +30,7 @@ import accord.messages.CalculateDeps;
import accord.messages.CalculateDeps.CalculateDepsOk;
import accord.primitives.*;
import accord.topology.Topologies;
+import accord.utils.SortedListMap;
import static accord.coordinate.tracking.RequestStatus.Failed;
import static accord.coordinate.tracking.RequestStatus.Success;
@@ -43,7 +42,7 @@ public class CollectCalculatedDeps implements Callback<CalculateDepsOk>
final RoutingKey homeKey;
final Timestamp executeAt;
- private final List<CalculateDepsOk> oks;
+ private final SortedListMap<Id, CalculateDepsOk> oks;
private final QuorumTracker tracker;
private final BiConsumer<Deps, Throwable> callback;
private boolean isDone;
@@ -55,18 +54,18 @@ public class CollectCalculatedDeps implements Callback<CalculateDepsOk>
this.homeKey = homeKey;
this.executeAt = executeAt;
this.callback = callback;
- this.oks = new ArrayList<>();
+ this.oks = new SortedListMap<>(topologies.nodes(), CalculateDepsOk[]::new);
this.tracker = new QuorumTracker(topologies);
}
- public static void withCalculatedDeps(Node node, TxnId txnId, FullRoute<?> fullRoute, Unseekables<?> sendTo, Seekables<?, ?> keysOrRanges, Timestamp executeAt, BiConsumer<Deps, Throwable> callback)
+ public static void withCalculatedDeps(Node node, TxnId txnId, FullRoute<?> fullRoute, Unseekables<?> sendTo, Timestamp executeAt, BiConsumer<Deps, Throwable> callback)
{
Topologies topologies = node.topology().withUnsyncedEpochs(sendTo, txnId, executeAt);
CollectCalculatedDeps collect = new CollectCalculatedDeps(node, topologies, txnId, fullRoute.homeKey(), executeAt, callback);
CommandStore store = CommandStore.maybeCurrent();
if (store == null)
store = node.commandStores().select(fullRoute);
- node.send(collect.tracker.nodes(), to -> new CalculateDeps(to, topologies, fullRoute, txnId, keysOrRanges, executeAt),
+ node.send(collect.tracker.nodes(), to -> new CalculateDeps(to, topologies, fullRoute, txnId, executeAt),
store, collect);
}
@@ -76,7 +75,7 @@ public class CollectCalculatedDeps implements Callback<CalculateDepsOk>
if (isDone)
return;
- oks.add(ok);
+ oks.put(from, ok);
if (tracker.recordSuccess(from) == Success)
onQuorum();
}
@@ -110,7 +109,7 @@ public class CollectCalculatedDeps implements Callback<CalculateDepsOk>
return;
isDone = true;
- Deps deps = Deps.merge(oks, ok -> ok.deps);
+ Deps deps = Deps.merge(oks, oks.domainSize(), SortedListMap::getValue, ok -> ok.deps);
callback.accept(deps, null);
}
}
diff --git a/accord-core/src/main/java/accord/coordinate/CoordinateEphemeralRead.java b/accord-core/src/main/java/accord/coordinate/CoordinateEphemeralRead.java
index 60e3ff34..aabfa9e9 100644
--- a/accord-core/src/main/java/accord/coordinate/CoordinateEphemeralRead.java
+++ b/accord-core/src/main/java/accord/coordinate/CoordinateEphemeralRead.java
@@ -18,9 +18,7 @@
package accord.coordinate;
-import java.util.ArrayList;
import java.util.Collection;
-import java.util.List;
import accord.api.Result;
import accord.coordinate.tracking.QuorumTracker;
@@ -31,13 +29,15 @@ import accord.messages.GetEphemeralReadDeps;
import accord.messages.GetEphemeralReadDeps.GetEphemeralReadDepsOk;
import accord.primitives.Deps;
import accord.primitives.FullRoute;
-import accord.primitives.Seekables;
import accord.primitives.Txn;
import accord.primitives.TxnId;
import accord.topology.Topologies;
+import accord.utils.SortedListMap;
import accord.utils.async.AsyncResult;
import accord.utils.async.AsyncResults;
+import static accord.api.ProtocolModifiers.QuorumEpochIntersections;
+import static accord.api.ProtocolModifiers.QuorumEpochIntersections.Include.Owned;
import static accord.coordinate.tracking.RequestStatus.Failed;
import static accord.coordinate.tracking.RequestStatus.Success;
@@ -58,7 +58,7 @@ public class CoordinateEphemeralRead extends AbstractCoordinatePreAccept<Result,
{
public static AsyncResult<Result> coordinate(Node node, FullRoute<?> route, TxnId txnId, Txn txn)
{
- TopologyMismatch mismatch = TopologyMismatch.checkForMismatch(node.topology().globalForEpoch(txnId.epoch()), txnId, route.homeKey(), txn.keys());
+ TopologyMismatch mismatch = TopologyMismatch.checkForMismatch(node.topology().globalForEpoch(txnId.epoch()), txnId, route.homeKey(), route);
if (mismatch != null)
return AsyncResults.failure(mismatch);
@@ -71,7 +71,7 @@ public class CoordinateEphemeralRead extends AbstractCoordinatePreAccept<Result,
private final Txn txn;
private final QuorumTracker tracker;
- private final List<GetEphemeralReadDepsOk> oks;
+ private final SortedListMap<Node.Id, GetEphemeralReadDepsOk> oks;
private long executeAtEpoch;
CoordinateEphemeralRead(Node node, Topologies topologies, FullRoute<?> route, TxnId txnId, Txn txn)
@@ -80,13 +80,7 @@ public class CoordinateEphemeralRead extends AbstractCoordinatePreAccept<Result,
this.txn = txn;
this.tracker = new QuorumTracker(topologies);
this.executeAtEpoch = txnId.epoch();
- this.oks = new ArrayList<>(topologies.estimateUniqueNodes());
- }
-
- @Override
- Seekables<?, ?> keysOrRanges()
- {
- return txn.keys();
+ this.oks = new SortedListMap<>(topologies.nodes(), GetEphemeralReadDepsOk[]::new);
}
@Override
@@ -94,7 +88,7 @@ public class CoordinateEphemeralRead extends AbstractCoordinatePreAccept<Result,
{
CommandStore commandStore = CommandStore.maybeCurrent();
if (commandStore == null) commandStore = node.commandStores().select(route.homeKey());
- node.send(nodes, to -> new GetEphemeralReadDeps(to, topologies, route, txnId, txn.keys(), executeAtEpoch), commandStore, callback);
+ node.send(nodes, to -> new GetEphemeralReadDeps(to, topologies, route, txnId, executeAtEpoch), commandStore, callback);
}
@Override
@@ -106,7 +100,7 @@ public class CoordinateEphemeralRead extends AbstractCoordinatePreAccept<Result,
@Override
public void onSuccessInternal(Node.Id from, GetEphemeralReadDepsOk ok)
{
- oks.add(ok);
+ oks.put(from, ok);
if (ok.latestEpoch > executeAtEpoch)
executeAtEpoch = ok.latestEpoch;
@@ -120,7 +114,7 @@ public class CoordinateEphemeralRead extends AbstractCoordinatePreAccept<Result,
if (ok.latestEpoch > executeAtEpoch)
executeAtEpoch = ok.latestEpoch;
- oks.add(ok);
+ oks.put(from, ok);
return true;
}
@@ -140,8 +134,8 @@ public class CoordinateEphemeralRead extends AbstractCoordinatePreAccept<Result,
@Override
void onPreAccepted(Topologies topologies)
{
- Deps deps = Deps.merge(oks, ok -> ok.deps);
- topologies = topologies.forEpochs(executeAtEpoch, executeAtEpoch);
+ Deps deps = Deps.merge(oks, oks.domainSize(), SortedListMap::getValue, ok -> ok.deps);
+ topologies = node.topology().reselect(topologies, QuorumEpochIntersections.preaccept.include, route, executeAtEpoch, executeAtEpoch, Owned);
new ExecuteEphemeralRead(node, topologies, route, txnId, txn, executeAtEpoch, deps, this).start();
}
}
diff --git a/accord-core/src/main/java/accord/coordinate/CoordinatePreAccept.java b/accord-core/src/main/java/accord/coordinate/CoordinatePreAccept.java
index 2baa2d83..06a023e0 100644
--- a/accord-core/src/main/java/accord/coordinate/CoordinatePreAccept.java
+++ b/accord-core/src/main/java/accord/coordinate/CoordinatePreAccept.java
@@ -32,12 +32,12 @@ import accord.messages.PreAccept.PreAcceptOk;
import accord.messages.PreAccept.PreAcceptReply;
import accord.primitives.Ballot;
import accord.primitives.FullRoute;
-import accord.primitives.Seekables;
import accord.primitives.Timestamp;
import accord.primitives.Txn;
import accord.primitives.TxnId;
import accord.topology.Topologies;
+import static accord.api.ProtocolModifiers.QuorumEpochIntersections;
import static accord.coordinate.tracking.RequestStatus.Success;
import static accord.primitives.Timestamp.mergeMax;
import static accord.utils.Functions.foldl;
@@ -51,19 +51,21 @@ import static accord.utils.Functions.foldl;
abstract class CoordinatePreAccept<T> extends AbstractCoordinatePreAccept<T, PreAcceptReply>
{
final FastPathTracker tracker;
- private final List<PreAcceptOk> oks; // TODO (expected): this can be cleared after preaccept
+ // TODO (expected): this can be cleared after preaccept
+ // TODO (expected): back by SortedListMap; must handle additional preaccepts (but this is no longer ordinarily enabled)
+ private final List<PreAcceptOk> oks;
final Txn txn;
CoordinatePreAccept(Node node, TxnId txnId, Txn txn, FullRoute<?> route)
{
- this(node, txnId, txn, route, node.topology().withUnsyncedEpochs(route, txnId, txnId));
+ this(node, txnId, txn, route, node.topology().select(route, txnId, txnId, QuorumEpochIntersections.preaccept.include));
}
CoordinatePreAccept(Node node, TxnId txnId, Txn txn, FullRoute<?> route, Topologies topologies)
{
- super(node, route, txnId);
+ super(node, route, txnId, topologies);
this.tracker = new FastPathTracker(topologies);
- this.oks = new ArrayList<>(topologies.estimateUniqueNodes());
+ this.oks = new ArrayList<>(topologies.nodes().size());
this.txn = txn;
}
@@ -82,12 +84,6 @@ abstract class CoordinatePreAccept<T> extends AbstractCoordinatePreAccept<T, Pre
return foldl(oks, (ok, prev) -> ok.witnessedAt.epoch() > prev.epoch() ? ok.witnessedAt : prev, Timestamp.NONE).epoch();
}
- @Override
- Seekables<?, ?> keysOrRanges()
- {
- return txn.keys();
- }
-
@Override
public void onFailureInternal(Id from, Throwable failure)
{
diff --git a/accord-core/src/main/java/accord/coordinate/CoordinateShardDurable.java b/accord-core/src/main/java/accord/coordinate/CoordinateShardDurable.java
index a055f1f6..ec6e8a85 100644
--- a/accord-core/src/main/java/accord/coordinate/CoordinateShardDurable.java
+++ b/accord-core/src/main/java/accord/coordinate/CoordinateShardDurable.java
@@ -18,24 +18,32 @@
package accord.coordinate;
+import accord.coordinate.ExecuteSyncPoint.ExecuteExclusiveSyncPoint;
import accord.coordinate.tracking.AllTracker;
import accord.local.Node;
import accord.messages.Callback;
import accord.messages.ReadData.ReadReply;
import accord.messages.SetShardDurable;
import accord.messages.WaitUntilApplied;
-import accord.primitives.Ranges;
+import accord.primitives.Range;
import accord.primitives.SyncPoint;
import accord.utils.async.AsyncResult;
-public class CoordinateShardDurable extends ExecuteSyncPoint<Ranges> implements Callback<ReadReply>
+public class CoordinateShardDurable extends ExecuteExclusiveSyncPoint implements Callback<ReadReply>
{
- private CoordinateShardDurable(Node node, SyncPoint<Ranges> exclusiveSyncPoint)
+ private CoordinateShardDurable(Node node, SyncPoint<Range> exclusiveSyncPoint)
{
- super(node, new AllTracker(node.topology().forEpoch(exclusiveSyncPoint.keysOrRanges, exclusiveSyncPoint.sourceEpoch())), exclusiveSyncPoint);
+ super(node, exclusiveSyncPoint, AllTracker::new);
+ addCallback((success, fail) -> {
+ if (fail == null)
+ {
+ node.configService().reportEpochRedundant(syncPoint.route.toRanges(), syncPoint.syncId.epoch());
+ node.send(tracker.nodes(), new SetShardDurable(syncPoint));
+ }
+ });
}
- public static AsyncResult<SyncPoint<Ranges>> coordinate(Node node, SyncPoint<Ranges> exclusiveSyncPoint)
+ public static AsyncResult<SyncPoint<Range>> coordinate(Node node, SyncPoint<Range> exclusiveSyncPoint)
{
CoordinateShardDurable coordinate = new CoordinateShardDurable(node, exclusiveSyncPoint);
coordinate.start();
@@ -44,14 +52,6 @@ public class CoordinateShardDurable extends ExecuteSyncPoint<Ranges> implements
protected void start()
{
- node.send(tracker.nonStaleNodes(), to -> new WaitUntilApplied(to, tracker.topologies(), syncPoint.syncId, syncPoint.keysOrRanges, syncPoint.syncId.epoch()), this);
- }
-
- @Override
- protected void onSuccess()
- {
- node.configService().reportEpochRedundant(syncPoint.keysOrRanges, syncPoint.syncId.epoch());
- node.send(tracker.nonStaleNodes(), new SetShardDurable(syncPoint));
- super.onSuccess();
+ node.send(tracker.nodes(), to -> new WaitUntilApplied(to, tracker.topologies(), syncPoint.syncId, syncPoint.route, syncPoint.syncId.epoch()), this);
}
}
diff --git a/accord-core/src/main/java/accord/coordinate/CoordinateSyncPoint.java b/accord-core/src/main/java/accord/coordinate/CoordinateSyncPoint.java
index ee39096c..6fdb1ee9 100644
--- a/accord-core/src/main/java/accord/coordinate/CoordinateSyncPoint.java
+++ b/accord-core/src/main/java/accord/coordinate/CoordinateSyncPoint.java
@@ -24,24 +24,23 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import accord.coordinate.CoordinationAdapter.Adapters;
+import accord.coordinate.CoordinationAdapter.Adapters.SyncPointAdapter;
import accord.local.Node;
import accord.messages.Apply;
import accord.messages.PreAccept.PreAcceptOk;
import accord.primitives.Ballot;
import accord.primitives.Deps;
import accord.primitives.FullRoute;
-import accord.primitives.Seekables;
import accord.primitives.SyncPoint;
import accord.primitives.Timestamp;
import accord.primitives.Txn;
import accord.primitives.Txn.Kind;
import accord.primitives.TxnId;
+import accord.primitives.Unseekable;
+import accord.primitives.Unseekables;
import accord.topology.Topologies;
import accord.utils.async.AsyncResult;
-import accord.utils.async.AsyncResults;
-import static accord.coordinate.CoordinationAdapter.Invoke.execute;
-import static accord.coordinate.CoordinationAdapter.Invoke.propose;
import static accord.coordinate.ExecutePath.FAST;
import static accord.coordinate.Propose.Invalidate.proposeAndCommitInvalidate;
import static accord.primitives.Timestamp.mergeMax;
@@ -55,54 +54,85 @@ import static accord.utils.Invariants.checkArgument;
*
* TODO (desired, testing): dedicated burn test to validate outcomes
*/
-public class CoordinateSyncPoint<S extends Seekables<?, ?>> extends CoordinatePreAccept<SyncPoint<S>>
+public class CoordinateSyncPoint<U extends Unseekable> extends CoordinatePreAccept<SyncPoint<U>>
{
@SuppressWarnings("unused")
private static final Logger logger = LoggerFactory.getLogger(CoordinateSyncPoint.class);
- final CoordinationAdapter<SyncPoint<S>> adapter;
+ final CoordinationAdapter<SyncPoint<U>> adapter;
- private CoordinateSyncPoint(Node node, TxnId txnId, Txn txn, FullRoute<?> route, CoordinationAdapter<SyncPoint<S>> adapter)
+ private CoordinateSyncPoint(Node node, TxnId txnId, Topologies topologies, Txn txn, FullRoute<?> route, CoordinationAdapter<SyncPoint<U>> adapter)
{
- super(node, txnId, txn, route, node.topology().withOpenEpochs(route, txnId, txnId));
+ super(node, txnId, txn, route, topologies);
this.adapter = adapter;
}
- public static <S extends Seekables<?, ?>> AsyncResult<SyncPoint<S>> exclusive(Node node, S keysOrRanges)
+ public static <U extends Unseekable> AsyncResult<SyncPoint<U>> exclusiveSyncPoint(Node node, Unseekables<U> keysOrRanges)
{
return coordinate(node, ExclusiveSyncPoint, keysOrRanges, Adapters.exclusiveSyncPoint());
}
- public static <S extends Seekables<?, ?>> AsyncResult<SyncPoint<S>> exclusive(Node node, TxnId txnId, S keysOrRanges)
+ public static <U extends Unseekable> AsyncResult<SyncPoint<U>> exclusiveSyncPoint(Node node, TxnId txnId, Unseekables<U> keysOrRanges)
{
+ Topologies topologies = node.topology().withOpenEpochs(keysOrRanges, null, null);
return coordinate(node, txnId, keysOrRanges, Adapters.exclusiveSyncPoint());
}
- public static <S extends Seekables<?, ?>> AsyncResult<SyncPoint<S>> inclusive(Node node, S keysOrRanges)
+ public static <U extends Unseekable> AsyncResult<SyncPoint<U>> inclusive(Node node, Unseekables<U> keysOrRanges)
{
return coordinate(node, Kind.SyncPoint, keysOrRanges, Adapters.inclusiveSyncPoint());
}
- public static <S extends Seekables<?, ?>> AsyncResult<SyncPoint<S>> inclusiveAndAwaitQuorum(Node node, S keysOrRanges)
+ public static <U extends Unseekable> AsyncResult<SyncPoint<U>> inclusive(Node node, FullRoute<U> route)
+ {
+ return coordinate(node, Kind.SyncPoint, route, Adapters.inclusiveSyncPoint());
+ }
+
+ public static <U extends Unseekable> AsyncResult<SyncPoint<U>> inclusive(Node node, TxnId txnId, FullRoute<U> route)
+ {
+ return node.withEpoch(txnId.epoch(), () -> coordinate(node, txnId, route, Adapters.inclusiveSyncPoint())).beginAsResult();
+ }
+
+ public static <U extends Unseekable> AsyncResult<SyncPoint<U>> inclusiveAndAwaitQuorum(Node node, Unseekables<U> keysOrRanges)
{
return coordinate(node, Kind.SyncPoint, keysOrRanges, Adapters.inclusiveSyncPointBlocking());
}
- public static <S extends Seekables<?, ?>> AsyncResult<SyncPoint<S>> coordinate(Node node, Kind kind, S keysOrRanges, CoordinationAdapter<SyncPoint<S>> adapter)
+ public static <U extends Unseekable> AsyncResult<SyncPoint<U>> inclusiveAndAwaitQuorum(Node node, FullRoute<U> route)
+ {
+ return coordinate(node, Kind.SyncPoint, route, Adapters.inclusiveSyncPointBlocking());
+ }
+
+ public static <U extends Unseekable> AsyncResult<SyncPoint<U>> inclusiveAndAwaitQuorum(Node node, TxnId txnId, FullRoute<U> route)
{
- checkArgument(kind == Kind.SyncPoint || kind == ExclusiveSyncPoint);
+ return node.withEpoch(txnId.epoch(), () -> coordinate(node, txnId, route, Adapters.inclusiveSyncPointBlocking())).beginAsResult();
+ }
+
+ public static <U extends Unseekable> AsyncResult<SyncPoint<U>> coordinate(Node node, Kind kind, Unseekables<U> keysOrRanges, SyncPointAdapter<U> adapter)
+ {
+ checkArgument(kind.isSyncPoint());
TxnId txnId = node.nextTxnId(kind, keysOrRanges.domain());
return node.withEpoch(txnId.epoch(), () -> coordinate(node, txnId, keysOrRanges, adapter)).beginAsResult();
}
- private static <S extends Seekables<?, ?>> AsyncResult<SyncPoint<S>> coordinate(Node node, TxnId txnId, S keysOrRanges, CoordinationAdapter<SyncPoint<S>> adapter)
+ public static <U extends Unseekable> AsyncResult<SyncPoint<U>> coordinate(Node node, Kind kind, FullRoute<U> route, SyncPointAdapter<U> adapter)
+ {
+ checkArgument(kind.isSyncPoint());
+ TxnId txnId = node.nextTxnId(kind, route.domain());
+ return node.withEpoch(txnId.epoch(), () -> coordinate(node, txnId, route, adapter)).beginAsResult();
+ }
+
+ private static <U extends Unseekable> AsyncResult<SyncPoint<U>> coordinate(Node node, TxnId txnId, Unseekables<U> keysOrRanges, SyncPointAdapter<U> adapter)
+ {
+ checkArgument(txnId.isSyncPoint());
+ FullRoute<U> route = (FullRoute<U>) node.computeRoute(txnId, keysOrRanges);
+ return coordinate(node, txnId, route, adapter);
+ }
+
+ private static <U extends Unseekable> AsyncResult<SyncPoint<U>> coordinate(Node node, TxnId txnId, FullRoute<U> route, SyncPointAdapter<U> adapter)
{
- checkArgument(txnId.kind() == Kind.SyncPoint || txnId.kind() == ExclusiveSyncPoint);
- FullRoute<?> route = node.computeRoute(txnId, keysOrRanges);
- TopologyMismatch mismatch = TopologyMismatch.checkForMismatch(node.topology().globalForEpoch(txnId.epoch()), txnId, route.homeKey(), keysOrRanges);
- if (mismatch != null)
- return AsyncResults.failure(mismatch);
- CoordinateSyncPoint<S> coordinate = new CoordinateSyncPoint<>(node, txnId, node.agent().emptySystemTxn(txnId.kind(), keysOrRanges), route, adapter);
+ checkArgument(txnId.isSyncPoint());
+ CoordinateSyncPoint<U> coordinate = new CoordinateSyncPoint<>(node, txnId, adapter.forDecision(node, route, txnId), node.agent().emptySystemTxn(txnId.kind(), txnId.domain()), route, adapter);
coordinate.start();
return coordinate;
}
@@ -116,7 +146,7 @@ public class CoordinateSyncPoint<S extends Seekables<?, ?>> extends CoordinatePr
@Override
void onPreAccepted(Topologies topologies, Timestamp executeAt, List<PreAcceptOk> oks)
{
- Deps deps = Deps.merge(oks, ok -> ok.deps);
+ Deps deps = Deps.merge(oks, oks.size(), List::get, ok -> ok.deps);
Timestamp checkRejected = foldl(oks, (ok, prev) -> mergeMax(ok.witnessedAt, prev), Timestamp.NONE);
if (checkRejected.isRejected())
{
@@ -124,13 +154,10 @@ public class CoordinateSyncPoint<S extends Seekables<?, ?>> extends CoordinatePr
}
else
{
- // we don't need to fetch deps from Accept replies, so we don't need to contact unsynced epochs
- topologies = node.topology().forEpoch(route, txnId.epoch());
- // TODO (required, consider): consider the required semantics of a SyncPoint
if (tracker.hasFastPathAccepted() && txnId.kind() == Kind.SyncPoint)
- execute(adapter, node, topologies, route, FAST, txnId, txn, txnId, deps, this);
+ adapter.execute(node, topologies, route, FAST, txnId, txn, txnId, deps, this);
else
- propose(adapter, node, topologies, route, Ballot.ZERO, txnId, txn, executeAt, deps, this);
+ adapter.propose(node, topologies, route, Ballot.ZERO, txnId, txn, executeAt, deps, this);
}
}
@@ -138,7 +165,7 @@ public class CoordinateSyncPoint<S extends Seekables<?, ?>> extends CoordinatePr
{
TxnId txnId = syncPoint.syncId;
Timestamp executeAt = txnId;
- Txn txn = node.agent().emptySystemTxn(txnId.kind(), syncPoint.keysOrRanges);
+ Txn txn = node.agent().emptySystemTxn(txnId.kind(), txnId.domain());
Deps deps = syncPoint.waitFor;
Apply.sendMaximal(node, to, txnId, syncPoint.route(), txn, executeAt, deps, txn.execute(txnId, executeAt, null), txn.result(txnId, executeAt, null));
}
diff --git a/accord-core/src/main/java/accord/coordinate/CoordinateTransaction.java b/accord-core/src/main/java/accord/coordinate/CoordinateTransaction.java
index 0b05657d..d13f1253 100644
--- a/accord-core/src/main/java/accord/coordinate/CoordinateTransaction.java
+++ b/accord-core/src/main/java/accord/coordinate/CoordinateTransaction.java
@@ -36,8 +36,6 @@ import accord.utils.async.AsyncResult;
import accord.utils.async.AsyncResults;
import static accord.coordinate.CoordinationAdapter.Factory.Step.Continue;
-import static accord.coordinate.CoordinationAdapter.Invoke.execute;
-import static accord.coordinate.CoordinationAdapter.Invoke.propose;
import static accord.coordinate.ExecutePath.FAST;
import static accord.coordinate.Propose.Invalidate.proposeAndCommitInvalidate;
@@ -59,7 +57,7 @@ public class CoordinateTransaction extends CoordinatePreAccept<Result>
public static AsyncResult<Result> coordinate(Node node, FullRoute<?> route, TxnId txnId, Txn txn)
{
- TopologyMismatch mismatch = TopologyMismatch.checkForMismatch(node.topology().globalForEpoch(txnId.epoch()), txnId, route.homeKey(), txn.keys());
+ TopologyMismatch mismatch = TopologyMismatch.checkForMismatch(node.topology().globalForEpoch(txnId.epoch()), txnId, route.homeKey(), route);
if (mismatch != null)
return AsyncResults.failure(mismatch);
CoordinateTransaction coordinate = new CoordinateTransaction(node, txnId, txn, route);
@@ -72,13 +70,13 @@ public class CoordinateTransaction extends CoordinatePreAccept<Result>
{
if (tracker.hasFastPathAccepted())
{
- Deps deps = Deps.merge(oks, ok -> ok.witnessedAt.equals(txnId) ? ok.deps : null);
- execute(executeAdapter(), node, topologies, route, FAST, txnId, txn, txnId, deps, settingCallback());
+ Deps deps = Deps.merge(oks, oks.size(), List::get, ok -> ok.witnessedAt.equals(txnId) ? ok.deps : null);
+ executeAdapter().execute(node, topologies, route, FAST, txnId, txn, txnId, deps, settingCallback());
node.agent().metricsEventsListener().onFastPathTaken(txnId, deps);
}
else
{
- Deps deps = Deps.merge(oks, ok -> ok.deps);
+ Deps deps = Deps.merge(oks, oks.size(), List::get, ok -> ok.deps);
// TODO (low priority, efficiency): perhaps don't submit Accept immediately if we almost have enough for fast-path,
// but by sending accept we rule out hybrid fast-path
@@ -93,7 +91,7 @@ public class CoordinateTransaction extends CoordinatePreAccept<Result>
if (PreAccept.rejectExecuteAt(txnId, topologies))
proposeAndCommitInvalidate(node, Ballot.ZERO, txnId, route.homeKey(), route, executeAt, this);
else
- propose(proposeAdapter(), node, topologies, route, Ballot.ZERO, txnId, txn, executeAt, deps, this);
+ proposeAdapter().propose(node, topologies, route, Ballot.ZERO, txnId, txn, executeAt, deps, this);
}
node.agent().metricsEventsListener().onSlowPathTaken(txnId, deps);
diff --git a/accord-core/src/main/java/accord/coordinate/CoordinationAdapter.java b/accord-core/src/main/java/accord/coordinate/CoordinationAdapter.java
index 9fe4e491..1719da1a 100644
--- a/accord-core/src/main/java/accord/coordinate/CoordinationAdapter.java
+++ b/accord-core/src/main/java/accord/coordinate/CoordinationAdapter.java
@@ -21,6 +21,7 @@ package accord.coordinate;
import java.util.function.BiConsumer;
import javax.annotation.Nullable;
+import accord.api.ProtocolModifiers;
import accord.api.Result;
import accord.coordinate.ExecuteSyncPoint.ExecuteBlocking;
import accord.local.Node;
@@ -29,21 +30,19 @@ import accord.primitives.Ballot;
import accord.primitives.Deps;
import accord.primitives.FullRoute;
import accord.primitives.Participants;
-import accord.primitives.Ranges;
-import accord.primitives.Seekables;
import accord.primitives.SyncPoint;
import accord.primitives.Timestamp;
import accord.primitives.Txn;
import accord.primitives.TxnId;
+import accord.primitives.Unseekable;
import accord.primitives.Writes;
import accord.topology.Topologies;
-import accord.utils.Faults;
+import static accord.api.ProtocolModifiers.QuorumEpochIntersections;
import static accord.coordinate.ExecutePath.FAST;
import static accord.coordinate.ExecutePath.SLOW;
import static accord.messages.Apply.Kind.Maximal;
import static accord.messages.Apply.Kind.Minimal;
-import static accord.primitives.Routable.Domain.Range;
public interface CoordinationAdapter<R>
{
@@ -53,10 +52,14 @@ public interface CoordinationAdapter<R>
<R> CoordinationAdapter<R> get(TxnId txnId, Step step);
}
- void propose(Node node, Topologies withUnsynced, FullRoute<?> route, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer<? super R, Throwable> callback);
- void stabilise(Node node, Topologies coordinates, Topologies all, FullRoute<?> route, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer<? super R, Throwable> callback);
- void execute(Node node, Topologies all, FullRoute<?> route, ExecutePath path, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer<? super R, Throwable> callback);
- void persist(Node node, Topologies all, FullRoute<?> route, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, BiConsumer<? super R, Throwable> callback);
+ void propose(Node node, @Nullable Topologies preaccept, FullRoute<?> route, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer<? super R, Throwable> callback);
+ void stabilise(Node node, @Nullable Topologies any, FullRoute<?> route, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer<? super R, Throwable> callback);
+ void execute(Node node, @Nullable Topologies any, FullRoute<?> route, ExecutePath path, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer<? super R, Throwable> callback);
+ void persist(Node node, @Nullable Topologies any, FullRoute<?> route, Participants<?> sendTo, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, BiConsumer<? super R, Throwable> callback);
+ default void persist(Node node, @Nullable Topologies any, FullRoute<?> route, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, BiConsumer<? super R, Throwable> callback)
+ {
+ persist(node, any, route, route, txnId, txn, executeAt, deps, writes, result, callback);
+ }
class DefaultFactory implements Factory
{
@@ -72,67 +75,6 @@ public interface CoordinationAdapter<R>
}
}
- /**
- * Utility methods for correctly invoking the next phase of the state machine via a CoordinationAdapter.
- * Simply ensures the topologies are correct before being passed to the instance method.
- */
- class Invoke
- {
- public static <R> void propose(CoordinationAdapter<R> adapter, Node node, FullRoute<?> route, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer<? super R, Throwable> callback)
- {
- propose(adapter, node, node.topology().withUnsyncedEpochs(route, txnId, executeAt), route, ballot, txnId, txn, executeAt, deps, callback);
- }
-
- public static <R> void propose(CoordinationAdapter<R> adapter, Node node, Topologies withUnsynced, FullRoute<?> route, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer<? super R, Throwable> callback)
- {
- adapter.propose(node, withUnsynced, route, ballot, txnId, txn, executeAt, deps, callback);
- }
-
- public static <R> void stabilise(CoordinationAdapter<R> adapter, Node node, Topologies any, FullRoute<?> route, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer<? super R, Throwable> callback)
- {
- if (!node.topology().hasEpoch(executeAt.epoch()))
- {
- node.withEpoch(executeAt.epoch(), (ignore, withEpochFailure) -> {
- if (withEpochFailure != null)
- callback.accept(null, CoordinationFailed.wrap(withEpochFailure));
- else
- stabilise(adapter, node, any, route, ballot, txnId, txn, executeAt, deps, callback);
- });
- return;
- }
- Topologies coordinates = any.forEpochs(txnId.epoch(), txnId.epoch());
- Topologies all;
- if (txnId.epoch() == executeAt.epoch()) all = coordinates;
- else if (any.currentEpoch() >= executeAt.epoch() && any.oldestEpoch() <= txnId.epoch()) all = any.forEpochs(txnId.epoch(), executeAt.epoch());
- else all = node.topology().preciseEpochs(route, txnId.epoch(), executeAt.epoch());
-
- adapter.stabilise(node, coordinates, all, route, ballot, txnId, txn, executeAt, deps, callback);
- }
-
- public static <R> void execute(CoordinationAdapter<R> adapter, Node node, Topologies any, FullRoute<?> route, ExecutePath path, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer<? super R, Throwable> callback)
- {
- if (any.oldestEpoch() <= txnId.epoch() && any.currentEpoch() >= executeAt.epoch()) any = any.forEpochs(txnId.epoch(), executeAt.epoch());
- else any = node.topology().preciseEpochs(route, txnId.epoch(), executeAt.epoch());
- adapter.execute(node, any, route, path, txnId, txn, executeAt, deps, callback);
- }
-
- public static <R> void persist(CoordinationAdapter<R> adapter, Node node, Topologies any, FullRoute<?> route, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, @Nullable BiConsumer<? super R, Throwable> callback)
- {
- if (any.oldestEpoch() <= txnId.epoch() && any.currentEpoch() >= executeAt.epoch()) any = any.forEpochs(txnId.epoch(), executeAt.epoch());
- else any = node.topology().preciseEpochs(route, txnId.epoch(), executeAt.epoch());
-
- adapter.persist(node, any, route, txnId, txn, executeAt, deps, writes, result, callback);
- }
-
- public static <R> void persist(CoordinationAdapter<R> adapter, Node node, FullRoute<?> route, Participants<?> sendTo, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, @Nullable BiConsumer<? super R, Throwable> callback)
- {
- Topologies all = node.topology().preciseEpochs(sendTo, txnId.epoch(), executeAt.epoch());
- Topologies executes = all.forEpochs(executeAt.epoch(), executeAt.epoch());
-
- adapter.persist(node, all, route, txnId, txn, executeAt, deps, writes, result, callback);
- }
- }
-
class Adapters
{
public static CoordinationAdapter<Result> standard()
@@ -147,17 +89,17 @@ public interface CoordinationAdapter<R>
return RecoveryTxnAdapter.INSTANCE;
}
- public static <S extends Seekables<?, ?>> CoordinationAdapter<SyncPoint<S>> inclusiveSyncPoint()
+ public static <U extends Unseekable> SyncPointAdapter<U> inclusiveSyncPoint()
{
return AsyncInclusiveSyncPointAdapter.INSTANCE;
}
- public static <S extends Seekables<?, ?>> CoordinationAdapter<SyncPoint<S>> inclusiveSyncPointBlocking()
+ public static <U extends Unseekable> SyncPointAdapter<U> inclusiveSyncPointBlocking()
{
return InclusiveSyncPointBlockingAdapter.INSTANCE;
}
- public static <S extends Seekables<?, ?>> CoordinationAdapter<SyncPoint<S>> exclusiveSyncPoint()
+ public static <U extends Unseekable> SyncPointAdapter<U> exclusiveSyncPoint()
{
return ExclusiveSyncPointAdapter.INSTANCE;
}
@@ -165,32 +107,59 @@ public interface CoordinationAdapter<R>
public static abstract class AbstractTxnAdapter implements CoordinationAdapter<Result>
{
@Override
- public void propose(Node node, Topologies withUnsynced, FullRoute<?> route, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer<? super Result, Throwable> callback)
+ public void propose(Node node, @Nullable Topologies preacceptOrRecovery, FullRoute<?> route, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer<? super Result, Throwable> callback)
{
- new ProposeTxn(node, withUnsynced, route, ballot, txnId, txn, executeAt, deps, callback).start();
+ Topologies accept = node.topology().reselect(preacceptOrRecovery, QuorumEpochIntersections.preacceptOrRecover,
+ route, txnId, executeAt, QuorumEpochIntersections.accept);
+ new ProposeTxn(node, accept, route, ballot, txnId, txn, executeAt, deps, callback).start();
}
@Override
- public void stabilise(Node node, Topologies coordinates, Topologies all, FullRoute<?> route, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer<? super Result, Throwable> callback)
+ public void stabilise(Node node, Topologies accept, FullRoute<?> route, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer<? super Result, Throwable> callback)
{
- if (Faults.TRANSACTION_INSTABILITY) execute(node, all, route, SLOW, txnId, txn, executeAt, deps, callback);
+ if (!node.topology().hasEpoch(executeAt.epoch()))
+ {
+ node.withEpoch(executeAt.epoch(), (success, fail) -> {
+ if (fail != null) callback.accept(null, fail);
+ else stabilise(node, accept, route, ballot, txnId, txn, executeAt, deps, callback);
+ });
+ return;
+ }
+
+ Topologies all = node.topology().reselect(accept, QuorumEpochIntersections.accept,
+ route, txnId, executeAt, QuorumEpochIntersections.commit);
+ Topologies coordinates = all.size() == 1 ? all : accept.forEpochs(txnId.epoch(), txnId.epoch());
+
+ if (ProtocolModifiers.Faults.txnInstability) execute(node, all, route, SLOW, txnId, txn, executeAt, deps, callback);
else new StabiliseTxn(node, coordinates, all, route, ballot, txnId, txn, executeAt, deps, callback).start();
}
@Override
- public void execute(Node node, Topologies all, FullRoute<?> route, ExecutePath path, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer<? super Result, Throwable> callback)
+ public void execute(Node node, Topologies any, FullRoute<?> route, ExecutePath path, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer<? super Result, Throwable> callback)
{
- if (txn.read().keys().isEmpty()) Invoke.persist(this, node, all, route, txnId, txn, executeAt, deps, txn.execute(txnId, executeAt, null), txn.result(txnId, executeAt, null), callback);
+ Topologies all = execution(node, any, route, txnId, executeAt);
+
+ if (txn.read().keys().isEmpty()) persist(node, all, route, txnId, txn, executeAt, deps, txn.execute(txnId, executeAt, null), txn.result(txnId, executeAt, null), callback);
else new ExecuteTxn(node, all, route, path, txnId, txn, txn.read().keys().toParticipants(), executeAt, deps, callback).start();
}
+
+ Topologies execution(Node node, Topologies preacceptOrCommit, Participants<?> participants, TxnId txnId, Timestamp executeAt)
+ {
+ return node.topology().reselect(preacceptOrCommit, QuorumEpochIntersections.preacceptOrCommit,
+ participants, txnId, executeAt, QuorumEpochIntersections.stable);
+ }
}
public static class StandardTxnAdapter extends AbstractTxnAdapter
{
public static final StandardTxnAdapter INSTANCE = new StandardTxnAdapter();
+
@Override
- public void persist(Node node, Topologies all, FullRoute<?> route, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, BiConsumer<? super Result, Throwable> callback)
+ public void persist(Node node, Topologies any, FullRoute<?> route, Participants<?> sendTo, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, BiConsumer<? super Result, Throwable> callback)
{
+ // TODO (required): we aren't using sendTo
+ Topologies all = execution(node, any, sendTo, txnId, executeAt);
+
if (callback != null) callback.accept(result, null);
new PersistTxn(node, all, txnId, route, txn, executeAt, deps, writes, result)
.start(Apply.FACTORY, Minimal, all, writes, result);
@@ -201,86 +170,115 @@ public interface CoordinationAdapter<R>
{
public static final RecoveryTxnAdapter INSTANCE = new RecoveryTxnAdapter();
@Override
- public void persist(Node node, Topologies all, FullRoute<?> route, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, BiConsumer<? super Result, Throwable> callback)
+ public void persist(Node node, Topologies any, FullRoute<?> route, Participants<?> participants, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, BiConsumer<? super Result, Throwable> callback)
{
+ Topologies all = execution(node, any, participants, txnId, executeAt);
+
if (callback != null) callback.accept(result, null);
new PersistTxn(node, all, txnId, route, txn, executeAt, deps, writes, result)
.start(Apply.FACTORY, Maximal, all, writes, result);
}
}
- public static abstract class AbstractSyncPointAdapter<S extends Seekables<?, ?>> implements CoordinationAdapter<SyncPoint<S>>
+ public static abstract class SyncPointAdapter<U extends Unseekable> implements CoordinationAdapter<SyncPoint<U>>
{
- void invokeSuccess(Node node, FullRoute<?> route, TxnId txnId, Txn txn, Deps deps, BiConsumer<? super SyncPoint<S>, Throwable> callback)
+ abstract Topologies forDecision(Node node, FullRoute<?> route, TxnId txnId);
+ abstract Topologies forExecution(Node node, FullRoute<?> route, TxnId txnId, Timestamp executeAt, Deps deps);
+
+ void invokeSuccess(Node node, FullRoute<?> route, TxnId txnId, Txn txn, Deps deps, BiConsumer<? super SyncPoint<U>, Throwable> callback)
{
- if (txn.keys().domain() == Range)
- node.configService().reportEpochClosed((Ranges)txn.keys(), txnId.epoch());
- callback.accept(new SyncPoint<>(txnId, deps, (S)txn.keys(), route), null);
+ callback.accept(new SyncPoint<>(txnId, deps, (FullRoute<U>)route), null);
}
@Override
- public void propose(Node node, Topologies withUnsynced, FullRoute<?> route, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer<? super SyncPoint<S>, Throwable> callback)
+ public void propose(Node node, Topologies any, FullRoute<?> route, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer<? super SyncPoint<U>, Throwable> callback)
{
- new ProposeSyncPoint<>(this, node, withUnsynced, route, ballot, txnId, txn, executeAt, deps, callback).start();
+ Topologies all = forDecision(node, route, txnId);
+ new ProposeSyncPoint<>(this, node, all, route, ballot, txnId, txn, executeAt, deps, callback).start();
}
@Override
- public void stabilise(Node node, Topologies coordinates, Topologies all, FullRoute<?> route, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer<? super SyncPoint<S>, Throwable> callback)
+ public void stabilise(Node node, Topologies any, FullRoute<?> route, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer<? super SyncPoint<U>, Throwable> callback)
{
+ Topologies all = forExecution(node, route, txnId, executeAt, deps);
+ Topologies coordinates = all.forEpochs(txnId.epoch(), txnId.epoch());
new StabiliseSyncPoint<>(this, node, coordinates, all, route, ballot, txnId, txn, executeAt, deps, callback).start();
}
@Override
- public void execute(Node node, Topologies all, FullRoute<?> route, ExecutePath path, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer<? super SyncPoint<S>, Throwable> callback)
+ public void execute(Node node, Topologies any, FullRoute<?> route, ExecutePath path, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer<? super SyncPoint<U>, Throwable> callback)
{
- Invoke.persist(this, node, all, route, txnId, txn, executeAt, deps, txn.execute(txnId, executeAt, null), txn.result(txnId, executeAt, null), callback);
+ Topologies all = forExecution(node, route, txnId, executeAt, deps);
+ persist(node, all, route, txnId, txn, executeAt, deps, txn.execute(txnId, executeAt, null), txn.result(txnId, executeAt, null), callback);
}
@Override
- public void persist(Node node, Topologies all, FullRoute<?> route, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, BiConsumer<? super SyncPoint<S>, Throwable> callback)
+ public void persist(Node node, Topologies any, FullRoute<?> route, Participants<?> participants, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, BiConsumer<? super SyncPoint<U>, Throwable> callback)
{
+ Topologies all = forExecution(node, route, txnId, executeAt, deps);
+
invokeSuccess(node, route, txnId, txn, deps, callback);
- new PersistTxn(node, all, txnId, route, txn, executeAt, deps, writes, result)
- .start(Apply.FACTORY, Maximal, all, writes, result);
+ new PersistExclusiveSyncPoint(node, all, txnId, route, txn, executeAt, deps, writes, result)
+ .start(Apply.FACTORY, Maximal, any, writes, result);
}
}
- public static class ExclusiveSyncPointAdapter<S extends Seekables<?, ?>> extends AbstractSyncPointAdapter<S>
+ public static class ExclusiveSyncPointAdapter<U extends Unseekable> extends SyncPointAdapter<U>
{
private static final ExclusiveSyncPointAdapter INSTANCE = new ExclusiveSyncPointAdapter();
@Override
- public void execute(Node node, Topologies all, FullRoute<?> route, ExecutePath path, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer<? super SyncPoint<S>, Throwable> callback)
+ Topologies forDecision(Node node, FullRoute<?> route, TxnId txnId)
+ {
+ return node.topology().withOpenEpochs(route, null, txnId);
+ }
+
+ @Override
+ Topologies forExecution(Node node, FullRoute<?> route, TxnId txnId, Timestamp executeAt, Deps deps)
+ {
+ TxnId minId = TxnId.nonNullOrMin(txnId, deps.minTxnId());
+ return node.topology().withUncompletedEpochs(route, minId, txnId);
+ }
+
+ @Override
+ public void execute(Node node, Topologies any, FullRoute<?> route, ExecutePath path, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer<? super SyncPoint<U>, Throwable> callback)
{
// TODO (required, consider): remember and document why we don't use fast path for exclusive sync points
- if (path == FAST)
- {
- Invoke.stabilise(this, node, all, route, Ballot.ZERO, txnId, txn, executeAt, deps, callback);
- }
- else
- {
- super.execute(node, all, route, path, txnId, txn, executeAt, deps, callback);
- }
+ if (path == FAST) stabilise(node, any, route, Ballot.ZERO, txnId, txn, executeAt, deps, callback);
+ else super.execute(node, any, route, path, txnId, txn, executeAt, deps, callback);
}
}
- private static abstract class AbstractInclusiveSyncPointAdapter<S extends Seekables<?, ?>> extends AbstractSyncPointAdapter<S>
+ public static abstract class AbstractInclusiveSyncPointAdapter<U extends Unseekable> extends SyncPointAdapter<U>
{
-
protected AbstractInclusiveSyncPointAdapter()
{
super();
}
@Override
- public void execute(Node node, Topologies all, FullRoute<?> route, ExecutePath path, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer<? super SyncPoint<S>, Throwable> callback)
+ Topologies forDecision(Node node, FullRoute<?> route, TxnId txnId)
+ {
+ return node.topology().withUnsyncedEpochs(route, txnId, txnId);
+ }
+
+ @Override
+ Topologies forExecution(Node node, FullRoute<?> route, TxnId txnId, Timestamp executeAt, Deps deps)
+ {
+ return node.topology().preciseEpochs(route, txnId.epoch(), executeAt.epoch());
+ }
+
+ @Override
+ public void execute(Node node, Topologies any, FullRoute<?> route, ExecutePath path, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer<? super SyncPoint<U>, Throwable> callback)
{
- ExecuteBlocking<S> execute = ExecuteBlocking.atQuorum(node, all, new SyncPoint<>(txnId, deps, (S)txn.keys(), route), executeAt);
+ Topologies all = forExecution(node, route, txnId, executeAt, deps);
+
+ ExecuteBlocking<U> execute = ExecuteBlocking.atQuorum(node, all, new SyncPoint<>(txnId, deps, (FullRoute<U>) route), executeAt);
execute.start();
addOrExecuteCallback(execute, callback);
}
- protected abstract void addOrExecuteCallback(ExecuteBlocking<S> execute, BiConsumer<? super SyncPoint<S>, Throwable> callback);
+ protected abstract void addOrExecuteCallback(ExecuteBlocking<U> execute, BiConsumer<? super SyncPoint<U>, Throwable> callback);
}
/*
@@ -289,7 +287,7 @@ public interface CoordinationAdapter<R>
* point is being executed and that eventually information will be known locally everywhere about the last
* sync point for the keys/ranges this sync point covered.
*/
- private static class AsyncInclusiveSyncPointAdapter<S extends Seekables<?, ?>> extends AbstractInclusiveSyncPointAdapter<S>
+ private static class AsyncInclusiveSyncPointAdapter<U extends Unseekable> extends AbstractInclusiveSyncPointAdapter<U>
{
private static final AsyncInclusiveSyncPointAdapter INSTANCE = new AsyncInclusiveSyncPointAdapter();
@@ -298,7 +296,7 @@ public interface CoordinationAdapter<R>
}
@Override
- protected void addOrExecuteCallback(ExecuteBlocking<S> execute, BiConsumer<? super SyncPoint<S>, Throwable> callback)
+ protected void addOrExecuteCallback(ExecuteBlocking<U> execute, BiConsumer<? super SyncPoint<U>, Throwable> callback)
{
// If this is the async adapter then we want to invoke the callback immediately
// and the caller can wait on the txn locally if they want
@@ -306,7 +304,7 @@ public interface CoordinationAdapter<R>
}
}
- private static class InclusiveSyncPointBlockingAdapter<S extends Seekables<?, ?>> extends AbstractInclusiveSyncPointAdapter<S>
+ private static class InclusiveSyncPointBlockingAdapter<U extends Unseekable> extends AbstractInclusiveSyncPointAdapter<U>
{
private static final InclusiveSyncPointBlockingAdapter INSTANCE = new InclusiveSyncPointBlockingAdapter();
@@ -315,13 +313,13 @@ public interface CoordinationAdapter<R>
}
@Override
- protected void addOrExecuteCallback(ExecuteBlocking<S> execute, BiConsumer<? super SyncPoint<S>, Throwable> callback)
+ protected void addOrExecuteCallback(ExecuteBlocking<U> execute, BiConsumer<? super SyncPoint<U>, Throwable> callback)
{
execute.addCallback(callback);
}
@Override
- public void persist(Node node, Topologies all, FullRoute<?> route, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, BiConsumer<? super SyncPoint<S>, Throwable> callback)
+ public void persist(Node node, Topologies any, FullRoute<?> route, Participants<?> participants, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, BiConsumer<? super SyncPoint<U>, Throwable> callback)
{
throw new UnsupportedOperationException();
}
diff --git a/accord-core/src/main/java/accord/coordinate/ExecuteEphemeralRead.java b/accord-core/src/main/java/accord/coordinate/ExecuteEphemeralRead.java
index bec5483e..e0f11de2 100644
--- a/accord-core/src/main/java/accord/coordinate/ExecuteEphemeralRead.java
+++ b/accord-core/src/main/java/accord/coordinate/ExecuteEphemeralRead.java
@@ -80,7 +80,7 @@ public class ExecuteEphemeralRead extends ReadCoordinator<ReadReply>
@Override
public void contact(Id to)
{
- node.send(to, new ReadEphemeralTxnData(to, allTopologies, txnId, txn.keys().toParticipants(), allTopologies.currentEpoch(), txn, deps, route), this);
+ node.send(to, new ReadEphemeralTxnData(to, allTopologies, txnId, route, allTopologies.currentEpoch(), txn, deps, route), this);
}
@Override
diff --git a/accord-core/src/main/java/accord/coordinate/ExecuteSyncPoint.java b/accord-core/src/main/java/accord/coordinate/ExecuteSyncPoint.java
index 503801ef..b1979f2c 100644
--- a/accord-core/src/main/java/accord/coordinate/ExecuteSyncPoint.java
+++ b/accord-core/src/main/java/accord/coordinate/ExecuteSyncPoint.java
@@ -18,7 +18,10 @@
package accord.coordinate;
+import java.util.function.Function;
+
import accord.api.Result;
+import accord.coordinate.CoordinationAdapter.Adapters;
import accord.coordinate.tracking.AbstractSimpleTracker;
import accord.coordinate.tracking.QuorumTracker;
import accord.coordinate.tracking.RequestStatus;
@@ -27,58 +30,127 @@ import accord.messages.ApplyThenWaitUntilApplied;
import accord.messages.Callback;
import accord.messages.ReadData;
import accord.messages.ReadData.ReadReply;
+import accord.messages.WaitUntilApplied;
import accord.primitives.Participants;
-import accord.primitives.Seekables;
+import accord.primitives.Range;
import accord.primitives.SyncPoint;
import accord.primitives.Timestamp;
import accord.primitives.Txn;
+import accord.primitives.Unseekable;
import accord.primitives.Writes;
import accord.topology.Topologies;
+import accord.utils.Invariants;
import accord.utils.async.AsyncResults.SettableResult;
-public abstract class ExecuteSyncPoint<S extends Seekables<?, ?>> extends SettableResult<SyncPoint<S>> implements Callback<ReadReply>
+import static accord.primitives.Txn.Kind.ExclusiveSyncPoint;
+
+public abstract class ExecuteSyncPoint<U extends Unseekable> extends SettableResult<SyncPoint<U>> implements Callback<ReadReply>
{
public static class SyncPointErased extends Throwable {}
- public static class ExecuteBlocking<S extends Seekables<?, ?>> extends ExecuteSyncPoint<S>
+ public static class ExecuteBlocking<U extends Unseekable> extends ExecuteSyncPoint<U>
{
private final Timestamp executeAt;
- public ExecuteBlocking(Node node, AbstractSimpleTracker<?> tracker, SyncPoint<S> syncPoint, Timestamp executeAt)
+ public ExecuteBlocking(Node node, SyncPoint<U> syncPoint, AbstractSimpleTracker<?> tracker, Timestamp executeAt)
{
- super(node, tracker, syncPoint);
+ super(node, syncPoint, tracker);
+ Invariants.checkArgument(!syncPoint.syncId.awaitsOnlyDeps());
this.executeAt = executeAt;
}
- public static <S extends Seekables<?, ?>> ExecuteBlocking<S> atQuorum(Node node, Topologies topologies, SyncPoint<S> syncPoint, Timestamp executeAt)
+ public static <U extends Unseekable> ExecuteBlocking<U> atQuorum(Node node, Topologies topologies, SyncPoint<U> syncPoint, Timestamp executeAt)
{
- return new ExecuteBlocking<>(node, new QuorumTracker(topologies), syncPoint, executeAt);
+ return new ExecuteBlocking<>(node, syncPoint, new QuorumTracker(topologies), executeAt);
}
@Override
public void start()
{
- Txn txn = node.agent().emptySystemTxn(syncPoint.syncId.kind(), syncPoint.keysOrRanges);
+ Txn txn = node.agent().emptySystemTxn(syncPoint.syncId.kind(), syncPoint.syncId.domain());
Writes writes = txn.execute(syncPoint.syncId, syncPoint.syncId, null);
Result result = txn.result(syncPoint.syncId, syncPoint.syncId, null);
node.send(tracker.topologies().nodes(), to -> {
- Seekables<?, ?> notify = to.equals(node.id()) ? null : syncPoint.keysOrRanges;
- Participants<?> participants = syncPoint.keysOrRanges.toParticipants();
- return new ApplyThenWaitUntilApplied(to, tracker.topologies(), executeAt, syncPoint.route(), syncPoint.syncId, txn, syncPoint.waitFor, participants, writes, result, notify);
+ Participants<?> participants = syncPoint.route.participants();
+ return new ApplyThenWaitUntilApplied(to, tracker.topologies(), executeAt, syncPoint.route(), syncPoint.syncId, txn, syncPoint.waitFor, participants, writes, result);
}, this);
}
}
+ public static class ExecuteExclusiveSyncPoint extends ExecuteSyncPoint<Range>
+ {
+ private long retryInFutureEpoch;
+ public ExecuteExclusiveSyncPoint(Node node, SyncPoint<Range> syncPoint, Function<Topologies, AbstractSimpleTracker<?>> trackerSupplier)
+ {
+ super(node, syncPoint, Adapters.exclusiveSyncPoint().forExecution(node, syncPoint.route(), syncPoint.syncId, syncPoint.syncId, syncPoint.waitFor), trackerSupplier);
+ Invariants.checkArgument(syncPoint.syncId.kind() == ExclusiveSyncPoint);
+ }
+
+ public ExecuteExclusiveSyncPoint(Node node, SyncPoint<Range> syncPoint, Function<Topologies, AbstractSimpleTracker<?>> trackerSupplier, AbstractSimpleTracker<?> tracker)
+ {
+ super(node, syncPoint, trackerSupplier, tracker);
+ Invariants.checkArgument(syncPoint.syncId.kind() == ExclusiveSyncPoint);
+ }
+
+ @Override
+ protected void start()
+ {
+ node.send(tracker.nodes(), to -> new WaitUntilApplied(to, tracker.topologies(), syncPoint.syncId, syncPoint.route, syncPoint.syncId.epoch()), this);
+ }
+
+ @Override
+ public synchronized void onSuccess(Node.Id from, ReadReply reply)
+ {
+ if (reply instanceof ReadData.ReadOkWithFutureEpoch)
+ retryInFutureEpoch = Math.max(retryInFutureEpoch, ((ReadData.ReadOkWithFutureEpoch) reply).futureEpoch);
+
+ super.onSuccess(from, reply);
+ }
+
+ @Override
+ protected void onSuccess()
+ {
+ if (retryInFutureEpoch > tracker.topologies().currentEpoch())
+ {
+ ExecuteExclusiveSyncPoint continuation = new ExecuteExclusiveSyncPoint(node, syncPoint, trackerSupplier, trackerSupplier.apply(node.topology().preciseEpochs(syncPoint.route(), tracker.topologies().currentEpoch(), retryInFutureEpoch)));
+ continuation.addCallback((success, failure) -> {
+ if (failure == null) trySuccess(success);
+ else tryFailure(failure);
+ });
+ continuation.start();
+ }
+ else
+ {
+ super.onSuccess();
+ }
+ }
+ }
+
final Node node;
+ final SyncPoint<U> syncPoint;
+
+ final Function<Topologies, AbstractSimpleTracker<?>> trackerSupplier;
final AbstractSimpleTracker<?> tracker;
- final SyncPoint<S> syncPoint;
private Throwable failures = null;
- ExecuteSyncPoint(Node node, AbstractSimpleTracker<?> tracker, SyncPoint<S> syncPoint)
+ ExecuteSyncPoint(Node node, SyncPoint<U> syncPoint, AbstractSimpleTracker<?> tracker)
{
- // TODO (required): this isn't correct, we need to potentially perform a second round if a dependency executes in a future epoch and we have lost ownership of that epoch
this.node = node;
+ this.syncPoint = syncPoint;
+ this.trackerSupplier = null;
this.tracker = tracker;
+ }
+
+ ExecuteSyncPoint(Node node, SyncPoint<U> syncPoint, Topologies topologies, Function<Topologies, AbstractSimpleTracker<?>> trackerSupplier)
+ {
+ this(node, syncPoint, trackerSupplier, trackerSupplier.apply(topologies));
+ }
+
+ ExecuteSyncPoint(Node node, SyncPoint<U> syncPoint, Function<Topologies, AbstractSimpleTracker<?>> trackerSupplier, AbstractSimpleTracker<?> tracker)
+ {
+ this.node = node;
this.syncPoint = syncPoint;
+ this.trackerSupplier = trackerSupplier;
+ this.tracker = tracker;
}
protected abstract void start();
@@ -103,7 +175,7 @@ public abstract class ExecuteSyncPoint<S extends Seekables<?, ?>> extends Settab
return;
case Invalid:
- tryFailure(new Invalidated(syncPoint.syncId, syncPoint.homeKey));
+ tryFailure(new Invalidated(syncPoint.syncId, syncPoint.route.homeKey()));
return;
}
}
@@ -126,7 +198,7 @@ public abstract class ExecuteSyncPoint<S extends Seekables<?, ?>> extends Settab
if (isDone()) return;
failures = FailureAccumulator.append(failures, failure);
if (tracker.recordFailure(from) == RequestStatus.Failed)
- tryFailure(FailureAccumulator.createFailure(failures, syncPoint.syncId, syncPoint.homeKey));
+ tryFailure(FailureAccumulator.createFailure(failures, syncPoint.syncId, syncPoint.route.homeKey()));
}
@Override
diff --git a/accord-core/src/main/java/accord/coordinate/ExecuteTxn.java b/accord-core/src/main/java/accord/coordinate/ExecuteTxn.java
index 3f689156..089b6816 100644
--- a/accord-core/src/main/java/accord/coordinate/ExecuteTxn.java
+++ b/accord-core/src/main/java/accord/coordinate/ExecuteTxn.java
@@ -147,7 +147,7 @@ public class ExecuteTxn extends ReadCoordinator<ReadReply>
if (failure == null)
{
Result result = txn.result(txnId, executeAt, data);
- CoordinationAdapter.Invoke.persist(adapter(), node, allTopologies, route, txnId, txn, executeAt, stableDeps, txn.execute(txnId, executeAt, data), result, callback);
+ adapter().persist(node, allTopologies, route, txnId, txn, executeAt, stableDeps, txn.execute(txnId, executeAt, data), result, callback);
}
else
{
diff --git a/accord-core/src/main/java/accord/coordinate/FetchCoordinator.java b/accord-core/src/main/java/accord/coordinate/FetchCoordinator.java
index 266321f2..063cf643 100644
--- a/accord-core/src/main/java/accord/coordinate/FetchCoordinator.java
+++ b/accord-core/src/main/java/accord/coordinate/FetchCoordinator.java
@@ -143,6 +143,7 @@ public abstract class FetchCoordinator
private Ranges needed;
private int inflight;
+ // TODO (required): use ExecuteExclusiveSyncPoint logic
protected FetchCoordinator(Node node, Ranges ranges, SyncPoint syncPoint, FetchRanges fetchRanges)
{
this.node = node;
@@ -151,6 +152,7 @@ public abstract class FetchCoordinator
this.fetchRanges = fetchRanges;
// TODO (expected): prioritise nodes that were members in the "prior" epoch also
// (by prior, we mean the prior epoch affecting ownership of this shard, not the prior numerical epoch)
+ // TODO (required): the sourceEpoch may be wrong, as a transaction may execute in a future epoch
Topology topology = node.topology().forEpoch(ranges, syncPoint.sourceEpoch()).get(0);
for (Node.Id id : topology.nodes())
{
diff --git a/accord-core/src/main/java/accord/coordinate/FetchData.java b/accord-core/src/main/java/accord/coordinate/FetchData.java
index 387a2d05..4efc272d 100644
--- a/accord-core/src/main/java/accord/coordinate/FetchData.java
+++ b/accord-core/src/main/java/accord/coordinate/FetchData.java
@@ -21,13 +21,14 @@ import java.util.function.BiConsumer;
import javax.annotation.Nullable;
import accord.local.Node;
-import accord.local.Status;
-import accord.local.Status.Known;
+import accord.primitives.Status;
+import accord.primitives.Known;
import accord.messages.CheckStatus;
import accord.messages.CheckStatus.CheckStatusOkFull;
import accord.messages.Propagate;
import accord.primitives.EpochSupplier;
import accord.primitives.FullRoute;
+import accord.primitives.Participants;
import accord.primitives.Ranges;
import accord.primitives.Route;
import accord.primitives.Timestamp;
@@ -38,6 +39,7 @@ import accord.utils.Invariants;
import javax.annotation.Nonnull;
import static accord.coordinate.Infer.InvalidateAndCallback.locallyInvalidateAndCallback;
+import static accord.primitives.EpochSupplier.constant;
import static accord.primitives.Route.castToRoute;
import static accord.primitives.Route.isFullRoute;
import static accord.primitives.Route.isRoute;
@@ -63,110 +65,142 @@ public class FetchData extends CheckShards<Route<?>>
}
}
- public static void fetch(Known fetch, Node node, TxnId txnId, Unseekables<?> someUnseekables, @Nullable EpochSupplier forLocalEpoch, @Nullable Timestamp executeAt, BiConsumer<? super FetchResult, Throwable> callback)
+ private static class FetchRequest
{
- if (someUnseekables.kind().isRoute()) fetch(fetch, node, txnId, castToRoute(someUnseekables), forLocalEpoch, executeAt, callback);
- else fetchViaSomeRoute(fetch, node, txnId, someUnseekables, forLocalEpoch, executeAt, callback);
+ final Known fetch;
+ final TxnId txnId;
+ final @Nullable Timestamp executeAt;
+ final long srcEpoch;
+ final Participants<?> fetchKeys;
+ final long lowEpoch, highEpoch;
+ final BiConsumer<? super FetchResult, Throwable> callback;
+
+ public FetchRequest(Known fetch, TxnId txnId, @Nullable Timestamp executeAt, Participants<?> fetchKeys, EpochSupplier lowEpoch, EpochSupplier highEpoch, BiConsumer<? super FetchResult, Throwable> callback)
+ {
+ this.fetch = fetch;
+ this.txnId = txnId;
+ this.executeAt = executeAt;
+ this.callback = callback;
+ this.srcEpoch = fetch.fetchEpoch(txnId, executeAt);
+ this.fetchKeys = fetchKeys;
+ this.lowEpoch = lowEpoch == null ? txnId.epoch() : lowEpoch.epoch();
+ this.highEpoch = highEpoch == null ? srcEpoch : highEpoch.epoch();
+ }
+
+ Ranges localRanges(Node node)
+ {
+ return node.topology().localRangesForEpochs(lowEpoch, highEpoch);
+ }
}
- public static void fetch(Known fetch, Node node, TxnId txnId, Route<?> route, @Nullable EpochSupplier forLocalEpoch, @Nullable Timestamp executeAt, BiConsumer<? super FetchResult, Throwable> callback)
+ public static void fetch(Known fetch, Node node, TxnId txnId, @Nullable Timestamp executeAt, Participants<?> someKeys, @Nullable EpochSupplier localLowEpoch, @Nullable EpochSupplier localHighEpoch, BiConsumer<? super FetchResult, Throwable> callback)
{
- long srcEpoch = fetch.fetchEpoch(txnId, executeAt);
+ FetchRequest request = new FetchRequest(fetch, txnId, executeAt, someKeys, localLowEpoch, localHighEpoch, callback);
+ if (someKeys.kind().isRoute()) fetch(node, castToRoute(someKeys), request);
+ else fetchViaSomeRoute(node, someKeys, request);
+ }
+
+ public static void fetch(Node node, Route<?> route, FetchRequest request)
+ {
+ long srcEpoch = request.srcEpoch;
if (!node.topology().hasEpoch(srcEpoch))
{
- node.withEpoch(srcEpoch, callback, () -> fetch(fetch, node, txnId, route, forLocalEpoch, executeAt, callback));
+ node.withEpoch(srcEpoch, request.callback, () -> fetch(node, route, request));
return;
}
Invariants.checkArgument(node.topology().hasEpoch(srcEpoch), "Unknown epoch %d, latest known is %d", srcEpoch, node.epoch());
- long toEpoch = Math.max(srcEpoch, forLocalEpoch == null ? 0 : forLocalEpoch.epoch());
- Ranges ranges = node.topology().localRangesForEpochs(txnId.epoch(), toEpoch);
+ Ranges ranges = request.localRanges(node);
if (!Route.isFullRoute(route))
{
- fetchWithIncompleteRoute(fetch, node, txnId, route, forLocalEpoch, executeAt, callback);
+ fetchWithIncompleteRoute(node, route, request);
}
else
{
Route<?> slicedRoute = route.slice(ranges);
- fetchInternal(ranges, fetch, node, txnId, slicedRoute, slicedRoute, executeAt, toEpoch, callback);
+ fetchInternal(node, ranges, slicedRoute, request);
}
}
/**
* Do not make an attempt to discern what keys need to be contacted; fetch from only the specific remote keys that were requested.
*/
- public static void fetchSpecific(Known fetch, Node node, TxnId txnId, Route<?> query, Unseekables<?> propagateTo, @Nullable EpochSupplier forLocalEpoch, @Nullable Timestamp executeAt, BiConsumer<? super FetchResult, Throwable> callback)
+ public static void fetchSpecific(Known fetch, Node node, TxnId txnId, Route<?> query, Route<?> maxRoute, Participants<?> localKeys, @Nullable EpochSupplier lowEpoch, @Nullable EpochSupplier highEpoch, @Nullable Timestamp executeAt, BiConsumer<? super FetchResult, Throwable> callback)
{
- long srcEpoch = fetch.fetchEpoch(txnId, executeAt);
+ fetchSpecific(node, query, maxRoute, new FetchRequest(fetch, txnId, executeAt, localKeys, lowEpoch, highEpoch, callback));
+ }
+
+ public static void fetchSpecific(Node node, Route<?> query, Route<?> maxRoute, FetchRequest request)
+ {
+ long srcEpoch = request.srcEpoch;
if (!node.topology().hasEpoch(srcEpoch))
{
- node.withEpoch(srcEpoch, callback, () -> fetchSpecific(fetch, node, txnId, query, propagateTo, forLocalEpoch, executeAt, callback));
+ node.withEpoch(srcEpoch, request.callback, () -> fetchSpecific(node, query, maxRoute, request));
return;
}
- Invariants.checkArgument(node.topology().hasEpoch(srcEpoch), "Unknown epoch %d, latest known is %d", srcEpoch, node.epoch());
- long toEpoch = Math.max(srcEpoch, forLocalEpoch == null ? 0 : forLocalEpoch.epoch());
- fetchData(fetch, node, txnId, query, propagateTo, srcEpoch, toEpoch, callback);
+ fetchData(node, query, maxRoute, request);
}
@SuppressWarnings({"unchecked", "rawtypes"})
- private static void fetchViaSomeRoute(Known fetch, Node node, TxnId txnId, Unseekables<?> someUnseekables, @Nullable EpochSupplier forLocalEpoch, @Nullable Timestamp executeAt, BiConsumer<? super FetchResult, Throwable> callback)
+ private static void fetchViaSomeRoute(Node node, Participants<?> someUnseekables, FetchRequest request)
{
- FindSomeRoute.findSomeRoute(node, txnId, someUnseekables, (foundRoute, fail) -> {
- if (fail != null) callback.accept(null, fail);
+ TxnId txnId = request.txnId;
+ FindSomeRoute.findSomeRoute(node, request.txnId, someUnseekables, (foundRoute, fail) -> {
+ if (fail != null) request.callback.accept(null, fail);
else if (foundRoute.route == null)
{
- reportRouteNotFound(node, txnId, executeAt, forLocalEpoch, someUnseekables, foundRoute.known, callback);
+ reportRouteNotFound(node, foundRoute.known, request);
}
else if (isFullRoute(foundRoute.route))
{
- fetch(fetch, node, txnId, Route.castToFullRoute(foundRoute.route), forLocalEpoch, executeAt, callback);
+ fetch(node, Route.castToFullRoute(foundRoute.route), request);
}
else if (isRoute(someUnseekables) && someUnseekables.containsAll(foundRoute.route))
{
// this is essentially a reentrancy check; we can only reach this point if we have already tried once to fetchSomeRoute
// (as a user-provided Route is used to fetchRoute, not fetchSomeRoute)
- reportRouteNotFound(node, txnId, executeAt, forLocalEpoch, someUnseekables, foundRoute.known, callback);
+ reportRouteNotFound(node, foundRoute.known, request);
}
else
{
Route<?> route = foundRoute.route;
if (isRoute(someUnseekables))
route = Route.merge(route, (Route)someUnseekables);
- fetch(fetch, node, txnId, route, forLocalEpoch, executeAt, callback);
+ fetch(node, route, request);
}
});
}
- private static void reportRouteNotFound(Node node, TxnId txnId, @Nullable Timestamp executeAt, @Nullable EpochSupplier forLocalEpoch, Unseekables<?> fetchKeys, Known found, BiConsumer<? super FetchResult, Throwable> callback)
+ private static void reportRouteNotFound(Node node, Known found, FetchRequest req)
{
- Invariants.checkState(executeAt == null);
+ Invariants.checkState(req.executeAt == null);
+ TxnId txnId = req.txnId;
switch (found.outcome)
{
default: throw new AssertionError("Unknown outcome: " + found.outcome);
case Invalidated:
- if (forLocalEpoch == null) forLocalEpoch = txnId;
- locallyInvalidateAndCallback(node, txnId, forLocalEpoch, fetchKeys, new FetchResult(found, fetchKeys, null), callback);
+ locallyInvalidateAndCallback(node, txnId, constant(req.lowEpoch), constant(req.highEpoch), req.fetchKeys, new FetchResult(found, req.fetchKeys, null), req.callback);
break;
case Unknown:
if (found.canProposeInvalidation())
{
- Invalidate.invalidate(node, txnId, fetchKeys, (outcome, throwable) -> {
+ Invalidate.invalidate(node, txnId, req.fetchKeys, (outcome, throwable) -> {
FetchResult result = null;
if (throwable == null)
{
Known achieved = Known.Invalidated;
- Unseekables<?> achievedTarget = fetchKeys, didNotAchieveTarget = null;
+ Unseekables<?> achievedTarget = req.fetchKeys, didNotAchieveTarget = null;
if (outcome.asProgressToken().status != Status.Invalidated)
{
- achievedTarget = fetchKeys.slice(0, 0);
- didNotAchieveTarget = fetchKeys;
+ achievedTarget = req.fetchKeys.slice(0, 0);
+ didNotAchieveTarget = req.fetchKeys;
achieved = Known.Nothing;
}
result = new FetchResult(achieved, achievedTarget, didNotAchieveTarget);
}
- callback.accept(result, throwable);
+ req.callback.accept(result, throwable);
});
break;
}
@@ -174,46 +208,35 @@ public class FetchData extends CheckShards<Route<?>>
case WasApply:
case Apply:
// TODO (expected): we may be stale
- callback.accept(new FetchResult(found, fetchKeys.slice(0, 0), fetchKeys), null);
+ req.callback.accept(new FetchResult(found, req.fetchKeys.slice(0, 0), req.fetchKeys), null);
}
}
- private static void fetchWithIncompleteRoute(Known fetch, Node node, TxnId txnId, Route<?> someRoute, @Nullable EpochSupplier forLocalEpoch, @Nullable Timestamp executeAt, BiConsumer<? super FetchResult, Throwable> callback)
+ private static void fetchWithIncompleteRoute(Node node, Route<?> someRoute, FetchRequest request)
{
- long srcEpoch = fetch.fetchEpoch(txnId, executeAt);
+ long srcEpoch = request.srcEpoch;
Invariants.checkArgument(node.topology().hasEpoch(srcEpoch), "Unknown epoch %d, latest known is %d", srcEpoch, node.epoch());
- FindRoute.findRoute(node, txnId, someRoute.withHomeKey(), (foundRoute, fail) -> {
- if (fail != null) callback.accept(null, fail);
- else if (foundRoute == null) fetchViaSomeRoute(fetch, node, txnId, someRoute, forLocalEpoch, executeAt, callback);
- else fetch(fetch, node, txnId, foundRoute.route, forLocalEpoch, foundRoute.executeAt, callback);
+ FindRoute.findRoute(node, request.txnId, someRoute.withHomeKey(), (foundRoute, fail) -> {
+ if (fail != null) request.callback.accept(null, fail);
+ else if (foundRoute == null) fetchViaSomeRoute(node, someRoute, request);
+ else fetch(node, foundRoute.route, request);
});
}
- public static void fetch(Known fetch, Node node, TxnId txnId, FullRoute<?> route, @Nullable EpochSupplier forLocalEpoch, @Nullable Timestamp executeAt, BiConsumer<? super FetchResult, Throwable> callback)
+ public static void fetch(Node node, FullRoute<?> route, FetchRequest request)
{
- node.withEpoch(executeAt, (ignore, withEpochFailure) -> {
- if (withEpochFailure != null)
- {
- callback.accept(null, CoordinationFailed.wrap(withEpochFailure));
- return;
- }
- long toEpoch = Math.max(fetch.fetchEpoch(txnId, executeAt), forLocalEpoch == null ? 0 : forLocalEpoch.epoch());
- Ranges ranges = node.topology().localRangesForEpochs(txnId.epoch(), toEpoch);
-
- Route<?> slicedRoute = route.slice(ranges);
- fetchInternal(ranges, fetch, node, txnId, slicedRoute, slicedRoute, executeAt, toEpoch, callback);
+ node.withEpoch(request.srcEpoch, request.callback, () -> {
+ Ranges ranges = request.localRanges(node);
+ fetchInternal(node, ranges, route, request);
});
}
- private static Object fetchInternal(Ranges ranges, Known target, Node node, TxnId txnId, Route<?> route, Unseekables<?> propagateTo, @Nullable Timestamp executeAt, long forLocalEpoch, BiConsumer<? super FetchResult, Throwable> callback)
+ private static Object fetchInternal(Node node, Ranges ranges, Route<?> route, FetchRequest request)
{
- long srcEpoch = target.fetchEpoch(txnId, executeAt);
+ long srcEpoch = request.srcEpoch;
Invariants.checkArgument(node.topology().hasEpoch(srcEpoch), "Unknown epoch %d, latest known is %d", srcEpoch, node.epoch());
- Route<?> fetch = route.slice(ranges);
- return fetchData(target, node, txnId, fetch, propagateTo, srcEpoch, forLocalEpoch, (result, fail) -> {
- if (fail != null) callback.accept(null, fail);
- else callback.accept(result, null);
- });
+ Route<?> slicedRoute = route.slice(ranges);
+ return fetchData(node, slicedRoute, route, request);
}
final BiConsumer<? super FetchResult, Throwable> callback;
@@ -221,37 +244,48 @@ public class FetchData extends CheckShards<Route<?>>
* The epoch until which we want to persist any response for locally
*/
final Known target;
+ final Route<?> maxRoute;
// to support cases where a later epoch that ultimately does not participate in execution has a vestigial entry
// (i.e. if preaccept/accept contact a later epoch than execution is decided for)
- final long toEpoch;
+ final long lowEpoch, highEpoch;
final Unseekables<?> propagateTo;
- private FetchData(Node node, Known target, TxnId txnId, Route<?> route, Unseekables<?> propagateTo, long sourceEpoch, long toEpoch, BiConsumer<? super FetchResult, Throwable> callback)
+ private FetchData(Node node, Known target, TxnId txnId, Route<?> route, Route<?> maxRoute, Unseekables<?> propagateTo, long sourceEpoch, long lowEpoch, long highEpoch, BiConsumer<? super FetchResult, Throwable> callback)
{
- this(node, target, txnId, route, route.withHomeKey(), propagateTo, sourceEpoch, toEpoch, callback);
+ this(node, target, txnId, route, route.withHomeKey(), maxRoute, propagateTo, sourceEpoch, lowEpoch, highEpoch, callback);
}
- private FetchData(Node node, Known target, TxnId txnId, Route<?> route, Route<?> routeWithHomeKey, Unseekables<?> propagateTo, long sourceEpoch, long toEpoch, BiConsumer<? super FetchResult, Throwable> callback)
+ private FetchData(Node node, Known target, TxnId txnId, Route<?> route, Route<?> routeWithHomeKey, Route<?> maxRoute, Unseekables<?> propagateTo, long sourceEpoch, long lowEpoch, long highEpoch, BiConsumer<? super FetchResult, Throwable> callback)
{
// TODO (desired, efficiency): restore behaviour of only collecting info if e.g. Committed or Executed
super(node, txnId, routeWithHomeKey, sourceEpoch, CheckStatus.IncludeInfo.All);
this.propagateTo = propagateTo;
+ this.lowEpoch = lowEpoch;
+ this.maxRoute = maxRoute;
Invariants.checkArgument(routeWithHomeKey.contains(route.homeKey()), "route %s does not contain %s", routeWithHomeKey, route.homeKey());
this.target = target;
- this.toEpoch = toEpoch;
+ this.highEpoch = highEpoch;
this.callback = callback;
}
- private static FetchData fetchData(Known sufficientStatus, Node node, TxnId txnId, Route<?> route, Unseekables<?> propagateTo, long sourceEpoch, long toEpoch, BiConsumer<? super FetchResult, Throwable> callback)
+ private static FetchData fetchData(Node node, Route<?> route, Route<?> maxRoute, FetchRequest req)
{
- Invariants.checkState(!propagateTo.isEmpty());
- FetchData fetch = new FetchData(node, sufficientStatus, txnId, route, propagateTo, sourceEpoch, toEpoch, callback);
+ Invariants.checkState(!req.fetchKeys.isEmpty());
+ FetchData fetch = new FetchData(node, req.fetch, req.txnId, route, maxRoute, req.fetchKeys, req.srcEpoch, req.lowEpoch, req.highEpoch, req.callback);
fetch.start();
return fetch;
}
+ private static FetchData fetchData(Node node, Known fetch, TxnId txnId, Route<?> route, Route<?> maxRoute, Unseekables<?> localKeys, long sourceEpoch, long lowEpoch, long highEpoch, BiConsumer<? super FetchResult, Throwable> callback)
+ {
+ Invariants.checkState(!localKeys.isEmpty());
+ FetchData fetchData = new FetchData(node, fetch, txnId, route, maxRoute, localKeys, sourceEpoch, lowEpoch, highEpoch, callback);
+ fetchData.start();
+ return fetchData;
+ }
+
protected Route<?> query()
{
return route;
@@ -273,7 +307,7 @@ public class FetchData extends CheckShards<Route<?>>
protected boolean isSufficient(Route<?> scope, CheckStatus.CheckStatusOk ok)
{
- return target.isSatisfiedBy(ok.knownFor(scope.participants()));
+ return target.isSatisfiedBy(ok.knownFor(txnId, scope.participants()));
}
@Override
@@ -293,7 +327,7 @@ public class FetchData extends CheckShards<Route<?>>
}
// TODO (expected): should we automatically trigger a new fetch if we find executeAt but did not request enough information? would be more rob ust
- Propagate.propagate(node, txnId, sourceEpoch, toEpoch, success.withQuorum, query(), propagateTo, target, (CheckStatusOkFull) merged, callback);
+ Propagate.propagate(node, txnId, sourceEpoch, lowEpoch, highEpoch, success.withQuorum, query(), propagateTo, target, (CheckStatusOkFull) merged, callback);
}
}
}
diff --git a/accord-core/src/main/java/accord/coordinate/FetchMaxConflict.java b/accord-core/src/main/java/accord/coordinate/FetchMaxConflict.java
index 677c40fc..325fee5c 100644
--- a/accord-core/src/main/java/accord/coordinate/FetchMaxConflict.java
+++ b/accord-core/src/main/java/accord/coordinate/FetchMaxConflict.java
@@ -26,7 +26,7 @@ import accord.messages.Callback;
import accord.messages.GetMaxConflict;
import accord.messages.GetMaxConflict.GetMaxConflictOk;
import accord.primitives.FullRoute;
-import accord.primitives.Seekables;
+import accord.primitives.Routables;
import accord.primitives.Timestamp;
import accord.topology.Topologies;
import accord.utils.async.AsyncResult;
@@ -44,25 +44,23 @@ import static accord.coordinate.tracking.RequestStatus.Success;
public class FetchMaxConflict extends AbstractCoordinatePreAccept<Timestamp, GetMaxConflictOk>
{
final QuorumTracker tracker;
- final Seekables<?, ?> keysOrRanges;
Timestamp maxConflict;
long executionEpoch;
- private FetchMaxConflict(Node node, FullRoute<?> route, Seekables<?, ?> keysOrRanges, long executionEpoch)
+ private FetchMaxConflict(Node node, FullRoute<?> route, long executionEpoch)
{
- this(node, route, keysOrRanges, executionEpoch, node.topology().withUnsyncedEpochs(route, executionEpoch, executionEpoch));
+ this(node, route, executionEpoch, node.topology().withUnsyncedEpochs(route, executionEpoch, executionEpoch));
}
- private FetchMaxConflict(Node node, FullRoute<?> route, Seekables<?, ?> keysOrRanges, long executionEpoch, Topologies topologies)
+ private FetchMaxConflict(Node node, FullRoute<?> route, long executionEpoch, Topologies topologies)
{
super(node, route, null, topologies);
- this.keysOrRanges = keysOrRanges;
this.maxConflict = Timestamp.NONE;
this.executionEpoch = executionEpoch;
this.tracker = new QuorumTracker(topologies);
}
- public static AsyncResult<Timestamp> fetchMaxConflict(Node node, Seekables<?, ?> keysOrRanges)
+ public static AsyncResult<Timestamp> fetchMaxConflict(Node node, Routables<?> keysOrRanges)
{
long epoch = node.epoch();
FullRoute<?> route = node.computeRoute(epoch, keysOrRanges);
@@ -70,22 +68,15 @@ public class FetchMaxConflict extends AbstractCoordinatePreAccept<Timestamp, Get
TopologyMismatch mismatch = TopologyMismatch.checkForMismatch(node.topology().globalForEpoch(epoch), null, route.homeKey(), keysOrRanges);
if (mismatch != null)
return AsyncResults.failure(mismatch);
- FetchMaxConflict coordinate = new FetchMaxConflict(node, route, keysOrRanges, epoch);
+ FetchMaxConflict coordinate = new FetchMaxConflict(node, route, epoch);
coordinate.start();
return coordinate;
}
-
- @Override
- Seekables<?, ?> keysOrRanges()
- {
- return keysOrRanges;
- }
-
@Override
void contact(Collection<Node.Id> nodes, Topologies topologies, Callback<GetMaxConflictOk> callback)
{
- node.send(nodes, to -> new GetMaxConflict(to, topologies, route, keysOrRanges, executionEpoch), callback);
+ node.send(nodes, to -> new GetMaxConflict(to, topologies, route, executionEpoch), callback);
}
@Override
diff --git a/accord-core/src/main/java/accord/coordinate/FindSomeRoute.java b/accord-core/src/main/java/accord/coordinate/FindSomeRoute.java
index 19060ed9..0d37da40 100644
--- a/accord-core/src/main/java/accord/coordinate/FindSomeRoute.java
+++ b/accord-core/src/main/java/accord/coordinate/FindSomeRoute.java
@@ -21,20 +21,20 @@ package accord.coordinate;
import java.util.function.BiConsumer;
import accord.local.Node;
-import accord.local.Status.Known;
+import accord.primitives.Known;
import accord.messages.CheckStatus.CheckStatusOk;
import accord.messages.CheckStatus.IncludeInfo;
-import accord.messages.CheckStatus.WithQuorum;
+import accord.primitives.WithQuorum;
+import accord.primitives.Participants;
import accord.primitives.Route;
-import accord.primitives.Unseekables;
import accord.primitives.TxnId;
-import static accord.local.Status.Known.Nothing;
+import static accord.primitives.Known.Nothing;
/**
* Find the homeKey of a txnId with some known keys
*/
-public class FindSomeRoute extends CheckShards<Unseekables<?>>
+public class FindSomeRoute extends CheckShards<Participants<?>>
{
static class Result
{
@@ -51,17 +51,22 @@ public class FindSomeRoute extends CheckShards<Unseekables<?>>
}
final BiConsumer<Result, Throwable> callback;
- FindSomeRoute(Node node, TxnId txnId, Unseekables<?> unseekables, BiConsumer<Result, Throwable> callback)
+ FindSomeRoute(Node node, TxnId txnId, Participants<?> unseekables, BiConsumer<Result, Throwable> callback)
{
super(node, txnId, unseekables, IncludeInfo.Route);
this.callback = callback;
}
- public static FindSomeRoute findSomeRoute(Node node, TxnId txnId, Unseekables<?> unseekables, BiConsumer<Result, Throwable> callback)
+ public static void findSomeRoute(Node node, TxnId txnId, Participants<?> unseekables, BiConsumer<Result, Throwable> callback)
{
+ if (!node.topology().hasEpoch(txnId.epoch()))
+ {
+ node.withEpoch(txnId.epoch(), callback, () -> findSomeRoute(node, txnId, unseekables, callback));
+ return;
+ }
+
FindSomeRoute findSomeRoute = new FindSomeRoute(node, txnId, unseekables, callback);
findSomeRoute.start();
- return findSomeRoute;
}
@Override
@@ -75,6 +80,6 @@ public class FindSomeRoute extends CheckShards<Unseekables<?>>
{
if (failure != null) callback.accept(null, failure);
else if (merged == null) callback.accept(new Result(null, Nothing, success.withQuorum), null);
- else callback.accept(new Result(merged.route, merged.finish(this.route, success.withQuorum).knownFor(this.route), success.withQuorum), null);
+ else callback.accept(new Result(merged.route, merged.finish(this.route, success.withQuorum).knownFor(txnId, this.route), success.withQuorum), null);
}
}
diff --git a/accord-core/src/main/java/accord/coordinate/Infer.java b/accord-core/src/main/java/accord/coordinate/Infer.java
index 4b41408d..787e45a7 100644
--- a/accord-core/src/main/java/accord/coordinate/Infer.java
+++ b/accord-core/src/main/java/accord/coordinate/Infer.java
@@ -19,207 +19,61 @@
package accord.coordinate;
import java.util.function.BiConsumer;
-import javax.annotation.Nullable;
-import accord.local.Cleanup;
import accord.local.Command;
import accord.local.Commands;
import accord.local.Node;
import accord.local.PreLoadContext;
-import accord.local.RedundantStatus;
import accord.local.SafeCommand;
import accord.local.SafeCommandStore;
-import accord.local.Status;
-import accord.local.Status.Known;
-import accord.messages.CheckStatus.FoundKnown;
-import accord.messages.CheckStatus.FoundKnownMap;
+import accord.primitives.Status;
+import accord.primitives.Known;
+import accord.local.StoreParticipants;
import accord.primitives.EpochSupplier;
-import accord.primitives.Range;
-import accord.primitives.Ranges;
-import accord.primitives.Route;
-import accord.primitives.Timestamp;
+import accord.primitives.Participants;
import accord.primitives.TxnId;
import accord.primitives.Unseekables;
import accord.utils.Invariants;
import accord.utils.MapReduceConsume;
-import static accord.coordinate.Infer.InvalidIf.IfPreempted;
-import static accord.coordinate.Infer.InvalidIf.IfQuorum;
-import static accord.coordinate.Infer.InvalidIf.NotInvalid;
-import static accord.coordinate.Infer.InvalidIf.NotKnown;
-import static accord.coordinate.Infer.InvalidIfNot.IfUndecided;
-import static accord.coordinate.Infer.InvalidIfNot.IfUnknown;
import static accord.local.PreLoadContext.contextFor;
-import static accord.local.Status.Durability.Majority;
-import static accord.local.Status.PreApplied;
-import static accord.local.Status.PreCommitted;
+import static accord.primitives.Status.PreCommitted;
import static accord.primitives.Route.castToRoute;
import static accord.primitives.Route.isRoute;
-import static accord.utils.Invariants.illegalState;
// TODO (required): dedicated randomised testing of all inferences
public class Infer
{
- public enum InvalidIfNot
+ public enum InvalidIf
{
/**
* There is no information to suggest the command is invalid
*/
- NotKnownToBeInvalid(NotKnown, NotKnown),
+ NotKnownToBeInvalid,
/**
- * If the command has not been preaccepted on a majority of any shard and
- * the command's original coordinator had been preempted prior to all responses we rely upon
- * (so we are not racing with it)
+ * If the command has not had its execution timestamp committed on any shard.
*/
- IfUnknownAndPreempted(IfPreempted, NotKnown),
-
- /**
- * If the command has not had its execution timestamp agreed on any shard and
- * the command's original coordinator had been preempted prior to all responses we rely upon
- * (so we are not racing with it)
- */
- IfUndecidedAndPreempted(IfPreempted, IfPreempted),
-
- /**
- * If the command has not been preaccepted on a majority of any shard
- */
- IfUnknown(IfQuorum, NotKnown),
-
- /**
- * If the command has not had its execution timestamp agreed on any shard and
- * the command's original coordinator had been preempted prior to all responses we rely upon
- * (so we are not racing with it)
- */
- IfUnknownOrIfUndecidedAndPreempted(IfQuorum, IfPreempted),
-
- /**
- * If the command has not had its execution timestamp agreed on any shard
- */
- IfUndecided(IfQuorum, IfQuorum),
+ IfUncommitted,
/**
* This command is known to be decided, so it is a logic bug if it is inferred elsewhere to be invalid.
*/
- IsNotInvalid(NotInvalid, NotInvalid);
+ IsNotInvalid;
- final InvalidIf unknown, undecided;
- private static final InvalidIfNot[] LOOKUP;
- private static final int invalidIfs = InvalidIf.values().length - 1;
-
- static
- {
- LOOKUP = new InvalidIfNot[invalidIfs * invalidIfs];
- InvalidIfNot[] invalidIfNot = InvalidIfNot.values();
- for (InvalidIfNot ifNot : invalidIfNot)
- {
- if (ifNot != IsNotInvalid)
- LOOKUP[ifNot.unknown.ordinal() * invalidIfs + ifNot.undecided.ordinal()] = ifNot;
- }
- }
-
- InvalidIfNot(InvalidIf unknown, InvalidIf undecided)
- {
- this.unknown = unknown;
- this.undecided = undecided;
- }
-
- public static boolean isMax(InvalidIfNot that)
+ public static boolean isMax(InvalidIf that)
{
return that == IsNotInvalid;
}
- public InvalidIfNot atLeast(InvalidIfNot that)
- {
- return lookup(atLeast(this.unknown, that.unknown), atLeast(this.undecided, that.undecided));
- }
-
- public InvalidIfNot reduce(InvalidIfNot that)
- {
- return lookup(reduce(this.unknown, that.unknown), reduce(this.undecided, that.undecided));
- }
-
- private InvalidIfNot lookup(InvalidIf unknown, InvalidIf undecided)
- {
- if (unknown == NotInvalid)
- return IsNotInvalid;
- return LOOKUP[unknown.ordinal() * invalidIfs + undecided.ordinal()];
- }
-
- private static InvalidIf atLeast(InvalidIf a, InvalidIf b)
- {
- if (a == NotInvalid || b == NotInvalid) return NotInvalid;
- if (a == b) return a;
- return IfPreempted;
- }
-
- private static InvalidIf reduce(InvalidIf a, InvalidIf b)
- {
- return a.compareTo(b) <= 0 ? a : b;
- }
-
- public boolean inferInvalidWithQuorum(IsPreempted isPreempted, Known minKnown)
+ public InvalidIf atLeast(InvalidIf that)
{
- return inferInvalidWithQuorum(undecided, isPreempted, !minKnown.isDecided())
- || inferInvalidWithQuorum(unknown, isPreempted, !minKnown.hasDefinitionBeenKnown());
+ return this.compareTo(that) >= 0 ? this : that;
}
- private static boolean inferInvalidWithQuorum(InvalidIf invalidIf, IsPreempted isPreempted, boolean hasCondition)
+ public boolean inferInvalidWithQuorum(Known known)
{
- if (!hasCondition)
- return false;
-
- switch (invalidIf)
- {
- default: throw new AssertionError("Unhandled InvalidIf: " + invalidIf);
- case NotInvalid:
- case NotKnown:
- break;
- case IfQuorum:
- return true;
- case IfPreempted:
- if (isPreempted == IsPreempted.Preempted)
- return true;
- }
-
- return false;
- }
- }
-
- enum InvalidIf
- {
- NotKnown,
-
- /**
- * We did not have a quorum of responses with the associated lower bound, so we require that the command has been preempted at a quorum
- */
- IfPreempted,
-
- /**
- * If we obtain a quorum of responses with the associated lower bound, we can infer the command is invalidated if it has not been witnessed at the lower bound
- */
- IfQuorum,
-
- /**
- * Definitely not invalid
- */
- NotInvalid
- }
-
- // only valid with a quorum of responses
- public enum IsPreempted
- {
- NotPreempted, MaybePreempted, Preempted;
-
- public IsPreempted merge(IsPreempted that)
- {
- if (this == that) return this;
- return MaybePreempted;
- }
-
- public IsPreempted validForBoth(IsPreempted that)
- {
- return this.compareTo(that) <= 0 ? this : that;
+ return this == IfUncommitted && !known.isDecided();
}
}
@@ -229,17 +83,18 @@ public class Infer
final TxnId txnId;
// TODO (expected): more consistent handling of transactions that only MAY intersect a commandStore
// (e.g. dependencies from an earlier epoch that have not yet committed, or commands that are proposed to execute in a later epoch than eventually agreed)
- final EpochSupplier untilEpoch;
- final Unseekables<?> someUnseekables;
+ final EpochSupplier lowEpoch, highEpoch;
+ final Participants<?> participants;
final T param;
final BiConsumer<T, Throwable> callback;
- private CleanupAndCallback(Node node, TxnId txnId, EpochSupplier untilEpoch, Unseekables<?> someUnseekables, T param, BiConsumer<T, Throwable> callback)
+ private CleanupAndCallback(Node node, TxnId txnId, EpochSupplier lowEpoch, EpochSupplier highEpoch, Participants<?> participants, T param, BiConsumer<T, Throwable> callback)
{
this.node = node;
this.txnId = txnId;
- this.untilEpoch = untilEpoch;
- this.someUnseekables = someUnseekables;
+ this.lowEpoch = lowEpoch;
+ this.highEpoch = highEpoch;
+ this.participants = participants;
this.param = param;
this.callback = callback;
}
@@ -247,15 +102,16 @@ public class Infer
void start()
{
PreLoadContext loadContext = contextFor(txnId);
- Unseekables<?> propagateTo = isRoute(someUnseekables) ? castToRoute(someUnseekables).withHomeKey() : someUnseekables;
- node.mapReduceConsumeLocal(loadContext, propagateTo, txnId.epoch(), untilEpoch.epoch(), this);
+ Unseekables<?> propagateTo = isRoute(participants) ? castToRoute(participants).withHomeKey() : participants;
+ node.mapReduceConsumeLocal(loadContext, propagateTo, lowEpoch.epoch(), highEpoch.epoch(), this);
}
@Override
public Void apply(SafeCommandStore safeStore)
{
// we're applying an invalidation, so the record will not be cleaned up until the whole range is truncated
- return apply(safeStore, safeStore.get(txnId, untilEpoch, someUnseekables));
+ StoreParticipants participants = StoreParticipants.invalidate(safeStore, this.participants, txnId);
+ return apply(safeStore, safeStore.get(txnId, participants));
}
abstract Void apply(SafeCommandStore safeStore, SafeCommand safeCommand);
@@ -273,16 +129,17 @@ public class Infer
}
}
+ // TODO (required, consider): low and high bounds are correct?
static class InvalidateAndCallback<T> extends CleanupAndCallback<T>
{
- private InvalidateAndCallback(Node node, TxnId txnId, EpochSupplier untilEpoch, Unseekables<?> someUnseekables, T param, BiConsumer<T, Throwable> callback)
+ private InvalidateAndCallback(Node node, TxnId txnId, EpochSupplier lowEpoch, EpochSupplier highEpoch, Participants<?> someUnseekables, T param, BiConsumer<T, Throwable> callback)
{
- super(node, txnId, untilEpoch, someUnseekables, param, callback);
+ super(node, txnId, lowEpoch, highEpoch, someUnseekables, param, callback);
}
- public static <T> void locallyInvalidateAndCallback(Node node, TxnId txnId, EpochSupplier untilEpoch, Unseekables<?> someUnseekables, T param, BiConsumer<T, Throwable> callback)
+ public static <T> void locallyInvalidateAndCallback(Node node, TxnId txnId, EpochSupplier lowEpoch, EpochSupplier highEpoch, Participants<?> someUnseekables, T param, BiConsumer<T, Throwable> callback)
{
- new InvalidateAndCallback<T>(node, txnId, untilEpoch, someUnseekables, param, callback).start();
+ new InvalidateAndCallback<T>(node, txnId, lowEpoch, highEpoch, someUnseekables, param, callback).start();
}
@Override
@@ -292,123 +149,8 @@ public class Infer
Command command = safeCommand.current();
// TODO (required, consider): consider the !command.hasBeen(PreCommitted) condition
Invariants.checkState(!command.hasBeen(PreCommitted) || command.hasBeen(Status.Truncated), "Unexpected status for %s", command);
- Commands.commitInvalidate(safeStore, safeCommand, someUnseekables);
- return null;
- }
- }
-
- /**
- * Erase if it is safe to do so, i.e. if Infer.safeToCleanup permits it.
- */
- static class SafeEraseAndCallback<T> extends CleanupAndCallback<T>
- {
- private SafeEraseAndCallback(Node node, TxnId txnId, EpochSupplier untilEpoch, Unseekables<?> someUnseekables, T param, BiConsumer<T, Throwable> callback)
- {
- super(node, txnId, untilEpoch, someUnseekables, param, callback);
- }
-
- public static <T> void safeEraseAndCallback(Node node, TxnId txnId, EpochSupplier untilEpoch, Unseekables<?> someUnseekables, T param, BiConsumer<T, Throwable> callback)
- {
- if (!Route.isRoute(someUnseekables)) callback.accept(param, null);
- else new SafeEraseAndCallback<>(node, txnId, untilEpoch, someUnseekables, param, callback).start();
- }
-
- @Override
- Void apply(SafeCommandStore safeStore, SafeCommand safeCommand)
- {
- Command command = safeCommand.current();
- // TODO (required, consider): introduce a special form of Erased where we do not imply the phase is "Cleanup"
- if (!command.hasBeen(PreApplied) && safeToCleanup(safeStore, command, Route.castToRoute(someUnseekables), null))
- Commands.setErased(safeStore, safeCommand);
+ Commands.commitInvalidate(safeStore, safeCommand, participants);
return null;
}
}
-
- public static FoundKnownMap withInvalidIfNot(SafeCommandStore safeStore, TxnId txnId, Unseekables<?> localKeys, Unseekables<?> maxKeys, FoundKnown known)
- {
- if (isRoute(maxKeys))
- maxKeys = castToRoute(maxKeys).withHomeKey();
-
- if (safeStore.commandStore().globalDurability(txnId).compareTo(Majority) >= 0)
- {
- InvalidIfNot invalidIfNot = safeStore.commandStore().isRejectedIfNotPreAccepted(txnId, maxKeys) ? IfUnknown : IfUndecided;
- return FoundKnownMap.create(maxKeys, known.withAtLeast(invalidIfNot));
- }
-
- // TODO (required): document the cleanup semantics that make this safe and tie the locations together preferably by compiler
- // (in essence iirc we permit
-
- Ranges coordinateRanges = safeStore.ranges().allAt(txnId.epoch());
- FoundKnownMap map = FoundKnownMap.create(localKeys, known);
- if (Cleanup.isSafeToCleanup(safeStore.commandStore().durableBefore(), txnId, coordinateRanges))
- {
- // it is safe to cleanup for all keys we own, as they are all known to have UniversalOrInvalidated durability past the TxnId in question
- map = FoundKnownMap.merge(map, FoundKnownMap.create(localKeys, known.withAtLeast(IfUndecided)));
- }
-
- // TODO (desired): limit to local participants to avoid O(n2) work across cluster
- class Builder extends FoundKnownMap.Builder
- {
- int minIndex;
- public Builder(boolean inclusiveEnds, int capacity)
- {
- super(inclusiveEnds, capacity);
- }
- }
- Builder builder = new Builder(maxKeys.get(0).asRange().endInclusive(), 2 * maxKeys.size());
- safeStore.commandStore().durableBefore().foldl(maxKeys, (e, b, q, id, i, j, k) -> {
- if (e.majorityBefore.compareTo(id) > 0)
- {
- i = Math.max(i, b.minIndex);
- while (i < j)
- {
- Range range = q.get(i++).asRange();
- b.append(range.start(), FoundKnown.Nothing.withAtLeast(IfUndecided), (i1, i2) -> { throw illegalState(); });
- b.append(range.end(), null, (i1, i2) -> { throw illegalState(); });
- }
- b.minIndex = i;
- }
- return b;
- }, builder, maxKeys, txnId, i -> false);
-
- if (!builder.isEmpty())
- map = FoundKnownMap.merge(map, builder.build());
-
- return map;
- }
-
- public static boolean safeToCleanup(SafeCommandStore safeStore, Command command, Route<?> fetchedWith, @Nullable Timestamp executeAt)
- {
- Invariants.checkArgument(fetchedWith != null || command.route() != null);
- TxnId txnId = command.txnId();
- Route<?> route = Route.merge(command.route(), (Route)fetchedWith);
-
- if (!Route.isFullRoute(route))
- return false;
-
- // TODO (required): is it safe to cleanup without an executeAt? We don't know for sure which ranges it might participate in.
- // We can infer the upper bound of execution by the "execution" of any ExclusiveSyncPoint used to infer the invalidation.
- // We should begin evaluating and tracking this.
- executeAt = command.executeAtIfKnown(Timestamp.nonNullOrMax(executeAt, txnId));
- Ranges coordinateRanges = safeStore.ranges().coordinates(txnId);
- Ranges acceptRanges = executeAt.epoch() == txnId.epoch() ? coordinateRanges : safeStore.ranges().allBetween(txnId, executeAt);
- if (!route.participatesIn(coordinateRanges) && !route.participatesIn(acceptRanges))
- return true;
-
- RedundantStatus status = safeStore.commandStore().redundantBefore().status(txnId, executeAt, route.participants());
- switch (status)
- {
- default: throw new AssertionError("Unhandled RedundantStatus: " + status);
- case NOT_OWNED:
- case LIVE:
- case REDUNDANT_PRE_BOOTSTRAP_OR_STALE:
- case PARTIALLY_PRE_BOOTSTRAP_OR_STALE:
- return false;
- case LOCALLY_REDUNDANT:
- case SHARD_REDUNDANT:
- Invariants.checkState(!command.hasBeen(PreCommitted));
- case PRE_BOOTSTRAP_OR_STALE:
- return true;
- }
- }
}
diff --git a/accord-core/src/main/java/accord/coordinate/Invalidate.java b/accord-core/src/main/java/accord/coordinate/Invalidate.java
index f1af5287..36db93cb 100644
--- a/accord-core/src/main/java/accord/coordinate/Invalidate.java
+++ b/accord-core/src/main/java/accord/coordinate/Invalidate.java
@@ -27,7 +27,6 @@ import accord.local.Node.Id;
import accord.messages.Commit;
import accord.local.*;
import accord.primitives.*;
-import accord.topology.Shard;
import accord.topology.Topologies;
import accord.api.RoutingKey;
@@ -41,8 +40,8 @@ import javax.annotation.Nullable;
import static accord.coordinate.Propose.Invalidate.proposeInvalidate;
import static accord.local.PreLoadContext.contextFor;
-import static accord.local.Status.Accepted;
-import static accord.local.Status.PreAccepted;
+import static accord.primitives.Status.Accepted;
+import static accord.primitives.Status.PreAccepted;
import static accord.primitives.ProgressToken.INVALIDATED;
import static accord.primitives.ProgressToken.TRUNCATED_DURABLE_OR_INVALIDATED;
import static accord.utils.Invariants.illegalState;
@@ -52,7 +51,7 @@ public class Invalidate implements Callback<InvalidateReply>
private final Node node;
private final Ballot ballot;
private final TxnId txnId;
- private final Unseekables<?> invalidateWith;
+ private final Participants<?> invalidateWith;
private final BiConsumer<Outcome, Throwable> callback;
private boolean isDone;
@@ -63,7 +62,7 @@ public class Invalidate implements Callback<InvalidateReply>
private final InvalidationTracker tracker;
private Throwable failure;
- private Invalidate(Node node, Ballot ballot, TxnId txnId, Unseekables<?> invalidateWith, boolean transitivelyInvokedByPriorInvalidation, BiConsumer<Outcome, Throwable> callback)
+ private Invalidate(Node node, Ballot ballot, TxnId txnId, Participants<?> invalidateWith, boolean transitivelyInvokedByPriorInvalidation, BiConsumer<Outcome, Throwable> callback)
{
this.callback = callback;
this.node = node;
@@ -78,12 +77,12 @@ public class Invalidate implements Callback<InvalidateReply>
this.replies = new InvalidateReply[topology.nodes().size()];
}
- public static Invalidate invalidate(Node node, TxnId txnId, Unseekables<?> invalidateWith, BiConsumer<Outcome, Throwable> callback)
+ public static Invalidate invalidate(Node node, TxnId txnId, Participants<?> invalidateWith, BiConsumer<Outcome, Throwable> callback)
{
return invalidate(node, txnId, invalidateWith, false, callback);
}
- public static Invalidate invalidate(Node node, TxnId txnId, Unseekables<?> invalidateWith, boolean transitivelyInvokedByPriorInvalidation, BiConsumer<Outcome, Throwable> callback)
+ public static Invalidate invalidate(Node node, TxnId txnId, Participants<?> invalidateWith, boolean transitivelyInvokedByPriorInvalidation, BiConsumer<Outcome, Throwable> callback)
{
Ballot ballot = new Ballot(node.uniqueNow());
Invalidate invalidate = new Invalidate(node, ballot, txnId, invalidateWith, transitivelyInvokedByPriorInvalidation, callback);
@@ -103,7 +102,11 @@ public class Invalidate implements Callback<InvalidateReply>
return;
replies[topology.nodes().find(from)] = reply;
- handle(tracker.recordSuccess(from, reply.isPromised(), reply.hasDecision(), reply.acceptedFastPath));
+
+ Participants<?> truncated = reply.truncated;
+ Participants<?> notTruncated = truncated == null ? invalidateWith : invalidateWith.without(truncated);
+ Participants<?> promised = reply.isPromiseRejected() ? null : notTruncated;
+ handle(tracker.recordSuccess(from, promised, notTruncated, truncated, reply.hasDecision(), reply.acceptedFastPath));
}
@Override
@@ -154,7 +157,7 @@ public class Invalidate implements Callback<InvalidateReply>
// first look to see if it has already been decided/invalidated
// check each shard independently - if we find any that can be invalidated, do so
InvalidateReply max = InvalidateReply.max(replies);
- InvalidateReply maxNotTruncated = max.maxKnowledgeStatus.is(Status.Truncated) ? max : InvalidateReply.maxNotTruncated(replies);
+ InvalidateReply maxNotTruncated = !max.maxKnowledgeStatus.is(Status.Truncated) ? max : InvalidateReply.maxNotTruncated(replies);
if (maxNotTruncated != null)
{
@@ -214,18 +217,29 @@ public class Invalidate implements Callback<InvalidateReply>
if (max != maxNotTruncated)
{
- boolean allShardsTruncated = true;
- for (Shard shard : topology.shards())
+ Invariants.checkState(maxNotTruncated == null || !maxNotTruncated.maxKnowledgeStatus.hasBeen(Status.PreCommitted));
+ isDone = true;
+ if (!tracker.isAnyNotTruncated())
{
- InvalidateReply maxReply = InvalidateReply.max(replies, shard, topology.nodes());
- allShardsTruncated &= maxReply.maxStatus.is(Status.Truncated);
+ commitInvalidate();
}
- if (allShardsTruncated)
+ else
{
- isDone = true;
+ // TODO (required): revisit this logic some more
+ // this exists because if we have truncated a range we cannot seek votes to advance the state machine, so we may not be able to reach a quorum
+ // but, equally, we may not propagate the GC point to all replicas.
+ // the current logic has pitfalls when *some* replica has a GC point that others cannot reach.
+ // we should reconsider and at least
+ // 1) ensure GC points can be propagated between nodes;
+ // 2) broaden cases where we can derive that the command has not been decided and send invalidated/erased to everyone
+ // TODO (required): we have another edge cases to consider here only when we don't have a FullRoute and
+ // we report that the outcome is durable - which may not be the case for all of the shards.
+ // The home shard will stop attempting to recover the transaction in this case.
+ // This is perhaps not even a problem, and requires that no healthy home shard even has the FullRoute.
+ // Other shards would be expected to coordinate the invalidation of this transaction themselves.
callback.accept(TRUNCATED_DURABLE_OR_INVALIDATED, null);
- return;
}
+ return;
}
// if we have witnessed the transaction, but are able to invalidate, do we want to proceed?
@@ -267,17 +281,19 @@ public class Invalidate implements Callback<InvalidateReply>
// TODO (desired, efficiency): commitInvalidate (and others) should skip the network for local applications,
// so we do not need to explicitly do so here before notifying the waiter
- Unseekables<?> commitTo = Unseekables.merge(route, (Unseekables) invalidateWith);
+ Participants<?> commitTo = Participants.merge(route, (Participants) invalidateWith);
Commit.Invalidate.commitInvalidate(node, txnId, commitTo, txnId);
commitInvalidateLocal(commitTo);
}
- private void commitInvalidateLocal(Unseekables<?> commitTo)
+ private void commitInvalidateLocal(Participants<?> commitTo)
{
// TODO (desired): merge with FetchData.InvalidateOnDone
// TODO (desired): when sending to network, register a callback for when local application of commitInvalidate message ahs been performed, so no need to special-case
node.forEachLocal(contextFor(txnId), commitTo, txnId.epoch(), txnId.epoch(), safeStore -> {
- Commands.commitInvalidate(safeStore, safeStore.get(txnId, txnId, commitTo), commitTo);
+ // TODO (expected): consid
+ StoreParticipants participants = StoreParticipants.invalidate(safeStore, commitTo, txnId);
+ Commands.commitInvalidate(safeStore, safeStore.get(txnId, participants), commitTo);
}).begin((s, f) -> {
callback.accept(INVALIDATED, null);
if (f != null) // TODO (required): consider exception handling more carefully: should we catch these prior to passing to callbacks?
diff --git a/accord-core/src/main/java/accord/coordinate/MaybeRecover.java b/accord-core/src/main/java/accord/coordinate/MaybeRecover.java
index fea1aec3..b568a65a 100644
--- a/accord-core/src/main/java/accord/coordinate/MaybeRecover.java
+++ b/accord-core/src/main/java/accord/coordinate/MaybeRecover.java
@@ -20,7 +20,6 @@ package accord.coordinate;
import java.util.function.BiConsumer;
-import accord.local.Status.Known;
import accord.messages.InformDurable;
import accord.primitives.*;
import accord.utils.Invariants;
@@ -29,7 +28,6 @@ import accord.local.Node;
import accord.messages.CheckStatus.CheckStatusOk;
import accord.messages.CheckStatus.IncludeInfo;
-import static accord.coordinate.Infer.SafeEraseAndCallback.safeEraseAndCallback;
import static accord.coordinate.Infer.InvalidateAndCallback.locallyInvalidateAndCallback;
/**
@@ -68,7 +66,6 @@ public class MaybeRecover extends CheckShards<Route<?>>
|| ok.toProgressToken().compareTo(prevProgress) > 0);
}
- @SuppressWarnings({"unchecked", "rawtypes"})
@Override
protected void onDone(Success success, Throwable fail)
{
@@ -87,12 +84,15 @@ public class MaybeRecover extends CheckShards<Route<?>>
{
default: throw new AssertionError();
case Unknown:
- // TODO (required): ErasedOrInvalidated takes Unknown here. This is probably wrong, consider it more carefully.
- // Specifically, we should probably not propose invalidation if it's possible the command has been Erased.
- // We should probably introduce a MaybeErased Outcome.
- // If we only have a partial route, and all end-points we contact are MaybeErased, we are either stale or
- // we have enough local information to know the command's outcome is durable.
- // If any shard is not MaybeErased but is also Unknown, then we are safe to Invalidate.
+ // ErasedOrInvalidated takes Unknown, and so permits invalidation to be initiated.
+ // This might prima facie seem unsafe, as ErasedOrInvalidated might mean the command
+ // has been executed and erased, in which case it is not safe to invalidate.
+ // However, replicas that have erased the history for these commands also cannot vote to proceed with
+ // invalidation, so the Invalidation state machine must special-case this scenario.
+ // If there exists a shard that has not been decided, then the outcome must be invalidated and it
+ // may be disseminated globally. However, if all shards are erased then the outcome must be
+ // decided locally by the application of GC points.
+ // TODO (expected): replicas may be stale in this case, and should detect this and stop attempting to coordinate/invalidate.
if (known.canProposeInvalidation() && !Route.isFullRoute(full.route))
{
// for correctness reasons, we have not necessarily preempted the initial pre-accept round and
@@ -118,18 +118,15 @@ public class MaybeRecover extends CheckShards<Route<?>>
break;
case WasApply:
+ case Erased:
+ // TODO (required): on Erased, should we maybe mark stale, or leave to FetchData?
callback.accept(full.toProgressToken(), null);
break;
case Invalidated:
- locallyInvalidateAndCallback(node, txnId, txnId, someRoute, full.toProgressToken(), callback);
+ // TODO (required): revisit lowEpoch here
+ locallyInvalidateAndCallback(node, txnId, txnId, txnId, someRoute, full.toProgressToken(), callback);
break;
-
- case Erased:
- // TODO (required): this isn't valid. This is either an invalidated command or we're stale. Most likely the latter.
- // this is because Erased is only permitted to be adopted when every shard has made the command durable.
- Invariants.checkState(!full.knownFor(route.participants()).isInvalidated());
- safeEraseAndCallback(node, txnId, txnId, someRoute, full.toProgressToken(), callback);
}
}
}
diff --git a/accord-core/src/main/java/accord/coordinate/Persist.java b/accord-core/src/main/java/accord/coordinate/Persist.java
index e0e2c92b..1027dde2 100644
--- a/accord-core/src/main/java/accord/coordinate/Persist.java
+++ b/accord-core/src/main/java/accord/coordinate/Persist.java
@@ -36,9 +36,10 @@ import accord.primitives.Txn;
import accord.primitives.TxnId;
import accord.primitives.Writes;
import accord.topology.Topologies;
+import accord.topology.Topology;
import static accord.coordinate.tracking.RequestStatus.Success;
-import static accord.local.Status.Durability.Majority;
+import static accord.primitives.Status.Durability.Majority;
public abstract class Persist implements Callback<ApplyReply>
{
@@ -85,7 +86,11 @@ public abstract class Persist implements Callback<ApplyReply>
{
isDone = true;
Topologies topologies = tracker.topologies();
- node.send(topologies.nodes(), to -> new InformDurable(to, topologies, route, txnId, executeAt, Majority));
+ Topology topology = topologies.forEpoch(txnId.epoch());
+ int homeShardIndex = topology.indexForKey(route.homeKey());
+ // we can persist only partially if some shards are already completed; in this case the home shard may not participate
+ if (homeShardIndex >= 0)
+ node.send(topology.get(homeShardIndex).nodes, to -> new InformDurable(to, topologies, route, txnId, executeAt, Majority));
}
}
break;
diff --git a/accord-core/src/main/java/accord/coordinate/StabiliseTxn.java b/accord-core/src/main/java/accord/coordinate/PersistExclusiveSyncPoint.java
similarity index 56%
copy from accord-core/src/main/java/accord/coordinate/StabiliseTxn.java
copy to accord-core/src/main/java/accord/coordinate/PersistExclusiveSyncPoint.java
index da36f659..ab22888d 100644
--- a/accord-core/src/main/java/accord/coordinate/StabiliseTxn.java
+++ b/accord-core/src/main/java/accord/coordinate/PersistExclusiveSyncPoint.java
@@ -18,30 +18,35 @@
package accord.coordinate;
-import java.util.function.BiConsumer;
-
import accord.api.Result;
import accord.local.Node;
-import accord.primitives.Ballot;
+import accord.messages.Apply;
import accord.primitives.Deps;
import accord.primitives.FullRoute;
import accord.primitives.Timestamp;
import accord.primitives.Txn;
import accord.primitives.TxnId;
+import accord.primitives.Writes;
import accord.topology.Topologies;
-import static accord.coordinate.CoordinationAdapter.Factory.Step.Continue;
-
-public class StabiliseTxn extends Stabilise<Result>
+public class PersistExclusiveSyncPoint extends Persist
{
- StabiliseTxn(Node node, Topologies coordinates, Topologies allTopologies, FullRoute<?> route, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Deps unstableDeps, BiConsumer<? super Result, Throwable> callback)
+ public PersistExclusiveSyncPoint(Node node, Topologies topologies, TxnId txnId, FullRoute<?> route, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result)
{
- super(node, coordinates, allTopologies, route, txnId, ballot, txn, executeAt, unstableDeps, callback);
+ super(node, topologies, txnId, route, txn, executeAt, deps, writes, result);
}
@Override
- protected CoordinationAdapter<Result> adapter()
+ public void start(Apply.Factory factory, Apply.Kind kind, Topologies all, Writes writes, Result result)
{
- return node.coordinationAdapter(txnId, Continue);
+ factory = Apply.wrapForExclusiveSyncPoint(factory);
+ for (Node.Id to : topologies.nodes())
+ {
+ Apply apply = factory.create(kind, to, all, txnId, route, txn, executeAt, stableDeps, writes, result);
+ if (apply == null)
+ tracker.recordSuccess(to);
+ else
+ node.send(to, apply, this);
+ }
}
}
diff --git a/accord-core/src/main/java/accord/coordinate/Preempted.java b/accord-core/src/main/java/accord/coordinate/Preempted.java
index 31e23dca..f4fde633 100644
--- a/accord-core/src/main/java/accord/coordinate/Preempted.java
+++ b/accord-core/src/main/java/accord/coordinate/Preempted.java
@@ -36,7 +36,7 @@ public class Preempted extends CoordinationFailed
super(txnId, homeKey);
}
- private Preempted(TxnId txnId, @Nullable RoutingKey homeKey, Preempted cause)
+ Preempted(TxnId txnId, @Nullable RoutingKey homeKey, Preempted cause)
{
super(txnId, homeKey, cause);
}
diff --git a/accord-core/src/main/java/accord/coordinate/Propose.java b/accord-core/src/main/java/accord/coordinate/Propose.java
index 74cf7c38..b4d4cf5c 100644
--- a/accord-core/src/main/java/accord/coordinate/Propose.java
+++ b/accord-core/src/main/java/accord/coordinate/Propose.java
@@ -18,10 +18,7 @@
package accord.coordinate;
-import java.util.ArrayList;
-import java.util.List;
import java.util.Map;
-import java.util.TreeMap;
import java.util.function.BiConsumer;
import accord.api.RoutingKey;
@@ -44,6 +41,7 @@ import accord.primitives.TxnId;
import accord.topology.Shard;
import accord.topology.Topologies;
import accord.utils.Invariants;
+import accord.utils.SortedListMap;
import static accord.coordinate.tracking.AbstractTracker.ShardOutcomes.Fail;
import static accord.coordinate.tracking.RequestStatus.Failed;
@@ -59,8 +57,7 @@ abstract class Propose<R> implements Callback<AcceptReply>
final FullRoute<?> route;
final Deps deps;
- final List<AcceptReply> acceptOks;
- private final Map<Id, AcceptReply> debug = debug() ? new TreeMap<>() : null;
+ final SortedListMap<Id, AcceptReply> acceptOks;
final Timestamp executeAt;
final QuorumTracker acceptTracker;
final BiConsumer<? super R, Throwable> callback;
@@ -76,15 +73,15 @@ abstract class Propose<R> implements Callback<AcceptReply>
this.deps = deps;
this.executeAt = executeAt;
this.callback = callback;
- this.acceptOks = new ArrayList<>();
+ this.acceptOks = new SortedListMap<>(topologies.nodes(), AcceptReply[]::new);
this.acceptTracker = new QuorumTracker(topologies);
- Invariants.checkState(txnId.kind().isSyncPoint() || deps.maxTxnId(txnId).compareTo(executeAt) <= 0,
+ Invariants.checkState(txnId.isSyncPoint() || deps.maxTxnId(txnId).compareTo(executeAt) <= 0,
"Attempted to propose %s with an earlier executeAt than a conflicting transaction it witnessed: %s vs executeAt: %s", txnId, deps, executeAt);
}
void start()
{
- node.send(acceptTracker.nodes(), to -> new Accept(to, acceptTracker.topologies(), ballot, txnId, route, executeAt, txn.keys(), deps), this);
+ node.send(acceptTracker.nodes(), to -> new Accept(to, acceptTracker.topologies(), ballot, txnId, route, executeAt, deps), this);
}
@Override
@@ -93,8 +90,6 @@ abstract class Propose<R> implements Callback<AcceptReply>
if (isDone)
return;
- if (debug != null) debug.put(from, reply);
-
switch (reply.outcome())
{
default: throw new IllegalStateException();
@@ -102,13 +97,20 @@ abstract class Propose<R> implements Callback<AcceptReply>
isDone = true;
callback.accept(null, new Truncated(txnId, route.homeKey()));
break;
- case Redundant:
+
case RejectedBallot:
isDone = true;
callback.accept(null, new Preempted(txnId, route.homeKey()));
break;
+
+ case Redundant:
+ isDone = true;
+ if (reply.supersededBy != null || ballot.equals(Ballot.ZERO)) callback.accept(null, new Preempted(txnId, route.homeKey()));
+ else callback.accept(null, new Redundant(txnId, route.homeKey(), reply.committedExecuteAt));
+ break;
+
case Success:
- acceptOks.add(reply);
+ acceptOks.put(from, reply);
if (acceptTracker.recordSuccess(from) == RequestStatus.Success)
{
isDone = true;
@@ -120,6 +122,7 @@ abstract class Propose<R> implements Callback<AcceptReply>
@Override
public void onFailure(Id from, Throwable failure)
{
+ // TODO (expected): we aren't tracking the specific failure here to report
if (acceptTracker.recordFailure(from) == Failed)
{
isDone = true;
@@ -144,7 +147,7 @@ abstract class Propose<R> implements Callback<AcceptReply>
final TxnId txnId;
final RoutingKey someParticipant;
final BiConsumer<Void, Throwable> callback;
- final Map<Id, AcceptReply> debug = debug() ? new TreeMap<>() : null;
+ final Map<Id, AcceptReply> debug;
private final QuorumShardTracker acceptTracker;
private boolean isDone;
@@ -157,6 +160,7 @@ abstract class Propose<R> implements Callback<AcceptReply>
this.txnId = txnId;
this.someParticipant = someParticipant;
this.callback = callback;
+ this.debug = debug() ? new SortedListMap<>(shard.nodes, AcceptReply[]::new) : null;
}
public static Invalidate proposeInvalidate(Node node, Ballot ballot, TxnId txnId, RoutingKey invalidateWithParticipant, BiConsumer<Void, Throwable> callback)
diff --git a/accord-core/src/main/java/accord/coordinate/ProposeSyncPoint.java b/accord-core/src/main/java/accord/coordinate/ProposeSyncPoint.java
index 6d3aab26..cdfaa3af 100644
--- a/accord-core/src/main/java/accord/coordinate/ProposeSyncPoint.java
+++ b/accord-core/src/main/java/accord/coordinate/ProposeSyncPoint.java
@@ -27,23 +27,23 @@ import accord.local.Node;
import accord.primitives.Ballot;
import accord.primitives.Deps;
import accord.primitives.FullRoute;
-import accord.primitives.Seekables;
import accord.primitives.SyncPoint;
import accord.primitives.Timestamp;
import accord.primitives.Txn;
import accord.primitives.TxnId;
+import accord.primitives.Unseekable;
import accord.topology.Topologies;
-import accord.utils.Faults;
+import accord.utils.SortedListMap;
-import static accord.coordinate.CoordinationAdapter.Invoke.stabilise;
+import static accord.api.ProtocolModifiers.Faults;
-public class ProposeSyncPoint<S extends Seekables<?, ?>> extends Propose<SyncPoint<S>>
+public class ProposeSyncPoint<U extends Unseekable> extends Propose<SyncPoint<U>>
{
@SuppressWarnings("unused")
private static final Logger logger = LoggerFactory.getLogger(ProposeSyncPoint.class);
- private final CoordinationAdapter<SyncPoint<S>> adapter;
+ private final CoordinationAdapter<SyncPoint<U>> adapter;
- ProposeSyncPoint(CoordinationAdapter<SyncPoint<S>> adapter, Node node, Topologies topologies, FullRoute<?> route, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer<? super SyncPoint<S>, Throwable> callback)
+ ProposeSyncPoint(CoordinationAdapter<SyncPoint<U>> adapter, Node node, Topologies topologies, FullRoute<?> route, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer<? super SyncPoint<U>, Throwable> callback)
{
super(node, topologies, ballot, txnId, txn, route, executeAt, deps, callback);
this.adapter = adapter;
@@ -52,7 +52,7 @@ public class ProposeSyncPoint<S extends Seekables<?, ?>> extends Propose<SyncPoi
@Override
void onAccepted()
{
- Deps deps = Faults.SYNCPOINT_UNMERGED_DEPS ? this.deps : this.deps.with(Deps.merge(acceptOks, ok -> ok.deps));
- stabilise(adapter, node, acceptTracker.topologies(), route, ballot, txnId, txn, executeAt, deps, callback);
+ Deps deps = Faults.syncPointDiscardPreAcceptDeps ? this.deps : this.deps.with(Deps.merge(acceptOks, acceptOks.domainSize(), SortedListMap::getValue, ok -> ok.deps));
+ adapter.stabilise(node, acceptTracker.topologies(), route, ballot, txnId, txn, executeAt, deps, callback);
}
}
diff --git a/accord-core/src/main/java/accord/coordinate/ProposeTxn.java b/accord-core/src/main/java/accord/coordinate/ProposeTxn.java
index 075839d7..b45b9931 100644
--- a/accord-core/src/main/java/accord/coordinate/ProposeTxn.java
+++ b/accord-core/src/main/java/accord/coordinate/ProposeTxn.java
@@ -29,10 +29,10 @@ import accord.primitives.Timestamp;
import accord.primitives.Txn;
import accord.primitives.TxnId;
import accord.topology.Topologies;
-import accord.utils.Faults;
+import accord.utils.SortedListMap;
+import static accord.api.ProtocolModifiers.Faults;
import static accord.coordinate.CoordinationAdapter.Factory.Step.Continue;
-import static accord.coordinate.CoordinationAdapter.Invoke.stabilise;
class ProposeTxn extends Propose<Result>
{
@@ -44,11 +44,11 @@ class ProposeTxn extends Propose<Result>
@Override
void onAccepted()
{
- Deps deps = this.deps;
- if (!Faults.TRANSACTION_UNMERGED_DEPS)
- deps = deps.with(Deps.merge(acceptOks, ok -> ok.deps));
+ Deps deps = Deps.merge(acceptOks, acceptOks.domainSize(), SortedListMap::getValue, ok -> ok.deps);
+ if (!Faults.txnDiscardPreAcceptDeps)
+ deps = deps.with(this.deps);
- stabilise(adapter(), node, acceptTracker.topologies(), route, ballot, txnId, txn, executeAt, deps, callback);
+ adapter().stabilise(node, acceptTracker.topologies(), route, ballot, txnId, txn, executeAt, deps, callback);
}
protected CoordinationAdapter<Result> adapter()
diff --git a/accord-core/src/main/java/accord/coordinate/ReadCoordinator.java b/accord-core/src/main/java/accord/coordinate/ReadCoordinator.java
index 1e2e99da..68910f8d 100644
--- a/accord-core/src/main/java/accord/coordinate/ReadCoordinator.java
+++ b/accord-core/src/main/java/accord/coordinate/ReadCoordinator.java
@@ -21,7 +21,6 @@ package accord.coordinate;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-import java.util.TreeMap;
import com.google.common.collect.Lists;
@@ -30,14 +29,15 @@ import accord.coordinate.tracking.RequestStatus;
import accord.local.Node;
import accord.local.Node.Id;
import accord.messages.Callback;
-import accord.messages.CheckStatus.WithQuorum;
+import accord.primitives.WithQuorum;
import accord.primitives.Ranges;
import accord.primitives.TxnId;
import accord.topology.Topologies;
import accord.utils.Invariants;
+import accord.utils.SortedListMap;
-import static accord.messages.CheckStatus.WithQuorum.HasQuorum;
-import static accord.messages.CheckStatus.WithQuorum.NoQuorum;
+import static accord.primitives.WithQuorum.HasQuorum;
+import static accord.primitives.WithQuorum.NoQuorum;
import static accord.utils.Invariants.debug;
// TODO (expected): configure the number of initial requests we send
@@ -99,13 +99,14 @@ public abstract class ReadCoordinator<Reply extends accord.messages.Reply> exten
protected final TxnId txnId;
private boolean isDone;
private Throwable failure;
- final Map<Id, Object> debug = debug() ? new TreeMap<>() : null;
+ final Map<Id, Object> debug;
protected ReadCoordinator(Node node, Topologies topologies, TxnId txnId)
{
super(topologies);
this.node = node;
this.txnId = txnId;
+ this.debug = debug() ? new SortedListMap<>(topologies.nodes(), Object[]::new) : null;
}
protected abstract Action process(Id from, Reply reply);
diff --git a/accord-core/src/main/java/accord/coordinate/Recover.java b/accord-core/src/main/java/accord/coordinate/Recover.java
index 132baf4b..5f89c835 100644
--- a/accord-core/src/main/java/accord/coordinate/Recover.java
+++ b/accord-core/src/main/java/accord/coordinate/Recover.java
@@ -18,22 +18,19 @@
package accord.coordinate;
-import java.util.Arrays;
import java.util.Collection;
import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.BiConsumer;
-import java.util.stream.Stream;
+
+import javax.annotation.Nullable;
import accord.api.ProgressLog.BlockedUntil;
import accord.api.Result;
-import accord.coordinate.CoordinationAdapter.Invoke;
import accord.coordinate.tracking.RecoveryTracker;
import accord.local.Node;
import accord.local.Node.Id;
-import accord.local.Status;
+import accord.primitives.Status;
import accord.messages.BeginRecovery;
import accord.messages.BeginRecovery.RecoverOk;
import accord.messages.BeginRecovery.RecoverReply;
@@ -45,21 +42,20 @@ import accord.primitives.FullRoute;
import accord.primitives.LatestDeps;
import accord.primitives.Participants;
import accord.primitives.ProgressToken;
-import accord.primitives.Seekables;
import accord.primitives.Timestamp;
import accord.primitives.Txn;
import accord.primitives.TxnId;
+import accord.primitives.Unseekables;
import accord.topology.Shard;
import accord.topology.Topologies;
import accord.topology.Topology;
import accord.utils.Invariants;
+import accord.utils.SortedListMap;
import accord.utils.async.AsyncResult;
import accord.utils.async.AsyncResults;
+import static accord.api.ProtocolModifiers.QuorumEpochIntersections;
import static accord.coordinate.CoordinationAdapter.Factory.Step.InitiateRecovery;
-import static accord.coordinate.CoordinationAdapter.Invoke.execute;
-import static accord.coordinate.CoordinationAdapter.Invoke.persist;
-import static accord.coordinate.CoordinationAdapter.Invoke.stabilise;
import static accord.coordinate.ExecutePath.RECOVER;
import static accord.coordinate.Infer.InvalidateAndCallback.locallyInvalidateAndCallback;
import static accord.coordinate.Propose.Invalidate.proposeInvalidate;
@@ -68,13 +64,10 @@ import static accord.coordinate.tracking.RequestStatus.Success;
import static accord.messages.BeginRecovery.RecoverOk.maxAccepted;
import static accord.messages.BeginRecovery.RecoverOk.maxAcceptedNotTruncated;
import static accord.primitives.ProgressToken.TRUNCATED_DURABLE_OR_INVALIDATED;
-import static accord.utils.Invariants.debug;
import static accord.utils.Invariants.illegalState;
// TODO (low priority, cleanup): rename to Recover (verb); rename Recover message to not clash
-// TODO (expected): do not recover transactions that are known to be Stable and waiting to execute.
-// TODO (expected): separate out recovery of sync points from standard transactions
-// TODO (required): do not wait for a quorum if we find enough information to execute
+// TODO (expected): do not wait for a quorum if we find enough information to execute
public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throwable>
{
AsyncResult<Object> awaitCommits(Node node, Deps waitOn)
@@ -104,21 +97,18 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
private final TxnId txnId;
private final Txn txn;
private final FullRoute<?> route;
+ private final @Nullable Timestamp executeAt;
private final BiConsumer<Outcome, Throwable> callback;
private boolean isDone;
- private final RecoverOk[] recoverOks;
+ private final SortedListMap<Id, RecoverOk> recoverOks;
private final RecoveryTracker tracker;
- private final Topology topology;
private boolean isBallotPromised;
- private final Map<Id, RecoverReply> debug = debug() ? new TreeMap<>() : null;
- private Recover(Node node, Ballot ballot, TxnId txnId, Txn txn, FullRoute<?> route, BiConsumer<Outcome, Throwable> callback, Topologies topologies)
+ private Recover(Node node, Topologies topologies, Ballot ballot, TxnId txnId, Txn txn, FullRoute<?> route, @Nullable Timestamp executeAt, BiConsumer<Outcome, Throwable> callback)
{
- Invariants.checkState(txnId.kind().isGloballyVisible());
- // TODO (required, correctness): we may have to contact all epochs to ensure we spot any future transaction that might not have taken us as dependency?
- // or we need an exclusive sync point covering us and closing out the old epoch before recovering;
- // or we need to manage dependencies for ranges we don't own in future epochs; this might be simplest
+ this.executeAt = executeAt;
+ Invariants.checkState(txnId.isVisible());
this.adapter = node.coordinationAdapter(txnId, InitiateRecovery);
this.node = node;
this.ballot = ballot;
@@ -126,10 +116,8 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
this.txn = txn;
this.route = route;
this.callback = callback;
- assert topologies.oldestEpoch() == topologies.currentEpoch() && topologies.currentEpoch() == txnId.epoch();
this.tracker = new RecoveryTracker(topologies);
- this.topology = topologies.current();
- this.recoverOks = new RecoverOk[topology.nodes().size()];
+ this.recoverOks = new SortedListMap<>(topologies.nodes(), RecoverOk[]::new);
}
@Override
@@ -141,6 +129,10 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
callback.accept(ProgressToken.APPLIED, null);
node.agent().metricsEventsListener().onRecover(txnId, ballot);
}
+ else if (failure instanceof Redundant)
+ {
+ retry(((Redundant) failure).committedExecuteAt);
+ }
else
{
callback.accept(null, failure);
@@ -157,30 +149,31 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
public static Recover recover(Node node, TxnId txnId, Txn txn, FullRoute<?> route, BiConsumer<Outcome, Throwable> callback)
{
- return recover(node, txnId, txn, route, callback, node.topology().forEpoch(route, txnId.epoch()));
+ Ballot ballot = new Ballot(node.uniqueNow());
+ return recover(node, ballot, txnId, txn, route, callback);
}
- private static Recover recover(Node node, TxnId txnId, Txn txn, FullRoute<?> route, BiConsumer<Outcome, Throwable> callback, Topologies topologies)
+ private static Recover recover(Node node, Ballot ballot, TxnId txnId, Txn txn, FullRoute<?> route, BiConsumer<Outcome, Throwable> callback)
{
- Ballot ballot = new Ballot(node.uniqueNow());
- return recover(node, ballot, txnId, txn, route, callback, topologies);
+ return recover(node, ballot, txnId, txn, route, null, callback);
}
- private static Recover recover(Node node, Ballot ballot, TxnId txnId, Txn txn, FullRoute<?> route, BiConsumer<Outcome, Throwable> callback)
+ public static Recover recover(Node node, Ballot ballot, TxnId txnId, Txn txn, FullRoute<?> route, Timestamp executeAt, BiConsumer<Outcome, Throwable> callback)
{
- return recover(node, ballot, txnId, txn, route, callback, node.topology().forEpoch(route, txnId.epoch()));
+ Topologies topologies = node.topology().select(route, txnId, executeAt == null ? txnId : executeAt, QuorumEpochIntersections.recover);
+ return recover(node, topologies, ballot, txnId, txn, route, executeAt, callback);
}
- private static Recover recover(Node node, Ballot ballot, TxnId txnId, Txn txn, FullRoute<?> route, BiConsumer<Outcome, Throwable> callback, Topologies topologies)
+ private static Recover recover(Node node, Topologies topologies, Ballot ballot, TxnId txnId, Txn txn, FullRoute<?> route, Timestamp executeAt, BiConsumer<Outcome, Throwable> callback)
{
- Recover recover = new Recover(node, ballot, txnId, txn, route, callback, topologies);
+ Recover recover = new Recover(node, topologies, ballot, txnId, txn, route, executeAt, callback);
recover.start(topologies.nodes());
return recover;
}
void start(Collection<Id> nodes)
{
- node.send(nodes, to -> new BeginRecovery(to, tracker.topologies(), txnId, txn, route, ballot), this);
+ node.send(nodes, to -> new BeginRecovery(to, tracker.topologies(), txnId, executeAt, txn, route, ballot), this);
}
@Override
@@ -189,8 +182,6 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
if (isDone || isBallotPromised)
return;
- if (debug != null) debug.put(from, reply);
-
if (!reply.isOk())
{
accept(null, new Preempted(txnId, route.homeKey()));
@@ -198,9 +189,8 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
}
RecoverOk ok = (RecoverOk) reply;
- recoverOks[topology.nodes().find(from)] = ok;
- boolean fastPath = ok.executeAt.compareTo(txnId) == 0;
- if (tracker.recordSuccess(from, fastPath) == Success)
+ recoverOks.put(from, ok);
+ if (tracker.recordSuccess(from, ok.acceptsFastPath) == Success)
recover();
}
@@ -209,7 +199,7 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
Invariants.checkState(!isBallotPromised);
isBallotPromised = true;
- List<RecoverOk> recoverOkList = Arrays.asList(recoverOks);
+ List<RecoverOk> recoverOkList = recoverOks.valuesAsNullableList();
RecoverOk acceptOrCommit = maxAccepted(recoverOkList);
RecoverOk acceptOrCommitNotTruncated = acceptOrCommit == null || acceptOrCommit.status != Status.Truncated
? acceptOrCommit : maxAcceptedNotTruncated(recoverOkList);
@@ -236,8 +226,10 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
node.agent().onUncaughtException(CoordinationFailed.wrap(withEpochFailure));
return;
}
+ // TODO (required): if we have to calculate deps for any shard, should we first ensure they are stable?
+ // it should only be possible for a fast path decision, but we might have Stable in only one shard.
// TODO (future development correctness): when writes/result are partially replicated, need to confirm we have quorum of these
- persist(adapter, node, tracker.topologies(), route, txnId, txn, executeAt, stableDeps, acceptOrCommitNotTruncated.writes, acceptOrCommitNotTruncated.result, (ignored, failure) -> {
+ adapter.persist(node, tracker.topologies(), route, txnId, txn, executeAt, stableDeps, acceptOrCommitNotTruncated.writes, acceptOrCommitNotTruncated.result, (ignored, failure) -> {
if (failure != null)
node.agent().onUncaughtException(CoordinationFailed.wrap(withEpochFailure));
});
@@ -248,13 +240,15 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
case Stable:
{
+ // TODO (required): if we have to calculate deps for any shard, should we first ensure they are stable?
+ // it should only be possible for a fast path decision, but we might have Stable in only one shard.
withCommittedDeps(executeAt, (stableDeps, withEpochFailure) -> {
if (withEpochFailure != null)
{
node.agent().onUncaughtException(CoordinationFailed.wrap(withEpochFailure));
return;
}
- execute(adapter, node, tracker.topologies(), route, RECOVER, txnId, txn, executeAt, stableDeps, this);
+ adapter.execute(node, tracker.topologies(), route, RECOVER, txnId, txn, executeAt, stableDeps, this);
});
return;
}
@@ -262,13 +256,14 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
case PreCommitted:
case Committed:
{
+ // TODO (expected): should we only ask for txns with t0 < t0' < t?
withCommittedDeps(executeAt, (committedDeps, withEpochFailure) -> {
if (withEpochFailure != null)
{
node.agent().onUncaughtException(CoordinationFailed.wrap(withEpochFailure));
return;
}
- stabilise(adapter, node, tracker.topologies(), route, ballot, txnId, txn, executeAt, committedDeps, this);
+ adapter.stabilise(node, tracker.topologies(), route, ballot, txnId, txn, executeAt, committedDeps, this);
});
return;
}
@@ -297,37 +292,44 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
if (acceptOrCommit != null)
{
+ // TODO (required): match logic in Invalidate; we need to know which keys have been invalidated
+ Topologies topologies = tracker.topologies();
boolean allShardsTruncated = true;
- for (Shard shard : topology.shards())
+ for (int topologyIndex = 0 ; topologyIndex < topologies.size() ; ++topologyIndex)
{
- RecoverOk maxReply = maxAccepted(topology.nodes().select(recoverOks, shard.nodes));
- allShardsTruncated &= maxReply.status == Status.Truncated;
- }
- if (allShardsTruncated)
- {
- // TODO (required, correctness): this is not a safe inference in the case of an ErasedOrInvalidOrVestigial response.
- // We need to tighten up the inference and spread of truncation/invalid outcomes.
- // In this case, at minimum this can lead to liveness violations as the home shard stops coordinating
- // a command that it hasn't invalidated, but nor is it possible to recover. This happens because
- // when the home shard shares all of its replicas with another shard that has autonomously invalidated
- // the transaction, so that all received InvalidateReply show truncation (when in fact this is only partial).
- // We could paper over this, but better to revisit and provide stronger invariants we can rely on.
- isDone = true;
- callback.accept(TRUNCATED_DURABLE_OR_INVALIDATED, null);
- return;
+ Topology topology = topologies.get(topologyIndex);
+ for (Shard shard : topology.shards())
+ {
+ RecoverOk maxReply = maxAccepted(recoverOks.select(shard.nodes));
+ allShardsTruncated &= maxReply.status == Status.Truncated;
+ }
+ if (allShardsTruncated)
+ {
+ // TODO (required, correctness): this is not a safe inference in the case of an ErasedOrInvalidOrVestigial response.
+ // We need to tighten up the inference and spread of truncation/invalid outcomes.
+ // In this case, at minimum this can lead to liveness violations as the home shard stops coordinating
+ // a command that it hasn't invalidated, but nor is it possible to recover. This happens because
+ // when the home shard shares all of its replicas with another shard that has autonomously invalidated
+ // the transaction, so that all received InvalidateReply show truncation (when in fact this is only partial).
+ // We could paper over this, but better to revisit and provide stronger invariants we can rely on.
+ isDone = true;
+ callback.accept(TRUNCATED_DURABLE_OR_INVALIDATED, null);
+ return;
+ }
}
}
- if (tracker.rejectsFastPath() || Stream.of(recoverOks).anyMatch(ok -> ok != null && ok.rejectsFastPath))
+ if (tracker.rejectsFastPath() || recoverOks.valuesAsNullableStream().anyMatch(ok -> ok != null && ok.rejectsFastPath))
{
invalidate();
return;
}
+ Invariants.checkState(executeAt == null || executeAt.equals(txnId));
// should all be PreAccept
Deps proposeDeps = LatestDeps.mergeProposal(recoverOkList, ok -> ok == null ? null : ok.deps);
- Deps earlierAcceptedNoWitness = Deps.merge(recoverOkList, ok -> ok == null ? null : ok.earlierAcceptedNoWitness);
- Deps earlierCommittedWitness = Deps.merge(recoverOkList, ok -> ok == null ? null : ok.earlierCommittedWitness);
+ Deps earlierAcceptedNoWitness = Deps.merge(recoverOkList, recoverOkList.size(), List::get, ok -> ok.earlierAcceptedNoWitness);
+ Deps earlierCommittedWitness = Deps.merge(recoverOkList, recoverOkList.size(), List::get, ok -> ok.earlierCommittedWitness);
earlierAcceptedNoWitness = earlierAcceptedNoWitness.without(earlierCommittedWitness::contains);
if (!earlierAcceptedNoWitness.isEmpty())
{
@@ -342,29 +344,27 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
});
return;
}
- // TODO (required): if there are epochs before txnId.epoch that haven't propagated their dependencies we potentially have a problem
- // as we may propose deps that don't witness a transaction they should witness.
- // in this case we should run GetDeps before proposing
+
propose(txnId, proposeDeps);
}
private void withCommittedDeps(Timestamp executeAt, BiConsumer<Deps, Throwable> withDeps)
{
- LatestDeps.MergedCommitResult merged = LatestDeps.mergeCommit(txnId, executeAt, Arrays.asList(recoverOks), ok -> ok == null ? null : ok.deps);
+ LatestDeps.MergedCommitResult merged = LatestDeps.mergeCommit(txnId, executeAt, recoverOks.valuesAsNullableList(), ok -> ok == null ? null : ok.deps);
node.withEpoch(executeAt.epoch(), (ignored, withEpochFailure) -> {
if (withEpochFailure != null)
{
withDeps.accept(null, CoordinationFailed.wrap(withEpochFailure));
return;
}
- Seekables<?, ?> missing = txn.keys().without(merged.sufficientFor);
+ Unseekables<?> missing = route.without(merged.sufficientFor);
if (missing.isEmpty())
{
withDeps.accept(merged.deps, null);
}
else
{
- CollectCalculatedDeps.withCalculatedDeps(node, txnId, route, missing.toParticipants(), missing, executeAt, (extraDeps, fail) -> {
+ CollectCalculatedDeps.withCalculatedDeps(node, txnId, route, missing, executeAt, (extraDeps, fail) -> {
if (fail != null)
{
isDone = true;
@@ -391,7 +391,10 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
{
// If not accepted then the executeAt is not consistent cross the peers and likely different on every node. There is also an edge case
// when ranges are removed from the topology, during this case the executeAt won't know the ranges and the invalidate commit will fail.
- Timestamp invalidateUntil = Stream.of(recoverOks).map(ok -> ok == null ? null : ok.status.hasBeen(Status.Accepted) ? ok.executeAt : ok.txnId).reduce(txnId, Timestamp::nonNullOrMax);
+ Timestamp invalidateUntil = recoverOks.valuesAsNullableStream()
+ .map(ok -> ok == null ? null : ok.status.hasBeen(Status.Accepted) ? ok.executeAt : ok.txnId)
+ .reduce(txnId, Timestamp::nonNullOrMax);
+
node.withEpoch(invalidateUntil.epoch(), (ignored, withEpochFailure) -> {
if (withEpochFailure != null)
{
@@ -401,7 +404,7 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
Commit.Invalidate.commitInvalidate(node, txnId, route, invalidateUntil);
});
isDone = true;
- locallyInvalidateAndCallback(node, txnId, invalidateUntil, route, ProgressToken.INVALIDATED, callback);
+ locallyInvalidateAndCallback(node, txnId, txnId, invalidateUntil, route, ProgressToken.INVALIDATED, callback);
}
private void propose(Timestamp executeAt, Deps deps)
@@ -412,13 +415,21 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
this.accept(null, CoordinationFailed.wrap(withEpochFailure));
return;
}
- Invoke.propose(adapter, node, route, ballot, txnId, txn, executeAt, deps, this);
+ adapter.propose(node, null, route, ballot, txnId, txn, executeAt, deps, this);
});
}
private void retry()
{
- Recover.recover(node, ballot, txnId, txn, route, callback, tracker.topologies());
+ retry(executeAt);
+ }
+
+ private void retry(Timestamp executeAt)
+ {
+ Topologies topologies = tracker.topologies();
+ if (executeAt != null && executeAt.epoch() != (this.executeAt == null ? txnId : this.executeAt).epoch())
+ topologies = node.topology().select(route, txnId, executeAt, QuorumEpochIntersections.recover);
+ Recover.recover(node, topologies, ballot, txnId, txn, route, executeAt, callback);
}
@Override
diff --git a/accord-core/src/main/java/accord/coordinate/RecoverWithRoute.java b/accord-core/src/main/java/accord/coordinate/RecoverWithRoute.java
index 8fcafe4b..f699a342 100644
--- a/accord-core/src/main/java/accord/coordinate/RecoverWithRoute.java
+++ b/accord-core/src/main/java/accord/coordinate/RecoverWithRoute.java
@@ -23,8 +23,8 @@ import javax.annotation.Nullable;
import accord.local.Node;
import accord.local.Node.Id;
-import accord.local.Status;
-import accord.local.Status.Known;
+import accord.primitives.Status;
+import accord.primitives.Known;
import accord.messages.CheckStatus;
import accord.messages.CheckStatus.CheckStatusOk;
import accord.messages.CheckStatus.CheckStatusOkFull;
@@ -42,11 +42,10 @@ import accord.topology.Topologies;
import accord.utils.Invariants;
import static accord.coordinate.CoordinationAdapter.Factory.Step.InitiateRecovery;
-import static accord.coordinate.CoordinationAdapter.Invoke.persist;
-import static accord.local.Status.Durability.Majority;
-import static accord.local.Status.KnownDeps.DepsKnown;
-import static accord.local.Status.KnownExecuteAt.ExecuteAtKnown;
-import static accord.local.Status.Outcome.Apply;
+import static accord.primitives.Status.Durability.Majority;
+import static accord.primitives.Known.KnownDeps.DepsKnown;
+import static accord.primitives.Known.KnownExecuteAt.ExecuteAtKnown;
+import static accord.primitives.Known.Outcome.Apply;
import static accord.primitives.ProgressToken.APPLIED;
import static accord.primitives.ProgressToken.INVALIDATED;
import static accord.primitives.ProgressToken.TRUNCATED_DURABLE_OR_INVALIDATED;
@@ -122,7 +121,7 @@ public class RecoverWithRoute extends CheckShards<FullRoute<?>>
protected boolean isSufficient(Route<?> route, CheckStatusOk ok)
{
CheckStatusOkFull full = (CheckStatusOkFull)ok;
- Known sufficientTo = full.knownFor(route.participants());
+ Known sufficientTo = full.knownFor(txnId, route.participants());
if (!sufficientTo.isDefinitionKnown())
return false;
@@ -143,8 +142,9 @@ public class RecoverWithRoute extends CheckShards<FullRoute<?>>
}
CheckStatusOkFull full = ((CheckStatusOkFull) this.merged).finish(route, success.withQuorum);
- Known known = full.knownFor(route.participants());
+ Known known = full.knownFor(txnId, route.participants());
+ // TODO (required): audit this logic, and centralise with e.g. FetchData inferences
switch (known.outcome)
{
default: throw new AssertionError();
@@ -180,14 +180,14 @@ public class RecoverWithRoute extends CheckShards<FullRoute<?>>
Participants<?> sendTo = route.participants().without(full.truncatedResponse());
if (!sendTo.isEmpty())
{
- known = full.knownFor(sendTo);
+ known = full.knownFor(txnId, sendTo);
// we might not know the Apply outcome because we might have raced with truncation on one shard, and the original apply on another,
// so that we know to WasApply, but not
if (known.executeAt == ExecuteAtKnown && known.deps == DepsKnown && known.outcome == Apply)
{
Invariants.checkState(full.stableDeps.covers(sendTo));
- Invariants.checkState(full.partialTxn.covers(sendTo));
- persist(node.coordinationAdapter(txnId, InitiateRecovery), node, route, sendTo, txnId, full.partialTxn, full.executeAt, full.stableDeps, full.writes, full.result, null);
+ Invariants.checkState(txnId.isSystemTxn() || full.partialTxn.covers(sendTo));
+ node.coordinationAdapter(txnId, InitiateRecovery).persist(node, null, route, sendTo, txnId, full.partialTxn, full.executeAt, full.stableDeps, full.writes, full.result, null);
}
propagate = full;
}
@@ -216,7 +216,7 @@ public class RecoverWithRoute extends CheckShards<FullRoute<?>>
node.agent().onUncaughtException(CoordinationFailed.wrap(withEpochFailure));
return;
}
- persist(node.coordinationAdapter(txnId, InitiateRecovery), node, topologies, route(), txnId, txn, full.executeAt, deps, full.writes, full.result, null);
+ node.coordinationAdapter(txnId, InitiateRecovery).persist(node, topologies, route(), txnId, txn, full.executeAt, deps, full.writes, full.result, null);
});
callback.accept(APPLIED, null);
}
diff --git a/accord-core/src/main/java/accord/coordinate/RecoverWithSomeRoute.java b/accord-core/src/main/java/accord/coordinate/RecoverWithSomeRoute.java
deleted file mode 100644
index 2e98333b..00000000
--- a/accord-core/src/main/java/accord/coordinate/RecoverWithSomeRoute.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package accord.coordinate;
-
-import java.util.function.BiConsumer;
-
-import accord.local.Node;
-import accord.local.Status;
-import accord.messages.CheckStatus.CheckStatusOk;
-import accord.messages.CheckStatus.IncludeInfo;
-import accord.primitives.ProgressToken;
-import accord.primitives.Route;
-import accord.primitives.TxnId;
-import accord.utils.Invariants;
-
-import static accord.primitives.Route.castToFullRoute;
-import static accord.primitives.Route.isFullRoute;
-import static accord.utils.Invariants.illegalState;
-
-/**
- * A result of null indicates the transaction is globally persistent
- * A result of CheckStatusOk indicates the maximum status found for the transaction, which may be used to assess progress
- */
-public class RecoverWithSomeRoute extends CheckShards<Route<?>> implements BiConsumer<Object, Throwable>
-{
- final BiConsumer<Outcome, Throwable> callback;
- final Status witnessedByInvalidation;
-
- RecoverWithSomeRoute(Node node, TxnId txnId, Route<?> someRoute, Status witnessedByInvalidation, BiConsumer<Outcome, Throwable> callback)
- {
- super(node, txnId, someRoute.withHomeKey(), IncludeInfo.Route);
- this.witnessedByInvalidation = witnessedByInvalidation;
- // if witnessedByInvalidation == AcceptedInvalidate then we cannot assume its definition was known, and our comparison with the status is invalid
- Invariants.checkState(witnessedByInvalidation != Status.AcceptedInvalidate);
- // if witnessedByInvalidation == Invalidated we should anyway not be recovering
- Invariants.checkState(witnessedByInvalidation != Status.Invalidated);
- this.callback = callback;
- }
-
- public static RecoverWithSomeRoute recover(Node node, TxnId txnId, Route<?> route, Status witnessedByInvalidation, BiConsumer<Outcome, Throwable> callback)
- {
- RecoverWithSomeRoute maybeRecover = new RecoverWithSomeRoute(node, txnId, route, witnessedByInvalidation, callback);
- maybeRecover.start();
- return maybeRecover;
- }
-
- @Override
- public void accept(Object unused, Throwable fail)
- {
- callback.accept(null, fail);
- }
-
- @Override
- protected boolean isSufficient(CheckStatusOk ok)
- {
- return ok.route != null;
- }
-
- @Override
- protected void onDone(Success success, Throwable fail)
- {
- if (fail != null)
- {
- callback.accept(null, fail);
- }
- else if (merged == null || !isFullRoute(merged.route))
- {
- switch (success)
- {
- default: throw new IllegalStateException();
- case Success:
- // home shard must know full Route. Our success criteria is that the response contained a route,
- // so reaching here without a response or one without a full Route is a bug.
- callback.accept(null, new IllegalStateException());
- return;
- case Quorum:
- if (merged != null && merged.isTruncatedResponse())
- {
- callback.accept(ProgressToken.TRUNCATED_DURABLE_OR_INVALIDATED, null);
- }
- else
- {
- if (witnessedByInvalidation != null && witnessedByInvalidation.compareTo(Status.PreAccepted) > 0)
- throw illegalState("We previously invalidated, finding a status that should be recoverable");
- Invalidate.invalidate(node, txnId, route, true, callback);
- }
- }
- }
- else
- {
- // start recovery
- RecoverWithRoute.recover(node, txnId, castToFullRoute(merged.route), witnessedByInvalidation, callback);
- }
- }
-}
diff --git a/accord-core/src/main/java/accord/coordinate/Preempted.java b/accord-core/src/main/java/accord/coordinate/Redundant.java
similarity index 61%
copy from accord-core/src/main/java/accord/coordinate/Preempted.java
copy to accord-core/src/main/java/accord/coordinate/Redundant.java
index 31e23dca..7067dd64 100644
--- a/accord-core/src/main/java/accord/coordinate/Preempted.java
+++ b/accord-core/src/main/java/accord/coordinate/Redundant.java
@@ -21,30 +21,33 @@ package accord.coordinate;
import javax.annotation.Nullable;
import accord.api.RoutingKey;
+import accord.primitives.Timestamp;
import accord.primitives.TxnId;
import static accord.utils.Invariants.checkState;
/**
- * Thrown when a coordinator is preempted by another recovery
- * coordinator intending to complete the transaction
+ * Thrown when a coordinator encounters a phase that is ahead of the one it is attempting to progress.
*/
-public class Preempted extends CoordinationFailed
+public class Redundant extends Preempted
{
- public Preempted(TxnId txnId, @Nullable RoutingKey homeKey)
+ public final Timestamp committedExecuteAt;
+ public Redundant(TxnId txnId, @Nullable RoutingKey homeKey, Timestamp committedExecuteAt)
{
super(txnId, homeKey);
+ this.committedExecuteAt = committedExecuteAt;
}
- private Preempted(TxnId txnId, @Nullable RoutingKey homeKey, Preempted cause)
+ Redundant(TxnId txnId, @Nullable RoutingKey homeKey, Redundant cause, Timestamp committedExecuteAt)
{
super(txnId, homeKey, cause);
+ this.committedExecuteAt = committedExecuteAt;
}
@Override
- public Preempted wrap()
+ public Redundant wrap()
{
- checkState(this.getClass() == Preempted.class);
- return new Preempted(txnId(), homeKey(), this);
+ checkState(this.getClass() == Redundant.class);
+ return new Redundant(txnId(), homeKey(), this, committedExecuteAt);
}
}
diff --git a/accord-core/src/main/java/accord/coordinate/Stabilise.java b/accord-core/src/main/java/accord/coordinate/Stabilise.java
index 4bacfb42..b6145100 100644
--- a/accord-core/src/main/java/accord/coordinate/Stabilise.java
+++ b/accord-core/src/main/java/accord/coordinate/Stabilise.java
@@ -19,7 +19,6 @@
package accord.coordinate;
import java.util.Map;
-import java.util.TreeMap;
import java.util.function.BiConsumer;
import accord.coordinate.tracking.QuorumTracker;
@@ -37,8 +36,8 @@ import accord.primitives.Timestamp;
import accord.primitives.Txn;
import accord.primitives.TxnId;
import accord.topology.Topologies;
+import accord.utils.SortedListMap;
-import static accord.coordinate.CoordinationAdapter.Invoke.execute;
import static accord.coordinate.ExecutePath.SLOW;
import static accord.coordinate.tracking.RequestStatus.Failed;
import static accord.messages.Commit.Kind.CommitWithTxn;
@@ -54,9 +53,9 @@ public abstract class Stabilise<R> implements Callback<ReadReply>
final Timestamp executeAt;
final Deps stabiliseDeps;
- private final Map<Node.Id, Object> debug = debug() ? new TreeMap<>() : null;
final QuorumTracker stableTracker;
final Topologies allTopologies;
+ private final Map<Node.Id, Object> debug;
final BiConsumer<? super R, Throwable> callback;
private boolean isDone;
@@ -72,6 +71,7 @@ public abstract class Stabilise<R> implements Callback<ReadReply>
// we only care about coordination epoch for stability, as it is a recovery condition
this.stableTracker = new QuorumTracker(coordinates);
this.allTopologies = allTopologies;
+ this.debug = debug() ? new SortedListMap<>(allTopologies.nodes(), Object[]::new) : null;
this.callback = callback;
}
@@ -101,8 +101,11 @@ public abstract class Stabilise<R> implements Callback<ReadReply>
switch ((CommitOrReadNack)reply)
{
default: throw new AssertionError("Unhandled CommitOrReadNack: " + reply);
- case Rejected:
case Redundant:
+ isDone = true;
+ callback.accept(null, new Redundant(txnId, route.homeKey(), executeAt));
+ break;
+ case Rejected:
case Invalid:
isDone = true;
callback.accept(null, new Preempted(txnId, route.homeKey()));
@@ -142,7 +145,7 @@ public abstract class Stabilise<R> implements Callback<ReadReply>
protected void onStabilised()
{
- execute(adapter(), node, allTopologies, route, SLOW, txnId, txn, executeAt, stabiliseDeps, callback);
+ adapter().execute(node, allTopologies, route, SLOW, txnId, txn, executeAt, stabiliseDeps, callback);
}
protected abstract CoordinationAdapter<R> adapter();
diff --git a/accord-core/src/main/java/accord/coordinate/StabiliseSyncPoint.java b/accord-core/src/main/java/accord/coordinate/StabiliseSyncPoint.java
index e4936d29..349fdcb4 100644
--- a/accord-core/src/main/java/accord/coordinate/StabiliseSyncPoint.java
+++ b/accord-core/src/main/java/accord/coordinate/StabiliseSyncPoint.java
@@ -24,24 +24,24 @@ import accord.local.Node;
import accord.primitives.Ballot;
import accord.primitives.Deps;
import accord.primitives.FullRoute;
-import accord.primitives.Seekables;
import accord.primitives.SyncPoint;
import accord.primitives.Timestamp;
import accord.primitives.Txn;
import accord.primitives.TxnId;
+import accord.primitives.Unseekable;
import accord.topology.Topologies;
-public class StabiliseSyncPoint<S extends Seekables<?, ?>> extends Stabilise<SyncPoint<S>>
+public class StabiliseSyncPoint<U extends Unseekable> extends Stabilise<SyncPoint<U>>
{
- final CoordinationAdapter<SyncPoint<S>> adapter;
- StabiliseSyncPoint(CoordinationAdapter<SyncPoint<S>> adapter, Node node, Topologies coordinates, Topologies allTopologies, FullRoute<?> route, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Deps unstableDeps, BiConsumer<? super SyncPoint<S>, Throwable> callback)
+ final CoordinationAdapter<SyncPoint<U>> adapter;
+ StabiliseSyncPoint(CoordinationAdapter<SyncPoint<U>> adapter, Node node, Topologies coordinates, Topologies all, FullRoute<?> route, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Deps unstableDeps, BiConsumer<? super SyncPoint<U>, Throwable> callback)
{
- super(node, coordinates, allTopologies, route, txnId, ballot, txn, executeAt, unstableDeps, callback);
+ super(node, coordinates, all, route, txnId, ballot, txn, executeAt, unstableDeps, callback);
this.adapter = adapter;
}
@Override
- protected CoordinationAdapter<SyncPoint<S>> adapter()
+ protected CoordinationAdapter<SyncPoint<U>> adapter()
{
return adapter;
}
diff --git a/accord-core/src/main/java/accord/coordinate/StabiliseTxn.java b/accord-core/src/main/java/accord/coordinate/StabiliseTxn.java
index da36f659..ffd174dd 100644
--- a/accord-core/src/main/java/accord/coordinate/StabiliseTxn.java
+++ b/accord-core/src/main/java/accord/coordinate/StabiliseTxn.java
@@ -34,9 +34,9 @@ import static accord.coordinate.CoordinationAdapter.Factory.Step.Continue;
public class StabiliseTxn extends Stabilise<Result>
{
- StabiliseTxn(Node node, Topologies coordinates, Topologies allTopologies, FullRoute<?> route, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Deps unstableDeps, BiConsumer<? super Result, Throwable> callback)
+ StabiliseTxn(Node node, Topologies coordinates, Topologies all, FullRoute<?> route, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Deps unstableDeps, BiConsumer<? super Result, Throwable> callback)
{
- super(node, coordinates, allTopologies, route, txnId, ballot, txn, executeAt, unstableDeps, callback);
+ super(node, coordinates, all, route, txnId, ballot, txn, executeAt, unstableDeps, callback);
}
@Override
diff --git a/accord-core/src/main/java/accord/coordinate/tracking/InvalidationTracker.java b/accord-core/src/main/java/accord/coordinate/tracking/InvalidationTracker.java
index c9582260..5f79455b 100644
--- a/accord-core/src/main/java/accord/coordinate/tracking/InvalidationTracker.java
+++ b/accord-core/src/main/java/accord/coordinate/tracking/InvalidationTracker.java
@@ -18,8 +18,11 @@
package accord.coordinate.tracking;
-import accord.api.RoutingKey;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
import accord.local.Node;
+import accord.primitives.Participants;
import accord.topology.Shard;
import accord.topology.Topologies;
@@ -31,10 +34,11 @@ public class InvalidationTracker extends AbstractTracker<InvalidationTracker.Inv
{
private int fastPathRejects;
private int fastPathInflight;
- private int promises;
+ private int promisesOrPartPromises;
private boolean hasDecision;
private int inflight;
private boolean isFinal;
+ private @Nullable Participants<?> neverTruncated;
private InvalidationShardTracker(Shard shard)
{
@@ -43,14 +47,19 @@ public class InvalidationTracker extends AbstractTracker<InvalidationTracker.Inv
fastPathInflight = shard.fastPathElectorate.size();
}
- public InvalidationShardTracker onSuccess(Node.Id from, boolean isPromised, boolean isDecided, boolean withFastPath)
+ public InvalidationShardTracker onSuccess(Node.Id from, @Nullable Participants<?> promised, @Nonnull Participants<?> notTruncated, @Nullable Participants<?> truncated, boolean isDecided, boolean withFastPath)
{
if (shard.fastPathElectorate.contains(from))
{
--fastPathInflight;
if (!withFastPath) ++fastPathRejects;
}
- if (isPromised) ++promises;
+ if (promised != null && promised.intersects(shard.range))
+ ++promisesOrPartPromises;
+ if (neverTruncated == null) neverTruncated = notTruncated;
+ if (truncated != null && truncated.intersects(shard.range))
+ neverTruncated = neverTruncated.without(truncated);
+
if (isDecided) hasDecision = true;
--inflight;
return this;
@@ -91,12 +100,17 @@ public class InvalidationTracker extends AbstractTracker<InvalidationTracker.Inv
public boolean isPromiseRejected()
{
- return promises + inflight < shard.slowPathQuorumSize;
+ return promisesOrPartPromises + inflight < shard.slowPathQuorumSize;
}
public boolean isPromised()
{
- return promises >= shard.slowPathQuorumSize;
+ return promisesOrPartPromises >= shard.slowPathQuorumSize;
+ }
+
+ public boolean isAnyNotTruncated()
+ {
+ return neverTruncated != null && !neverTruncated.isEmpty();
}
public boolean isPromisedOrHasDecision()
@@ -149,21 +163,19 @@ public class InvalidationTracker extends AbstractTracker<InvalidationTracker.Inv
return promisedShard >= 0;
}
- public boolean isSafeToInvalidate()
+ public boolean isAnyNotTruncated()
{
- return rejectsFastPath;
+ return any(InvalidationShardTracker::isAnyNotTruncated);
}
- public boolean isPromisedForKey(RoutingKey key, long epoch)
+ public boolean isSafeToInvalidate()
{
- int shardIndex = (int) (topologies.get(0).epoch() - epoch);
- int withinShardIndex = topologies.get(shardIndex).indexForKey(key);
- return get(shardIndex, withinShardIndex).isPromised();
+ return rejectsFastPath;
}
- public RequestStatus recordSuccess(Node.Id from, boolean isPromised, boolean hasDecision, boolean acceptedFastPath)
+ public RequestStatus recordSuccess(Node.Id from, @Nullable Participants<?> promised, @Nonnull Participants<?> notTruncated, @Nullable Participants<?> truncated, boolean hasDecision, boolean acceptedFastPath)
{
- return recordResponse(this, from, (shard, node) -> shard.onSuccess(node, isPromised, hasDecision, acceptedFastPath), from);
+ return recordResponse(this, from, (shard, node) -> shard.onSuccess(node, promised, notTruncated, truncated, hasDecision, acceptedFastPath), from);
}
// return true iff hasFailed()
diff --git a/accord-core/src/main/java/accord/coordinate/tracking/RecoveryTracker.java b/accord-core/src/main/java/accord/coordinate/tracking/RecoveryTracker.java
index 16f8b790..e7c6f5b2 100644
--- a/accord-core/src/main/java/accord/coordinate/tracking/RecoveryTracker.java
+++ b/accord-core/src/main/java/accord/coordinate/tracking/RecoveryTracker.java
@@ -22,6 +22,7 @@ import accord.coordinate.tracking.QuorumTracker.QuorumShardTracker;
import accord.local.Node;
import accord.topology.Shard;
import accord.topology.Topologies;
+import accord.topology.Topology;
public class RecoveryTracker extends AbstractTracker<RecoveryTracker.RecoveryShardTracker>
{
@@ -68,6 +69,16 @@ public class RecoveryTracker extends AbstractTracker<RecoveryTracker.RecoverySha
public boolean rejectsFastPath()
{
- return any(RecoveryShardTracker::rejectsFastPath);
+ // a fast path decision must have recorded itself to a fast quorum in an earlier epoch
+ // but the fast path votes may be taken from the proposal epoch only.
+ // Importantly, on recovery, since we do the full slow path we do not need to reach a fast quorum in the earlier epochs
+ // So, we can effectively ignore earlier epochs wrt fast path decisions.
+ Topology current = topologies.current();
+ for (int i = 0 ; i < current.size() ; ++i)
+ {
+ if (trackers[i].rejectsFastPath())
+ return true;
+ }
+ return false;
}
}
diff --git a/accord-core/src/main/java/accord/impl/AbstractFetchCoordinator.java b/accord-core/src/main/java/accord/impl/AbstractFetchCoordinator.java
index e79c9a79..2380a7f0 100644
--- a/accord-core/src/main/java/accord/impl/AbstractFetchCoordinator.java
+++ b/accord-core/src/main/java/accord/impl/AbstractFetchCoordinator.java
@@ -52,8 +52,8 @@ import accord.utils.async.AsyncResult;
import accord.utils.async.AsyncResults;
import javax.annotation.Nullable;
-import static accord.local.SaveStatus.Applied;
-import static accord.local.SaveStatus.TruncatedApply;
+import static accord.primitives.SaveStatus.Applied;
+import static accord.primitives.SaveStatus.TruncatedApply;
import static accord.messages.ReadData.CommitOrReadNack.Insufficient;
import static accord.primitives.Routables.Slice.Minimal;
diff --git a/accord-core/src/main/java/accord/impl/AbstractSafeCommandStore.java b/accord-core/src/main/java/accord/impl/AbstractSafeCommandStore.java
index f005e5b7..fc68cbfd 100644
--- a/accord-core/src/main/java/accord/impl/AbstractSafeCommandStore.java
+++ b/accord-core/src/main/java/accord/impl/AbstractSafeCommandStore.java
@@ -23,7 +23,7 @@ import java.util.function.Function;
import com.google.common.annotations.VisibleForTesting;
-import accord.api.Key;
+import accord.api.RoutingKey;
import accord.api.VisibleForImplementation;
import accord.local.*;
import accord.local.cfk.SafeCommandsForKey;
@@ -61,13 +61,13 @@ public abstract class AbstractSafeCommandStore<CommandType extends SafeCommand,
protected abstract void addCommandInternal(CommandType command);
protected abstract CommandType getIfLoaded(TxnId txnId);
- protected abstract TimestampsForKeyType getTimestampsForKeyInternal(Key key);
+ protected abstract TimestampsForKeyType getTimestampsForKeyInternal(RoutingKey key);
protected abstract void addTimestampsForKeyInternal(TimestampsForKeyType cfk);
- protected abstract TimestampsForKeyType getTimestampsForKeyIfLoaded(Key key);
+ protected abstract TimestampsForKeyType getTimestampsForKeyIfLoaded(RoutingKey key);
- protected abstract CommandsForKeyType getCommandsForKeyInternal(Key key);
+ protected abstract CommandsForKeyType getCommandsForKeyInternal(RoutingKey key);
protected abstract void addCommandsForKeyInternal(CommandsForKeyType cfk);
- protected abstract CommandsForKeyType getCommandsForKeyIfLoaded(Key key);
+ protected abstract CommandsForKeyType getCommandsForKeyIfLoaded(RoutingKey key);
@Override
protected CommandType getInternalIfLoadedAndInitialised(TxnId txnId)
@@ -89,12 +89,12 @@ public abstract class AbstractSafeCommandStore<CommandType extends SafeCommand,
return command;
}
- private CommandsForKeyType getCommandsIfLoaded(Key key)
+ private CommandsForKeyType getCommandsIfLoaded(RoutingKey key)
{
return getIfLoaded(key, this::getCommandsForKeyInternal, this::addCommandsForKeyInternal, this::getCommandsForKeyIfLoaded);
}
- protected CommandsForKeyType getInternalIfLoadedAndInitialised(Key key)
+ protected CommandsForKeyType getInternalIfLoadedAndInitialised(RoutingKey key)
{
CommandsForKeyType cfk = getCommandsIfLoaded(key);
if (cfk == null)
@@ -105,7 +105,7 @@ public abstract class AbstractSafeCommandStore<CommandType extends SafeCommand,
}
@VisibleForTesting
- protected CommandsForKeyType getInternal(Key key)
+ protected CommandsForKeyType getInternal(RoutingKey key)
{
CommandsForKeyType cfk = getCommandsIfLoaded(key);
Invariants.checkState(cfk != null, "%s was not specified in PreLoadContext", key);
@@ -115,7 +115,7 @@ public abstract class AbstractSafeCommandStore<CommandType extends SafeCommand,
}
@VisibleForImplementation
- public CommandsForKeyType maybeCommandsForKey(Key key)
+ public CommandsForKeyType maybeCommandsForKey(RoutingKey key)
{
CommandsForKeyType cfk = getCommandsIfLoaded(key);
if (cfk == null || cfk.isUnset())
@@ -123,7 +123,7 @@ public abstract class AbstractSafeCommandStore<CommandType extends SafeCommand,
return cfk;
}
- public TimestampsForKeyType timestampsIfLoadedAndInitialised(Key key)
+ public TimestampsForKeyType timestampsIfLoadedAndInitialised(RoutingKey key)
{
TimestampsForKeyType cfk = getIfLoaded(key, this::getTimestampsForKeyInternal, this::addTimestampsForKeyInternal, this::getTimestampsForKeyIfLoaded);
if (cfk == null)
@@ -135,7 +135,7 @@ public abstract class AbstractSafeCommandStore<CommandType extends SafeCommand,
return cfk;
}
- public TimestampsForKeyType timestampsForKey(Key key)
+ public TimestampsForKeyType timestampsForKey(RoutingKey key)
{
TimestampsForKeyType tfk = getIfLoaded(key, this::getTimestampsForKeyInternal, this::addTimestampsForKeyInternal, this::getTimestampsForKeyIfLoaded);
Invariants.checkState(tfk != null, "%s was not specified in PreLoadContext", key);
@@ -146,7 +146,7 @@ public abstract class AbstractSafeCommandStore<CommandType extends SafeCommand,
@VisibleForImplementation
- public TimestampsForKeyType maybeTimestampsForKey(Key key)
+ public TimestampsForKeyType maybeTimestampsForKey(RoutingKey key)
{
TimestampsForKeyType tfk = getIfLoaded(key, this::getTimestampsForKeyInternal, this::addTimestampsForKeyInternal, this::getTimestampsForKeyIfLoaded);
if (tfk == null || tfk.isUnset())
diff --git a/accord-core/src/main/java/accord/impl/CoordinateDurabilityScheduling.java b/accord-core/src/main/java/accord/impl/CoordinateDurabilityScheduling.java
index 08cbbceb..8b9b688c 100644
--- a/accord-core/src/main/java/accord/impl/CoordinateDurabilityScheduling.java
+++ b/accord-core/src/main/java/accord/impl/CoordinateDurabilityScheduling.java
@@ -32,7 +32,6 @@ import org.slf4j.LoggerFactory;
import accord.api.Scheduler;
import accord.coordinate.CoordinateGloballyDurable;
import accord.coordinate.CoordinateShardDurable;
-import accord.coordinate.CoordinateSyncPoint;
import accord.coordinate.CoordinationFailed;
import accord.coordinate.ExecuteSyncPoint.SyncPointErased;
import accord.local.Node;
@@ -48,6 +47,7 @@ import accord.utils.Invariants;
import accord.utils.async.AsyncChain;
import accord.utils.async.AsyncResult;
+import static accord.coordinate.CoordinateSyncPoint.exclusiveSyncPoint;
import static accord.primitives.Txn.Kind.ExclusiveSyncPoint;
import static java.util.concurrent.TimeUnit.MICROSECONDS;
@@ -72,7 +72,8 @@ import static java.util.concurrent.TimeUnit.MICROSECONDS;
* The work for CoordinateShardDurable is further subdivided where each subrange a node operates on is divided a fixed
* number of times and then processed one at a time with a fixed wait between them.
*
- * // TODO (expected): cap number of coordinations we can have in flight at once
+ * TODO (expected): cap number of coordinations we can have in flight at once
+ * TODO (expected): do not start new ExclusiveSyncPoint if we have more than X already agreed and not yet applied
* Didn't go with recurring because it doesn't play well with async execution of these tasks
*/
public class CoordinateDurabilityScheduling
@@ -223,11 +224,18 @@ public class CoordinateDurabilityScheduling
node.agent().onUncaughtException(wrapped);
return;
}
- CoordinateSyncPoint.exclusive(node, at, ranges)
- .addCallback((success, fail) -> {
- if (fail != null) logger.trace("Exception coordinating exclusive sync point for local shard durability of {}", ranges, fail);
- else coordinateShardDurableAfterExclusiveSyncPoint(node, success);
- });
+ exclusiveSyncPoint(node, at, ranges)
+ .addCallback((success, fail) -> {
+ if (fail != null)
+ {
+ logger.trace("{}: Exception coordinating ExclusiveSyncPoint for local shard durability of {}", at, ranges, fail);
+ }
+ else
+ {
+ coordinateShardDurableAfterExclusiveSyncPoint(node, success);
+ logger.trace("{}: Successfully coordinated ExclusiveSyncPoint for local shard durability of {}", at, ranges);
+ }
+ });
}), txnIdLagMicros, MICROSECONDS);
}
@@ -272,8 +280,8 @@ public class CoordinateDurabilityScheduling
{
Shard shard = e.getKey();
int index = e.getValue();
- long microsOffset = (index * shardCycleTimeMicros) / shard.rf();
int shardCycleTimeMicros = Math.max(this.shardCycleTimeMicros, Ints.saturatedCast(shard.rf() * 3L * frequencyMicros));
+ long microsOffset = (index * (long)shardCycleTimeMicros) / shard.rf();
long prevSyncTimeMicros = Math.max(prevShardSyncTimeMicros, nowMicros - ((shardCycleTimeMicros / shard.rf()) / 2L));
int from = (int) ((prevSyncTimeMicros + microsOffset) % shardCycleTimeMicros);
int to = (int) ((nowMicros + microsOffset) % shardCycleTimeMicros);
diff --git a/accord-core/src/main/java/accord/impl/DefaultLocalListeners.java b/accord-core/src/main/java/accord/impl/DefaultLocalListeners.java
index d311475b..071987c6 100644
--- a/accord-core/src/main/java/accord/impl/DefaultLocalListeners.java
+++ b/accord-core/src/main/java/accord/impl/DefaultLocalListeners.java
@@ -32,7 +32,7 @@ import accord.local.Node;
import accord.local.PreLoadContext;
import accord.local.SafeCommand;
import accord.local.SafeCommandStore;
-import accord.local.SaveStatus;
+import accord.primitives.SaveStatus;
import accord.primitives.TxnId;
import accord.utils.AsymmetricComparator;
import accord.utils.Invariants;
@@ -256,6 +256,7 @@ public class DefaultLocalListeners implements LocalListeners
remove.index = -1;
// we don't decrement length even if count==length so as to simplify reentry
--count;
+ if (Invariants.isParanoid() && !notifying) checkIntegrity();
return count > 0 || notifying ? this : null;
}
@@ -299,6 +300,7 @@ public class DefaultLocalListeners implements LocalListeners
add.index = length;
length++;
count++;
+ if (Invariants.isParanoid() && !notifying) checkIntegrity();
}
/**
@@ -326,6 +328,7 @@ public class DefaultLocalListeners implements LocalListeners
}
else if (next.index >= 0) // can be cancelled by notify, without notify return false
{
+ Invariants.checkArgument(next.index == i);
if (i != count)
{
listeners[count] = next;
@@ -333,6 +336,7 @@ public class DefaultLocalListeners implements LocalListeners
}
++count;
}
+ else Invariants.checkState(listeners[i] == null);
}
notifying = false;
@@ -358,8 +362,19 @@ public class DefaultLocalListeners implements LocalListeners
Arrays.fill(listeners, count, length, null);
this.length = count;
+ if (Invariants.isParanoid()) checkIntegrity();
return count == 0 ? null : this;
}
+
+ private void checkIntegrity()
+ {
+ int c = 0;
+ for (int i = 0 ; i < length ; ++i)
+ c += listeners[i] != null ? 1 : 0;
+ Invariants.checkState(c == count);
+ for (int i = length ; i < listeners.length ; ++i)
+ Invariants.checkState(listeners[i] == null);
+ }
}
private static final EnumMap<SaveStatus, AsymmetricComparator<TxnId, TxnListeners>> compareExact, compareAfter;
diff --git a/accord-core/src/main/java/accord/impl/DefaultRemoteListeners.java b/accord-core/src/main/java/accord/impl/DefaultRemoteListeners.java
index ba72455b..e545ea3b 100644
--- a/accord-core/src/main/java/accord/impl/DefaultRemoteListeners.java
+++ b/accord-core/src/main/java/accord/impl/DefaultRemoteListeners.java
@@ -28,8 +28,8 @@ import accord.local.Command;
import accord.local.Node;
import accord.local.SafeCommand;
import accord.local.SafeCommandStore;
-import accord.local.SaveStatus;
-import accord.local.Status.Durability;
+import accord.primitives.SaveStatus;
+import accord.primitives.Status.Durability;
import accord.messages.Await.AsyncAwaitComplete;
import accord.primitives.Route;
import accord.primitives.TxnId;
diff --git a/accord-core/src/main/java/accord/impl/ErasedSafeCommand.java b/accord-core/src/main/java/accord/impl/ErasedSafeCommand.java
index 0da04ca7..c58528cc 100644
--- a/accord-core/src/main/java/accord/impl/ErasedSafeCommand.java
+++ b/accord-core/src/main/java/accord/impl/ErasedSafeCommand.java
@@ -20,14 +20,15 @@ package accord.impl;
import accord.local.Command;
import accord.local.SafeCommand;
-import accord.local.SaveStatus;
+import accord.local.StoreParticipants;
+import accord.primitives.SaveStatus;
import accord.primitives.TxnId;
import accord.utils.Invariants;
-import static accord.local.SaveStatus.Erased;
-import static accord.local.SaveStatus.ErasedOrInvalidOrVestigial;
-import static accord.local.Status.Durability.NotDurable;
-import static accord.local.Status.Durability.UniversalOrInvalidated;
+import static accord.primitives.SaveStatus.Erased;
+import static accord.primitives.SaveStatus.ErasedOrVestigial;
+import static accord.primitives.Status.Durability.NotDurable;
+import static accord.primitives.Status.Durability.UniversalOrInvalidated;
public class ErasedSafeCommand extends SafeCommand
{
@@ -36,8 +37,8 @@ public class ErasedSafeCommand extends SafeCommand
public ErasedSafeCommand(TxnId txnId, SaveStatus saveStatus)
{
super(txnId);
- Invariants.checkArgument(saveStatus == Erased || saveStatus == ErasedOrInvalidOrVestigial);
- this.erased = new Command.Truncated(txnId, saveStatus, saveStatus == ErasedOrInvalidOrVestigial ? NotDurable : UniversalOrInvalidated, null, null, null, null);
+ Invariants.checkArgument(saveStatus.compareTo(Erased) >= 0);
+ this.erased = new Command.Truncated(txnId, saveStatus, saveStatus == ErasedOrVestigial ? NotDurable : UniversalOrInvalidated, StoreParticipants.empty(txnId), null, null, null);
}
@Override
diff --git a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
index d932f6b8..5b425201 100644
--- a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
+++ b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
@@ -19,6 +19,7 @@
package accord.impl;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -42,15 +43,18 @@ import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import com.google.common.annotations.VisibleForTesting;
+
+import accord.api.LocalListeners;
+import accord.api.RoutingKey;
+import accord.impl.progresslog.DefaultProgressLog;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import accord.api.Agent;
import accord.api.DataStore;
import accord.api.Key;
-import accord.api.LocalListeners;
import accord.api.ProgressLog;
-import accord.impl.progresslog.DefaultProgressLog;
import accord.local.Cleanup;
import accord.local.Command;
import accord.local.CommandStore;
@@ -61,28 +65,30 @@ import accord.local.Node;
import accord.local.NodeTimeService;
import accord.local.PreLoadContext;
import accord.local.RedundantStatus;
+import accord.local.RejectBefore;
import accord.local.SafeCommand;
import accord.local.SafeCommandStore;
-import accord.local.SaveStatus;
-import accord.local.Status;
import accord.local.cfk.CommandsForKey;
-import accord.primitives.AbstractKeys;
+import accord.primitives.AbstractRanges;
+import accord.primitives.AbstractUnseekableKeys;
import accord.primitives.Deps;
import accord.primitives.Keys;
import accord.primitives.PartialDeps;
import accord.primitives.Participants;
import accord.primitives.Range;
import accord.primitives.Ranges;
-import accord.primitives.Routable;
import accord.primitives.Routable.Domain;
import accord.primitives.RoutableKey;
import accord.primitives.Routables;
import accord.primitives.Route;
-import accord.primitives.Seekable;
-import accord.primitives.Seekables;
+import accord.primitives.RoutingKeys;
+import accord.primitives.SaveStatus;
+import accord.primitives.Status;
import accord.primitives.Timestamp;
import accord.primitives.Txn.Kind.Kinds;
import accord.primitives.TxnId;
+import accord.primitives.Unseekable;
+import accord.primitives.Unseekables;
import accord.utils.Invariants;
import accord.utils.ReducingRangeMap;
import accord.utils.async.AsyncChain;
@@ -92,16 +98,17 @@ import static accord.local.SafeCommandStore.TestDep.ANY_DEPS;
import static accord.local.SafeCommandStore.TestDep.WITH;
import static accord.local.SafeCommandStore.TestStartedAt.STARTED_BEFORE;
import static accord.local.SafeCommandStore.TestStatus.ANY_STATUS;
-import static accord.local.SaveStatus.Applying;
-import static accord.local.SaveStatus.Erased;
-import static accord.local.SaveStatus.ErasedOrInvalidOrVestigial;
-import static accord.local.SaveStatus.ReadyToExecute;
-import static accord.local.Status.Applied;
-import static accord.local.Status.Invalidated;
-import static accord.local.Status.NotDefined;
-import static accord.local.Status.PreCommitted;
-import static accord.local.Status.Stable;
-import static accord.local.Status.Truncated;
+import static accord.primitives.SaveStatus.Applying;
+import static accord.primitives.SaveStatus.Erased;
+import static accord.primitives.SaveStatus.ErasedOrVestigial;
+import static accord.primitives.SaveStatus.ReadyToExecute;
+import static accord.primitives.Status.Applied;
+import static accord.primitives.Status.Durability.Local;
+import static accord.primitives.Status.Invalidated;
+import static accord.primitives.Status.PreCommitted;
+import static accord.primitives.Status.Stable;
+import static accord.primitives.Status.Truncated;
+import static accord.primitives.Status.NotDefined;
import static accord.primitives.Routables.Slice.Minimal;
import static accord.utils.Invariants.illegalState;
import static java.lang.String.format;
@@ -216,7 +223,7 @@ public abstract class InMemoryCommandStore extends CommandStore
if (intersectingParticipants.isEmpty())
continue;
- if (!cur.txnId().kind().witnesses().test(prev.txnId().kind()) && !cur.partialDeps().contains(prev.txnId()))
+ if (!cur.txnId().witnesses().test(prev.txnId()) && !cur.partialDeps().contains(prev.txnId()))
continue;
Participants<?> depParticipants = cur.partialDeps().participants(prev.txnId());
@@ -252,41 +259,40 @@ public abstract class InMemoryCommandStore extends CommandStore
return commands.containsKey(txnId);
}
- public GlobalCommandsForKey commandsForKeyIfPresent(Key key)
+ public GlobalCommandsForKey commandsForKeyIfPresent(RoutingKey key)
{
return commandsForKey.get(key);
}
- public GlobalCommandsForKey commandsForKey(Key key)
+ public GlobalCommandsForKey commandsForKey(RoutingKey key)
{
return commandsForKey.computeIfAbsent(key, GlobalCommandsForKey::new);
}
- public boolean hasCommandsForKey(Key key)
+ public boolean hasCommandsForKey(RoutingKey key)
{
return commandsForKey.containsKey(key);
}
- public GlobalTimestampsForKey timestampsForKey(Key key)
+ public GlobalTimestampsForKey timestampsForKey(RoutingKey key)
{
return timestampsForKey.computeIfAbsent(key, GlobalTimestampsForKey::new);
}
- public GlobalTimestampsForKey timestampsForKeyIfPresent(Key key)
+ public GlobalTimestampsForKey timestampsForKeyIfPresent(RoutingKey key)
{
return timestampsForKey.get(key);
}
- private <O> O mapReduceForKey(InMemorySafeStore safeStore, Routables<?> keysOrRanges, Ranges slice, BiFunction<CommandsForKey, O, O> map, O accumulate)
+ private <O> O mapReduceForKey(InMemorySafeStore safeStore, Unseekables<?> keysOrRanges, BiFunction<CommandsForKey, O, O> map, O accumulate)
{
switch (keysOrRanges.domain()) {
default:
throw new AssertionError();
case Key:
- AbstractKeys<Key> keys = (AbstractKeys<Key>) keysOrRanges;
- for (Key key : keys)
+ AbstractUnseekableKeys keys = (AbstractUnseekableKeys) keysOrRanges;
+ for (RoutingKey key : keys)
{
- if (!slice.contains(key)) continue;
CommandsForKey commands = safeStore.ifLoadedAndInitialised(key).current();
if (commands == null)
continue;
@@ -295,9 +301,8 @@ public abstract class InMemoryCommandStore extends CommandStore
}
break;
case Range:
- Ranges ranges = (Ranges) keysOrRanges;
- Ranges sliced = ranges.slice(slice, Minimal);
- for (Range range : sliced)
+ AbstractRanges ranges = (AbstractRanges) keysOrRanges;
+ for (Range range : ranges)
{
// TODO (required): this method should fail if it requires more info than available
// TODO (required): I don't think this can possibly work in C*, as we don't know which timestampsForKey we need
@@ -314,43 +319,6 @@ public abstract class InMemoryCommandStore extends CommandStore
return accumulate;
}
- private void forEach(Seekables<?, ?> keysOrRanges, Ranges slice, Consumer<RoutableKey> forEach)
- {
- switch (keysOrRanges.domain()) {
- default:
- throw new AssertionError();
- case Key:
- AbstractKeys<Key> keys = (AbstractKeys<Key>) keysOrRanges;
- keys.forEach(slice, key -> forEach.accept(key));
- break;
- case Range:
- Ranges ranges = (Ranges) keysOrRanges;
- ranges.slice(slice).forEach(range -> {
- commandsForKey.subMap(range.start(), range.startInclusive(), range.end(), range.endInclusive())
- .keySet().forEach(forEach);
- });
- }
- }
-
- private void forEach(Routable keyOrRange, Ranges slice, Consumer<RoutableKey> forEach)
- {
- switch (keyOrRange.domain())
- {
- default: throw new AssertionError();
- case Key:
- Key key = (Key) keyOrRange;
- if (slice.contains(key))
- forEach.accept(key);
- break;
- case Range:
- Range range = (Range) keyOrRange;
- Ranges.of(range).slice(slice).forEach(r -> {
- commandsForKey.subMap(r.start(), r.startInclusive(), r.end(), r.endInclusive())
- .keySet().forEach(forEach);
- });
- }
- }
-
@Override
protected void updatedRedundantBefore(SafeCommandStore safeStore, TxnId syncId, Ranges ranges)
{
@@ -398,27 +366,66 @@ public abstract class InMemoryCommandStore extends CommandStore
});
// verify we're clearing the progress log
- ((Node)time).scheduler().once(() -> {
- DefaultProgressLog progressLog = (DefaultProgressLog) this.progressLog;
- commands.headMap(syncId, false).forEach((id, cmd) -> {
- Command command = cmd.value();
+ new VerifyProgressLogCleared(((Node)time), ranges, Arrays.asList(commands.headMap(syncId, false).keySet().toArray(TxnId[]::new))).run();
+ }
+
+ private class VerifyProgressLogCleared implements Runnable
+ {
+ final Node node;
+ final Ranges ranges;
+ List<TxnId> txnIds;
+ int rounds = 0;
+
+ private VerifyProgressLogCleared(Node node, Ranges ranges, List<TxnId> txnIds)
+ {
+ this.node = node;
+ this.ranges = ranges;
+ this.txnIds = txnIds;
+ }
+
+ @Override
+ public void run()
+ {
+ ++rounds;
+ DefaultProgressLog progressLog = (DefaultProgressLog) InMemoryCommandStore.this.progressLog;
+ List<TxnId> resubmit = new ArrayList<>();
+ for (TxnId txnId : txnIds)
+ {
+ Command command = commands.get(txnId).value();
if (!command.hasBeen(PreCommitted)) return;
- if (!command.txnId().kind().isGloballyVisible()) return;
+ if (!command.txnId().isVisible()) return;
- Ranges allRanges = unsafeRangesForEpoch().allBetween(id.epoch(), command.executeAtOrTxnId().epoch());
+ Ranges allRanges = unsafeRangesForEpoch().allBetween(txnId.epoch(), command.executeAtOrTxnId().epoch());
boolean done = command.hasBeen(Truncated);
if (!done)
{
- if (redundantBefore().status(cmd.txnId, command.executeAtOrTxnId(), command.route()) == RedundantStatus.PRE_BOOTSTRAP_OR_STALE)
+ if (redundantBefore().status(txnId, command.route()) == RedundantStatus.PRE_BOOTSTRAP_OR_STALE)
return;
- Route<?> route = cmd.value().route().slice(allRanges);
+ Route<?> route = command.route().slice(allRanges);
done = !route.isEmpty() && ranges.containsAll(route);
}
- if (done) Invariants.checkState(progressLog.get(id) == null);
- });
- }, 5L, TimeUnit.SECONDS);
+ if (done)
+ {
+ if (progressLog.isWaitingStateActive(txnId))
+ {
+ Invariants.checkState(rounds < 4);
+ resubmit.add(txnId);
+ }
+ else if (progressLog.isHomeStateActive(txnId))
+ {
+ Invariants.checkState(command.durability() == Local);
+ resubmit.add(txnId);
+ }
+ }
+ }
+ if (!resubmit.isEmpty())
+ {
+ txnIds = resubmit;
+ node.scheduler().once(this, 5L, TimeUnit.SECONDS);
+ }
+ }
}
protected InMemorySafeStore createSafeStore(PreLoadContext context, RangesForEpoch ranges,
@@ -447,21 +454,21 @@ public abstract class InMemoryCommandStore extends CommandStore
validateRead(current);
}
- for (Seekable seekable : context.keys())
+ for (Unseekable unseekable : context.keys())
{
- switch (seekable.domain())
+ switch (unseekable.domain())
{
case Key:
- RoutableKey key = (RoutableKey) seekable;
+ RoutableKey key = (RoutableKey) unseekable;
switch (context.keyHistory())
{
case NONE:
continue;
case COMMANDS:
- commandsForKey.put(key, commandsForKey((Key) key).createSafeReference());
+ commandsForKey.put(key, commandsForKey((RoutingKey) key).createSafeReference());
break;
case TIMESTAMPS:
- timestampsForKey.put(key, timestampsForKey((Key) key).createSafeReference());
+ timestampsForKey.put(key, timestampsForKey((RoutingKey) key).createSafeReference());
break;
default: throw new UnsupportedOperationException("Unknown key history: " + context.keyHistory());
}
@@ -539,7 +546,7 @@ public abstract class InMemoryCommandStore extends CommandStore
}
}
- private static Timestamp maxApplied(SafeCommandStore safeStore, Seekables<?, ?> keysOrRanges, Ranges slice)
+ private static Timestamp maxApplied(SafeCommandStore safeStore, Unseekables<?> keysOrRanges, Ranges slice)
{
Timestamp max = ((InMemoryCommandStore)safeStore.commandStore()).maxRedundant;
for (GlobalCommand command : ((InMemoryCommandStore) safeStore.commandStore()).commands.values())
@@ -550,7 +557,7 @@ public abstract class InMemoryCommandStore extends CommandStore
return max;
}
- public AsyncChain<Timestamp> maxAppliedFor(Seekables<?, ?> keysOrRanges, Ranges slice)
+ public AsyncChain<Timestamp> maxAppliedFor(Unseekables<?> keysOrRanges, Ranges slice)
{
return submit(PreLoadContext.contextFor(keysOrRanges), safeStore -> maxApplied(safeStore, keysOrRanges, slice));
}
@@ -558,7 +565,7 @@ public abstract class InMemoryCommandStore extends CommandStore
@Override
public String toString()
{
- return getClass().getSimpleName() + "{" + "id=" + id + ",node=" + time.id().id + '}';
+ return getClass().getSimpleName() + "{id=" + id + ",node=" + time.id().id + '}';
}
static class RangeCommand
@@ -571,10 +578,9 @@ public abstract class InMemoryCommandStore extends CommandStore
this.command = command;
}
- void update(Ranges add)
+ void update(Ranges set)
{
- if (ranges == null) ranges = add;
- else ranges = ranges.with(add);
+ ranges = set;
}
}
@@ -627,11 +633,11 @@ public abstract class InMemoryCommandStore extends CommandStore
public static class GlobalCommandsForKey extends GlobalState<CommandsForKey>
{
- private final Key key;
+ private final RoutingKey key;
public GlobalCommandsForKey(RoutableKey key)
{
- this.key = (Key) key;
+ this.key = (RoutingKey) key;
}
public InMemorySafeCommandsForKey createSafeReference()
@@ -642,11 +648,11 @@ public abstract class InMemoryCommandStore extends CommandStore
public static class GlobalTimestampsForKey extends GlobalState<TimestampsForKey>
{
- private final Key key;
+ private final RoutingKey key;
public GlobalTimestampsForKey(RoutableKey key)
{
- this.key = (Key) key;
+ this.key = (RoutingKey) key;
}
public InMemorySafeTimestampsForKey createSafeReference()
@@ -691,7 +697,7 @@ public abstract class InMemoryCommandStore extends CommandStore
}
@Override
- protected InMemorySafeTimestampsForKey getTimestampsForKeyInternal(Key key)
+ protected InMemorySafeTimestampsForKey getTimestampsForKeyInternal(RoutingKey key)
{
return timestampsForKey.get(key);
}
@@ -703,11 +709,11 @@ public abstract class InMemoryCommandStore extends CommandStore
}
@Override
- protected InMemorySafeTimestampsForKey getTimestampsForKeyIfLoaded(Key key)
+ protected InMemorySafeTimestampsForKey getTimestampsForKeyIfLoaded(RoutingKey key)
{
if (!commandStore.canExposeUnloaded())
return null;
- GlobalTimestampsForKey global = commandStore.timestampsForKeyIfPresent((Key) key);
+ GlobalTimestampsForKey global = commandStore.timestampsForKeyIfPresent(key);
return global != null ? global.createSafeReference() : null;
}
@@ -721,7 +727,7 @@ public abstract class InMemoryCommandStore extends CommandStore
}
@Override
- protected InMemorySafeCommandsForKey getCommandsForKeyInternal(Key key)
+ protected InMemorySafeCommandsForKey getCommandsForKeyInternal(RoutingKey key)
{
return commandsForKey.get(key);
}
@@ -733,7 +739,7 @@ public abstract class InMemoryCommandStore extends CommandStore
}
@Override
- protected InMemorySafeCommandsForKey getCommandsForKeyIfLoaded(Key key)
+ protected InMemorySafeCommandsForKey getCommandsForKeyIfLoaded(RoutingKey key)
{
if (!commandStore.canExposeUnloaded())
return null;
@@ -751,18 +757,13 @@ public abstract class InMemoryCommandStore extends CommandStore
return;
// TODO (expected): consider removing if erased
- if (updated.saveStatus() == Erased || updated.saveStatus() == ErasedOrInvalidOrVestigial)
- return;
-
- Seekables<?, ?> keysOrRanges = updated.keysOrRanges();
- if (keysOrRanges == null) keysOrRanges = prev.keysOrRanges();
- if (keysOrRanges == null)
+ if (updated.saveStatus() == Erased || updated.saveStatus() == ErasedOrVestigial)
return;
- Ranges slice = ranges().allBetween(txnId, updated.executeAtOrTxnId());
+ Ranges slice = ranges(txnId, updated.executeAtOrTxnId());
slice = commandStore.redundantBefore().removeShardRedundant(txnId, updated.executeAtOrTxnId(), slice);
commandStore.rangeCommands.computeIfAbsent(txnId, ignore -> new RangeCommand(commandStore.commands.get(txnId)))
- .update(((Ranges)keysOrRanges).slice(slice, Minimal));
+ .update(((AbstractRanges)updated.participants().touches()).toRanges().slice(slice, Minimal));
}
@Override
@@ -834,30 +835,29 @@ public abstract class InMemoryCommandStore extends CommandStore
}
@Override
- public <P1, T> T mapReduceActive(Seekables<?, ?> keysOrRanges, Ranges slice, Timestamp startedBefore, Kinds testKind, CommandFunction<P1, T, T> map, P1 p1, T accumulate)
+ public <P1, T> T mapReduceActive(Unseekables<?> keysOrRanges, Timestamp startedBefore, Kinds testKind, CommandFunction<P1, T, T> map, P1 p1, T accumulate)
{
- accumulate = commandStore.mapReduceForKey(this, keysOrRanges, slice, (commands, prev) -> {
+ accumulate = commandStore.mapReduceForKey(this, keysOrRanges, (commands, prev) -> {
return commands.mapReduceActive(startedBefore, testKind, map, p1, prev);
}, accumulate);
- return mapReduceRangesInternal(keysOrRanges, slice, startedBefore, null, testKind, STARTED_BEFORE, ANY_DEPS, ANY_STATUS, map, p1, accumulate);
+ return mapReduceRangesInternal(keysOrRanges, startedBefore, null, testKind, STARTED_BEFORE, ANY_DEPS, ANY_STATUS, map, p1, accumulate);
}
// TODO (expected): instead of accepting a slice, accept the min/max epoch and let implementation handle it
@Override
- public <P1, T> T mapReduceFull(Seekables<?, ?> keysOrRanges, Ranges slice, TxnId testTxnId, Kinds testKind, TestStartedAt testStartedAt, TestDep testDep, TestStatus testStatus, CommandFunction<P1, T, T> map, P1 p1, T accumulate)
+ public <P1, T> T mapReduceFull(Unseekables<?> keysOrRanges, TxnId testTxnId, Kinds testKind, TestStartedAt testStartedAt, TestDep testDep, TestStatus testStatus, CommandFunction<P1, T, T> map, P1 p1, T accumulate)
{
- accumulate = commandStore.mapReduceForKey(this, keysOrRanges, slice, (commands, prev) -> {
+ accumulate = commandStore.mapReduceForKey(this, keysOrRanges, (commands, prev) -> {
return commands.mapReduceFull(testTxnId, testKind, testStartedAt, testDep, testStatus, map, p1, prev);
}, accumulate);
- return mapReduceRangesInternal(keysOrRanges, slice, testTxnId, testTxnId, testKind, testStartedAt, testDep, testStatus, map, p1, accumulate);
+ return mapReduceRangesInternal(keysOrRanges, testTxnId, testTxnId, testKind, testStartedAt, testDep, testStatus, map, p1, accumulate);
}
- private <P1, T> T mapReduceRangesInternal(Seekables<?, ?> keysOrRanges, Ranges slice, @Nonnull Timestamp testTimestamp, @Nullable TxnId testTxnId, Kinds testKind, TestStartedAt testStartedAt, TestDep testDep, TestStatus testStatus, CommandFunction<P1, T, T> map, P1 p1, T accumulate)
+ private <P1, T> T mapReduceRangesInternal(Unseekables<?> keysOrRanges, @Nonnull Timestamp testTimestamp, @Nullable TxnId testTxnId, Kinds testKind, TestStartedAt testStartedAt, TestDep testDep, TestStatus testStatus, CommandFunction<P1, T, T> map, P1 p1, T accumulate)
{
// TODO (find lib, efficiency): this is super inefficient, need to store Command in something queryable
- Seekables<?, ?> sliced = keysOrRanges.slice(slice, Minimal);
Map<Range, List<TxnInfo>> collect = new TreeMap<>(Range::compare);
commandStore.rangeCommands.forEach(((txnId, rangeCommand) -> {
Command command = rangeCommand.command.value();
@@ -898,7 +898,7 @@ public abstract class InMemoryCommandStore extends CommandStore
return;
}
- if (!testKind.test(command.txnId().kind()))
+ if (!testKind.test(command.txnId()))
return;
if (testDep != ANY_DEPS)
@@ -921,10 +921,10 @@ public abstract class InMemoryCommandStore extends CommandStore
return;
}
- if (!rangeCommand.ranges.intersects(sliced))
+ if (!rangeCommand.ranges.intersects(keysOrRanges))
return;
- Routables.foldl(rangeCommand.ranges, sliced, (r, in, i) -> {
+ Routables.foldl(rangeCommand.ranges, keysOrRanges, (r, in, i) -> {
// TODO (easy, efficiency): pass command as a parameter to Fold
List<TxnInfo> list = in.computeIfAbsent(r, ignore -> new ArrayList<>());
if (list.isEmpty() || !list.get(list.size() - 1).txnId.equals(command.txnId()))
@@ -948,13 +948,13 @@ public abstract class InMemoryCommandStore extends CommandStore
case ANY:
}
- if (!testKind.test(txnId.kind()))
+ if (!testKind.test(txnId))
return;
- if (!ranges.intersects(sliced))
+ if (!ranges.intersects(keysOrRanges))
return;
- Routables.foldl(ranges, sliced, (r, in, i) -> {
+ Routables.foldl(ranges, keysOrRanges, (r, in, i) -> {
// TODO (easy, efficiency): pass command as a parameter to Fold
List<TxnInfo> list = in.computeIfAbsent(r, ignore -> new ArrayList<>());
if (list.isEmpty() || !list.get(list.size() - 1).txnId.equals(txnId))
@@ -1328,7 +1328,7 @@ public abstract class InMemoryCommandStore extends CommandStore
commandsForKey.clear();
rangeCommands.clear();
historicalRangeCommands.clear();
- unsafeSetRejectBefore(new ReducingRangeMap<>());
+ unsafeSetRejectBefore(new RejectBefore());
}
public interface Loader
@@ -1344,10 +1344,10 @@ public abstract class InMemoryCommandStore extends CommandStore
private PreLoadContext context(Command command, KeyHistory keyHistory)
{
TxnId txnId = command.txnId();
- Keys keys = null;
+ AbstractUnseekableKeys keys = null;
if (CommandsForKey.manages(txnId))
- keys = (Keys) command.keysOrRanges();
+ keys = (AbstractUnseekableKeys) command.participants().hasTouched();
else if (!CommandsForKey.managesExecution(txnId) && command.hasBeen(Status.Stable) && !command.hasBeen(Status.Truncated))
keys = command.asCommitted().waitingOn.keys;
@@ -1369,7 +1369,7 @@ public abstract class InMemoryCommandStore extends CommandStore
Command local = command;
if (local.status() != Truncated && local.status() != Invalidated)
{
- Cleanup cleanup = Cleanup.shouldCleanup(InMemoryCommandStore.this, local, null, local.route(), false);
+ Cleanup cleanup = Cleanup.shouldCleanup(safeStore, local, local.participants());
switch (cleanup)
{
case NO:
@@ -1378,7 +1378,7 @@ public abstract class InMemoryCommandStore extends CommandStore
case TRUNCATE_WITH_OUTCOME:
case TRUNCATE:
case ERASE:
- local = Commands.purge(local, local.route(), cleanup);
+ local = Commands.purge(local, local.participants(), cleanup);
}
}
@@ -1431,7 +1431,7 @@ public abstract class InMemoryCommandStore extends CommandStore
registerHistoricalTransactions(deps, (key, txnId) -> safeStore.get(key).registerHistorical(safeStore, txnId));
}
- private void registerHistoricalTransactions(Deps deps, BiConsumer<Key, TxnId> registerHistorical)
+ private void registerHistoricalTransactions(Deps deps, BiConsumer<RoutingKey, TxnId> registerHistorical)
{
RangesForEpoch rangesForEpoch = this.rangesForEpoch;
Ranges allRanges = rangesForEpoch.all();
@@ -1441,7 +1441,7 @@ public abstract class InMemoryCommandStore extends CommandStore
if (rangesForEpoch.coordinates(txnId).contains(key))
return; // already coordinates, no need to replicate
// TODO (required): check this logic, esp. next line, matches C*
- if (!rangesForEpoch.allAfter(txnId.epoch()).contains(key))
+ if (!rangesForEpoch.allSince(txnId.epoch()).contains(key))
return;
registerHistorical.accept(key, txnId);
@@ -1460,7 +1460,7 @@ public abstract class InMemoryCommandStore extends CommandStore
if (rangesForEpoch.coordinates(txnId).intersects(ranges))
return; // already coordinates, no need to replicate
// TODO (required): check this logic, esp. next line, matches C*
- if (!rangesForEpoch.allAfter(txnId.epoch()).intersects(ranges))
+ if (!rangesForEpoch.allSince(txnId.epoch()).intersects(ranges))
return;
historicalRangeCommands.merge(txnId, ranges.slice(allRanges), Ranges::with);
diff --git a/accord-core/src/main/java/accord/impl/InMemorySafeCommandsForKey.java b/accord-core/src/main/java/accord/impl/InMemorySafeCommandsForKey.java
index 14957750..fc24d131 100644
--- a/accord-core/src/main/java/accord/impl/InMemorySafeCommandsForKey.java
+++ b/accord-core/src/main/java/accord/impl/InMemorySafeCommandsForKey.java
@@ -18,7 +18,7 @@
package accord.impl;
-import accord.api.Key;
+import accord.api.RoutingKey;
import accord.impl.InMemoryCommandStore.GlobalCommandsForKey;
import accord.local.cfk.CommandsForKey;
import accord.local.cfk.SafeCommandsForKey;
@@ -28,7 +28,7 @@ public class InMemorySafeCommandsForKey extends SafeCommandsForKey
private boolean invalidated = false;
private final GlobalCommandsForKey global;
- public InMemorySafeCommandsForKey(Key key, GlobalCommandsForKey global)
+ public InMemorySafeCommandsForKey(RoutingKey key, GlobalCommandsForKey global)
{
super(key);
this.global = global;
diff --git a/accord-core/src/main/java/accord/impl/InMemorySafeTimestampsForKey.java b/accord-core/src/main/java/accord/impl/InMemorySafeTimestampsForKey.java
index 7db9f792..8b8ecf61 100644
--- a/accord-core/src/main/java/accord/impl/InMemorySafeTimestampsForKey.java
+++ b/accord-core/src/main/java/accord/impl/InMemorySafeTimestampsForKey.java
@@ -18,7 +18,7 @@
package accord.impl;
-import accord.api.Key;
+import accord.api.RoutingKey;
import accord.impl.InMemoryCommandStore.GlobalTimestampsForKey;
public class InMemorySafeTimestampsForKey extends SafeTimestampsForKey
@@ -26,7 +26,7 @@ public class InMemorySafeTimestampsForKey extends SafeTimestampsForKey
private boolean invalidated = false;
private final GlobalTimestampsForKey global;
- public InMemorySafeTimestampsForKey(Key key, GlobalTimestampsForKey global)
+ public InMemorySafeTimestampsForKey(RoutingKey key, GlobalTimestampsForKey global)
{
super(key);
this.global = global;
diff --git a/accord-core/src/main/java/accord/impl/SafeTimestampsForKey.java b/accord-core/src/main/java/accord/impl/SafeTimestampsForKey.java
index 42eb3c5f..72a00626 100644
--- a/accord-core/src/main/java/accord/impl/SafeTimestampsForKey.java
+++ b/accord-core/src/main/java/accord/impl/SafeTimestampsForKey.java
@@ -18,21 +18,21 @@
package accord.impl;
-import accord.api.Key;
+import accord.api.RoutingKey;
import accord.primitives.Timestamp;
public abstract class SafeTimestampsForKey implements SafeState<TimestampsForKey>
{
- private final Key key;
+ private final RoutingKey key;
- public SafeTimestampsForKey(Key key)
+ public SafeTimestampsForKey(RoutingKey key)
{
this.key = key;
}
protected abstract void set(TimestampsForKey update);
- public Key key()
+ public RoutingKey key()
{
return key;
}
diff --git a/accord-core/src/main/java/accord/impl/TimestampsForKey.java b/accord-core/src/main/java/accord/impl/TimestampsForKey.java
index 32494b61..3c2e816e 100644
--- a/accord-core/src/main/java/accord/impl/TimestampsForKey.java
+++ b/accord-core/src/main/java/accord/impl/TimestampsForKey.java
@@ -20,7 +20,7 @@ package accord.impl;
import java.util.Objects;
-import accord.api.Key;
+import accord.api.RoutingKey;
import accord.primitives.Timestamp;
import accord.primitives.TxnId;
@@ -30,7 +30,7 @@ public class TimestampsForKey
public static class SerializerSupport
{
- public static TimestampsForKey create(Key key,
+ public static TimestampsForKey create(RoutingKey key,
Timestamp lastExecutedTimestamp,
long lastExecutedHlc,
Timestamp lastWriteTimestamp)
@@ -39,13 +39,13 @@ public class TimestampsForKey
}
}
- private final Key key;
+ private final RoutingKey key;
private final Timestamp lastExecutedTimestamp;
// TODO (desired): we have leaked C* implementation details here
private final long rawLastExecutedHlc;
private final Timestamp lastWriteTimestamp;
- public TimestampsForKey(Key key, Timestamp lastExecutedTimestamp, long rawLastExecutedHlc, Timestamp lastWriteTimestamp)
+ public TimestampsForKey(RoutingKey key, Timestamp lastExecutedTimestamp, long rawLastExecutedHlc, Timestamp lastWriteTimestamp)
{
this.key = key;
this.lastExecutedTimestamp = lastExecutedTimestamp;
@@ -53,7 +53,7 @@ public class TimestampsForKey
this.lastWriteTimestamp = lastWriteTimestamp;
}
- public TimestampsForKey(Key key)
+ public TimestampsForKey(RoutingKey key)
{
this.key = key;
this.lastExecutedTimestamp = Timestamp.NONE;
@@ -74,7 +74,7 @@ public class TimestampsForKey
throw new UnsupportedOperationException();
}
- public Key key()
+ public RoutingKey key()
{
return key;
}
diff --git a/accord-core/src/main/java/accord/impl/TimestampsForKeys.java b/accord-core/src/main/java/accord/impl/TimestampsForKeys.java
index d46d3513..630ff87e 100644
--- a/accord-core/src/main/java/accord/impl/TimestampsForKeys.java
+++ b/accord-core/src/main/java/accord/impl/TimestampsForKeys.java
@@ -18,14 +18,19 @@
package accord.impl;
-import accord.api.Key;
+import accord.api.RoutingKey;
import accord.api.VisibleForImplementation;
import accord.local.CommandStore;
+import accord.primitives.RoutingKeys;
import accord.primitives.Timestamp;
+import accord.primitives.TxnId;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static accord.local.RedundantBefore.PreBootstrapOrStale.FULLY;
+import static accord.utils.Invariants.illegalState;
+
public class TimestampsForKeys
{
@@ -33,7 +38,7 @@ public class TimestampsForKeys
private TimestampsForKeys() {}
- public static TimestampsForKey updateLastExecutionTimestamps(CommandStore commandStore, SafeTimestampsForKey tfk, Timestamp executeAt, boolean isForWriteTxn)
+ public static TimestampsForKey updateLastExecutionTimestamps(CommandStore commandStore, SafeTimestampsForKey tfk, TxnId txnId, Timestamp executeAt, boolean isForWriteTxn)
{
TimestampsForKey current = tfk.current();
@@ -41,9 +46,9 @@ public class TimestampsForKeys
if (executeAt.compareTo(lastWrite) < 0)
{
- if (!commandStore.safeToReadAt(executeAt).contains(tfk.key().toUnseekable()))
+ if (commandStore.redundantBefore().preBootstrapOrStale(txnId, RoutingKeys.of(tfk.key().toUnseekable())) == FULLY)
return current;
- throw new IllegalArgumentException(String.format("%s is less than the most recent write timestamp %s for %s", executeAt, lastWrite, tfk.key()));
+ throw illegalState("%s is less than the most recent write timestamp %s", executeAt, lastWrite);
}
Timestamp lastExecuted = current.lastExecutedTimestamp();
@@ -56,7 +61,7 @@ public class TimestampsForKeys
{
if (!commandStore.safeToReadAt(executeAt).contains(tfk.key().toUnseekable()))
return current;
- throw new IllegalArgumentException(String.format("%s is less than the most recent executed timestamp %s", executeAt, lastExecuted));
+ throw illegalState("%s is less than the most recent executed timestamp %s", executeAt, lastExecuted);
}
long micros = executeAt.hlc();
@@ -70,8 +75,8 @@ public class TimestampsForKeys
}
@VisibleForImplementation
- public static <D> TimestampsForKey updateLastExecutionTimestamps(AbstractSafeCommandStore<?,?,?> safeStore, Key key, Timestamp executeAt, boolean isForWriteTxn)
+ public static <D> TimestampsForKey updateLastExecutionTimestamps(AbstractSafeCommandStore<?,?,?> safeStore, RoutingKey key, TxnId txnId, Timestamp executeAt, boolean isForWriteTxn)
{
- return updateLastExecutionTimestamps(safeStore.commandStore(), safeStore.timestampsForKey(key), executeAt, isForWriteTxn);
+ return updateLastExecutionTimestamps(safeStore.commandStore(), safeStore.timestampsForKey(key), txnId, executeAt, isForWriteTxn);
}
}
diff --git a/accord-core/src/main/java/accord/impl/progresslog/BaseTxnState.java b/accord-core/src/main/java/accord/impl/progresslog/BaseTxnState.java
index 0b3e1e4f..26e3970b 100644
--- a/accord-core/src/main/java/accord/impl/progresslog/BaseTxnState.java
+++ b/accord-core/src/main/java/accord/impl/progresslog/BaseTxnState.java
@@ -155,7 +155,7 @@ abstract class BaseTxnState extends LogGroupTimers.Timer implements Comparable<B
return pendingTimerDelay == 0 ? 0 : scheduledDeadline + pendingTimerDelay;
}
- long pendingTimerDeadline(DefaultProgressLog owner)
+ long pendingTimerDeadline()
{
long pendingTimerDelay = pendingTimerDelay();
return pendingTimerDelay == 0 ? 0 : deadline() + pendingTimerDelay;
diff --git a/accord-core/src/main/java/accord/impl/progresslog/DefaultProgressLog.java b/accord-core/src/main/java/accord/impl/progresslog/DefaultProgressLog.java
index 0932d92e..31ae8adc 100644
--- a/accord-core/src/main/java/accord/impl/progresslog/DefaultProgressLog.java
+++ b/accord-core/src/main/java/accord/impl/progresslog/DefaultProgressLog.java
@@ -25,7 +25,6 @@ import java.util.function.Consumer;
import java.util.function.Function;
import javax.annotation.Nullable;
-import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -38,7 +37,8 @@ import accord.local.Node;
import accord.local.PreLoadContext;
import accord.local.SafeCommand;
import accord.local.SafeCommandStore;
-import accord.local.SaveStatus;
+import accord.primitives.SaveStatus;
+import accord.local.StoreParticipants;
import accord.primitives.Participants;
import accord.primitives.ProgressToken;
import accord.primitives.Ranges;
@@ -62,7 +62,8 @@ import static accord.impl.progresslog.Progress.Queued;
import static accord.impl.progresslog.TxnStateKind.Home;
import static accord.impl.progresslog.TxnStateKind.Waiting;
import static accord.local.PreLoadContext.contextFor;
-import static accord.local.Status.PreApplied;
+import static accord.primitives.Status.PreApplied;
+import static accord.primitives.Status.PreCommitted;
import static accord.utils.ArrayBuffers.cachedAny;
import static accord.utils.btree.UpdateFunction.noOpReplace;
import static java.util.concurrent.TimeUnit.MICROSECONDS;
@@ -138,16 +139,15 @@ public class DefaultProgressLog implements ProgressLog, Runnable
maybeReschedule(nowMicros, deadline);
}
- @VisibleForTesting
- public @Nullable TxnState get(TxnId txnId)
+ @Nullable TxnState get(TxnId txnId)
{
- Invariants.checkState(txnId.kind().isGloballyVisible());
+ Invariants.checkState(txnId.isVisible());
return BTree.<TxnId, TxnState>find(stateMap, (id, state) -> id.compareTo(state.txnId), txnId);
}
TxnState ensure(TxnId txnId)
{
- Invariants.checkState(txnId.kind().isGloballyVisible());
+ Invariants.checkState(txnId.isVisible());
TxnState result = BTree.<TxnId, TxnState>find(stateMap, (id, state) -> id.compareTo(state.txnId), txnId);
if (result == null)
{
@@ -193,14 +193,14 @@ public class DefaultProgressLog implements ProgressLog, Runnable
@Override
public void update(SafeCommandStore safeStore, TxnId txnId, Command before, Command after)
{
- if (!txnId.kind().isGloballyVisible())
+ if (!txnId.isVisible())
return;
TxnState state = null;
if (before.route() == null && after.route() != null)
{
RoutingKey homeKey = after.homeKey();
- Ranges coordinateRanges = safeStore.ranges().allAt(txnId.epoch());
+ Ranges coordinateRanges = safeStore.coordinateRanges(txnId);
boolean isHome = coordinateRanges.contains(homeKey);
state = get(txnId);
if (isHome)
@@ -211,8 +211,7 @@ public class DefaultProgressLog implements ProgressLog, Runnable
if (after.durability().isDurableOrInvalidated())
{
state.setHomeDoneAnyMaybeRemove(this);
- if (!after.hasBeen(PreApplied))
- state.waiting().setBlockedUntil(safeStore, this, CanApply);
+ state = maybeFetch(safeStore, txnId, after, state);
}
else
state.set(safeStore, this, Undecided, Queued);
@@ -229,13 +228,7 @@ public class DefaultProgressLog implements ProgressLog, Runnable
if (state != null)
state.setHomeDoneAnyMaybeRemove(this);
- if (!after.hasBeen(PreApplied))
- {
- // this command should be ready to apply locally, so fetch it
- if (state == null)
- state = insert(txnId);
- state.waiting().setBlockedUntil(safeStore, this, CanApply);
- }
+ state = maybeFetch(safeStore, txnId, after, state);
}
SaveStatus beforeSaveStatus = before.saveStatus();
@@ -267,10 +260,27 @@ public class DefaultProgressLog implements ProgressLog, Runnable
}
}
+ private TxnState maybeFetch(SafeCommandStore safeStore, TxnId txnId, Command after, TxnState state)
+ {
+ if (after.hasBeen(PreApplied))
+ return state;
+
+ Ranges executeRanges = after.hasBeen(PreCommitted) ? safeStore.ranges().allAt(after.executeAt())
+ : safeStore.ranges().allSince(after.txnId().epoch());
+ if (executeRanges.intersects(after.participants().owns()))
+ {
+ // this command should be ready to apply locally, so fetch it
+ if (state == null)
+ state = insert(txnId);
+ state.waiting().setBlockedUntil(safeStore, this, CanApply);
+ }
+ return state;
+ }
+
@Override
public void clear(TxnId txnId)
{
- if (!txnId.kind().isGloballyVisible())
+ if (!txnId.isVisible())
return;
TxnState state = get(txnId);
@@ -324,30 +334,28 @@ public class DefaultProgressLog implements ProgressLog, Runnable
@Override
public void waiting(BlockedUntil blockedUntil, SafeCommandStore safeStore, SafeCommand blockedBy, Route<?> blockedOnRoute, Participants<?> blockedOnParticipants)
{
- if (!blockedBy.txnId().kind().isGloballyVisible())
+ if (!blockedBy.txnId().isVisible())
return;
// ensure we have a record to work with later; otherwise may think has been truncated
// TODO (expected): we shouldn't rely on this anymore
blockedBy.initialise();
Invariants.checkState(blockedBy.current().saveStatus().compareTo(blockedUntil.minSaveStatus) < 0);
+ Invariants.checkState(blockedOnParticipants == null || safeStore.ranges().all().containsAll(blockedOnParticipants)); // more permissive than strictly necessary, but just to cheaply catch errors
// first save the route/participant info into the Command if it isn't already there
+ Command command = blockedBy.current();
+
CommonAttributes update = blockedBy.current();
- Route<?> currentRoute = update.route();
- if (currentRoute != null)
- {
- Route<?> updatedRoute = currentRoute;
- if (blockedOnRoute != null) updatedRoute = currentRoute.with((Route) blockedOnRoute);
- else if (blockedOnParticipants != null)
- updatedRoute = currentRoute.with((Participants) blockedOnParticipants);
- if (updatedRoute != currentRoute)
- update = update.mutable().route(updatedRoute);
- }
- else if (blockedOnRoute != null) update = update.mutable().route(blockedOnRoute);
- else if (blockedOnParticipants != null) update = update.mutable().participants(blockedOnParticipants);
- if (update != blockedBy.current())
- blockedBy.updateAttributes(safeStore, update);
+ StoreParticipants participants = update.participants();
+ StoreParticipants updatedParticipants = participants.supplement(blockedOnRoute, null, blockedOnParticipants);
+ if (participants != updatedParticipants)
+ update = update.mutable().updateParticipants(updatedParticipants);
+
+ if (update != command)
+ command = blockedBy.updateAttributes(safeStore, update);
+
+ Invariants.checkState(safeStore.ranges().allSince(command.txnId().epoch()).intersects(command.participants().hasTouched()));
// TODO (consider): consider triggering a preemption of existing coordinator (if any) in some circumstances;
// today, an LWT can pre-empt more efficiently (i.e. instantly) a failed operation whereas Accord will
@@ -417,7 +425,7 @@ public class DefaultProgressLog implements ProgressLog, Runnable
TxnStateKind runKind = run.wasScheduledTimer();
validatePreRunState(run, runKind);
- long pendingTimerDeadline = run.pendingTimerDeadline(DefaultProgressLog.this);
+ long pendingTimerDeadline = run.pendingTimerDeadline();
if (pendingTimerDeadline > 0)
{
run.clearPendingTimerDelay();
@@ -550,4 +558,16 @@ public class DefaultProgressLog implements ProgressLog, Runnable
{
timers.remove(state);
}
+
+ public boolean isHomeStateActive(TxnId txnId)
+ {
+ TxnState state = get(txnId);
+ return state != null && !state.isHomeDone();
+ }
+
+ public boolean isWaitingStateActive(TxnId txnId)
+ {
+ TxnState state = get(txnId);
+ return state != null && !state.isWaitingDone();
+ }
}
diff --git a/accord-core/src/main/java/accord/impl/progresslog/HomeState.java b/accord-core/src/main/java/accord/impl/progresslog/HomeState.java
index d9268b61..3687aa8e 100644
--- a/accord-core/src/main/java/accord/impl/progresslog/HomeState.java
+++ b/accord-core/src/main/java/accord/impl/progresslog/HomeState.java
@@ -26,6 +26,7 @@ import accord.coordinate.Outcome;
import accord.local.Command;
import accord.local.SafeCommand;
import accord.local.SafeCommandStore;
+import accord.primitives.Status;
import accord.primitives.ProgressToken;
import accord.primitives.TxnId;
import accord.utils.Invariants;
@@ -131,7 +132,7 @@ abstract class HomeState extends WaitingState
{
Invariants.checkState(!isHomeDoneOrUninitialised());
Command command = safeCommand.current();
- Invariants.checkState(!safeStore.isTruncated(command), "Command %s is truncated", command);
+ Invariants.checkState(!command.hasBeen(Status.Truncated), "Command %s is truncated", command);
// TODO (expected): when invalidated, safer to maintain HomeState until known to be globally invalidated
// TODO (now): validate that we clear HomeState when we receive a Durable reply, to replace the token check logic
diff --git a/accord-core/src/main/java/accord/impl/progresslog/WaitingState.java b/accord-core/src/main/java/accord/impl/progresslog/WaitingState.java
index d60f2a28..7801f360 100644
--- a/accord-core/src/main/java/accord/impl/progresslog/WaitingState.java
+++ b/accord-core/src/main/java/accord/impl/progresslog/WaitingState.java
@@ -29,8 +29,9 @@ import accord.local.Command;
import accord.local.Node;
import accord.local.SafeCommand;
import accord.local.SafeCommandStore;
-import accord.local.SaveStatus;
-import accord.local.Status;
+import accord.primitives.SaveStatus;
+import accord.primitives.Status;
+import accord.local.StoreParticipants;
import accord.primitives.EpochSupplier;
import accord.primitives.Participants;
import accord.primitives.Ranges;
@@ -226,20 +227,27 @@ abstract class WaitingState extends BaseTxnState
/*
* Ranges may have moved between command stores locally so extend to a later epoch to invoke those command stores
*/
- private static EpochSupplier toLocalEpoch(SafeCommandStore safeStore, TxnId txnId, BlockedUntil blockedUntil, Command command, Timestamp executeAt)
+ private static EpochSupplier lowEpoch(SafeCommandStore safeStore, TxnId txnId, Command command)
+ {
+ EpochSupplier result = txnId;
+ StoreParticipants participants = command.participants();
+ long newEpoch = safeStore.ranges().latestEarlierEpochThatFullyCovers(txnId.epoch(), participants.hasTouched());
+ if (newEpoch < txnId.epoch())
+ result = constant(newEpoch);
+ return result;
+ }
+
+ private static EpochSupplier highEpoch(SafeCommandStore safeStore, TxnId txnId, BlockedUntil blockedUntil, Command command, Timestamp executeAt)
{
long epoch = blockedUntil.fetchEpoch(txnId, executeAt);
- if (command.route() != null)
- epoch = Math.max(epoch, safeStore.ranges().latestEpochWithNewParticipants(epoch, command.route()));
- if (command.additionalKeysOrRanges() != null)
- epoch = Math.max(epoch, safeStore.ranges().latestEpochWithNewParticipants(epoch, command.additionalKeysOrRanges()));
+ epoch = Math.max(epoch, safeStore.ranges().earliestLaterEpochThatFullyCovers(epoch, command.participants().hasTouched()));
return constant(epoch);
}
private static Route<?> slicedRoute(SafeCommandStore safeStore, TxnId txnId, Command command, BlockedUntil blockedUntil)
{
Timestamp executeAt = command.executeAtIfKnown();
- EpochSupplier toLocalEpoch = toLocalEpoch(safeStore, txnId, blockedUntil, command, executeAt);
+ EpochSupplier toLocalEpoch = highEpoch(safeStore, txnId, blockedUntil, command, executeAt);
Ranges ranges = safeStore.ranges().allBetween(txnId.epoch(), toLocalEpoch);
return command.route().slice(ranges);
@@ -248,7 +256,10 @@ abstract class WaitingState extends BaseTxnState
private static Route<?> slicedRoute(SafeCommandStore safeStore, TxnId txnId, Route<?> route, EpochSupplier toLocalEpoch)
{
Ranges ranges = safeStore.ranges().allBetween(txnId.epoch(), toLocalEpoch);
- return route.slice(ranges);
+ route = route.slice(ranges);
+ if (route.isEmpty())
+ route = route.homeKeyOnlyRoute();
+ return route;
}
private static Route<?> awaitRoute(Route<?> slicedRoute, BlockedUntil blockedUntil)
@@ -310,12 +321,13 @@ abstract class WaitingState extends BaseTxnState
TxnId txnId = safeCommand.txnId();
// first make sure we have enough information to obtain the command locally
Timestamp executeAt = command.executeAtIfKnown();
- EpochSupplier toLocalEpoch = toLocalEpoch(safeStore, txnId, blockedUntil, command, executeAt);
+ EpochSupplier lowEpoch = lowEpoch(safeStore, txnId, command);
+ EpochSupplier highEpoch = highEpoch(safeStore, txnId, blockedUntil, command, executeAt);
Participants<?> fetchKeys = Invariants.nonNull(command.maxContactable());
if (!Route.isRoute(fetchKeys))
{
- fetchRoute(owner, blockedUntil, txnId, executeAt, toLocalEpoch, fetchKeys);
+ fetchRoute(owner, blockedUntil, txnId, executeAt, lowEpoch, highEpoch, fetchKeys);
return;
}
@@ -327,12 +339,13 @@ abstract class WaitingState extends BaseTxnState
return;
}
- Route<?> slicedRoute = slicedRoute(safeStore, txnId, route, toLocalEpoch);
+ Route<?> slicedRoute = slicedRoute(safeStore, txnId, route, highEpoch);
+ Invariants.checkState(!slicedRoute.isEmpty());
if (!command.hasBeen(Status.PreCommitted))
{
// we know it has been decided one way or the other by the home shard at least, so we attempt a fetch
// including the home shard to get us to at least PreCommitted where we can safely wait on individual shards
- fetch(owner, blockedUntil, txnId, executeAt, toLocalEpoch, slicedRoute, slicedRoute.withHomeKey());
+ fetch(owner, blockedUntil, txnId, executeAt, lowEpoch, highEpoch, slicedRoute, slicedRoute.withHomeKey(), route);
return;
}
@@ -341,12 +354,10 @@ abstract class WaitingState extends BaseTxnState
Route<?> awaitRoute = awaitRoute(slicedRoute, blockedUntil);
Route<?> fetchRoute = fetchRoute(slicedRoute, awaitRoute, blockedUntil);
- if (slicedRoute.size() == 0 || awaitRoute.isHomeKeyOnlyRoute())
+ if (awaitRoute.isHomeKeyOnlyRoute())
{
- // either we await only the home shard (which is already known to meet our criteria)
- // or there are no specific local keys we want to update;
- // at this point we can switch to polling as we know someone (i.e. home shard) has the relevant state
- fetch(owner, blockedUntil, txnId, executeAt, toLocalEpoch, slicedRoute, fetchRoute);
+ // at this point we can switch to polling as we know someone has the relevant state
+ fetch(owner, blockedUntil, txnId, executeAt, lowEpoch, highEpoch, slicedRoute, fetchRoute, route);
return;
}
@@ -357,7 +368,7 @@ abstract class WaitingState extends BaseTxnState
{
// all of the shards we are awaiting have been processed and found at least one replica that has the state needed to answer our query
// at this point we can switch to polling as we know someone has the relevant state
- fetch(owner, blockedUntil, txnId, executeAt, toLocalEpoch, slicedRoute, fetchRoute);
+ fetch(owner, blockedUntil, txnId, executeAt, lowEpoch, highEpoch, slicedRoute, fetchRoute, route);
return;
}
@@ -399,7 +410,7 @@ abstract class WaitingState extends BaseTxnState
state.setHomeSatisfies(newHomeSatisfies);
}
- EpochSupplier toLocalEpoch = toLocalEpoch(safeStore, txnId, blockedUntil, command, command.executeAtOrTxnId());
+ EpochSupplier toLocalEpoch = highEpoch(safeStore, txnId, blockedUntil, command, command.executeAtOrTxnId());
Route<?> slicedRoute = slicedRoute(safeStore, txnId, route, toLocalEpoch); // the actual local keys we care about
Route<?> awaitRoute = awaitRoute(slicedRoute, blockedUntil); // either slicedRoute or just the home key
@@ -592,24 +603,20 @@ abstract class WaitingState extends BaseTxnState
}
}
- static void fetchRoute(DefaultProgressLog owner, BlockedUntil blockedUntil, TxnId txnId, Timestamp executeAt, EpochSupplier toLocalEpoch, Participants<?> fetchKeys)
+ static void fetchRoute(DefaultProgressLog owner, BlockedUntil blockedUntil, TxnId txnId, Timestamp executeAt, EpochSupplier lowEpoch, EpochSupplier highEpoch, Participants<?> fetchKeys)
{
// TODO (desired): fetch only the route
// we MUSt allocate before calling withEpoch to register cancellation, as async
BiConsumer<FetchData.FetchResult, Throwable> invoker = invokeWaitingCallback(owner, txnId, blockedUntil, WaitingState::fetchRouteCallback);
- owner.node().withEpoch(blockedUntil.fetchEpoch(txnId, executeAt), invoker, () -> {
- FetchData.fetch(blockedUntil.minSaveStatus.known, owner.node(), txnId, fetchKeys, toLocalEpoch, executeAt, invoker);
- });
+ FetchData.fetch(blockedUntil.minSaveStatus.known, owner.node(), txnId, executeAt, fetchKeys, lowEpoch, highEpoch, invoker);
}
- static void fetch(DefaultProgressLog owner, BlockedUntil blockedUntil, TxnId txnId, Timestamp executeAt, EpochSupplier toLocalEpoch, Route<?> slicedRoute, Route<?> fetchRoute)
+ static void fetch(DefaultProgressLog owner, BlockedUntil blockedUntil, TxnId txnId, Timestamp executeAt, EpochSupplier lowEpoch, EpochSupplier highEpoch, Route<?> slicedRoute, Route<?> fetchRoute, Route<?> maxRoute)
{
Invariants.checkState(!slicedRoute.isEmpty());
// we MUSt allocate before calling withEpoch to register cancellation, as async
BiConsumer<FetchData.FetchResult, Throwable> invoker = invokeWaitingCallback(owner, txnId, blockedUntil, WaitingState::fetchCallback);
- owner.node().withEpoch(blockedUntil.fetchEpoch(txnId, executeAt), invoker, () -> {
- FetchData.fetchSpecific(blockedUntil.minSaveStatus.known, owner.node(), txnId, fetchRoute, slicedRoute, toLocalEpoch, executeAt, invoker);
- });
+ FetchData.fetchSpecific(blockedUntil.minSaveStatus.known, owner.node(), txnId, fetchRoute, maxRoute, slicedRoute, lowEpoch, highEpoch, executeAt, invoker);
}
void awaitHomeKey(DefaultProgressLog owner, BlockedUntil blockedUntil, TxnId txnId, Timestamp executeAt, Route<?> route)
diff --git a/accord-core/src/main/java/accord/local/Bootstrap.java b/accord-core/src/main/java/accord/local/Bootstrap.java
index d77df6f5..2c9799e8 100644
--- a/accord-core/src/main/java/accord/local/Bootstrap.java
+++ b/accord-core/src/main/java/accord/local/Bootstrap.java
@@ -145,7 +145,7 @@ class Bootstrap
// we submit a separate execution so that we know markBootstrapping is durable before we initiate the fetch
safeStore.commandStore().submit(empty(), safeStore0 -> {
store.markBootstrapping(safeStore0, globalSyncId, commitRanges);
- return CoordinateSyncPoint.exclusive(node, globalSyncId, commitRanges);
+ return CoordinateSyncPoint.exclusiveSyncPoint(node, globalSyncId, commitRanges);
}).flatMap(i -> i).flatMap(syncPoint -> node.withEpoch(epoch, () -> store.submit(empty(), safeStore1 -> {
if (valid.isEmpty()) // we've lost ownership of the range
return AsyncResults.success(Ranges.EMPTY);
diff --git a/accord-core/src/main/java/accord/local/Cleanup.java b/accord-core/src/main/java/accord/local/Cleanup.java
index 578912a6..b1a4a901 100644
--- a/accord-core/src/main/java/accord/local/Cleanup.java
+++ b/accord-core/src/main/java/accord/local/Cleanup.java
@@ -18,25 +18,27 @@
package accord.local;
-import accord.local.Status.Durability;
-import accord.primitives.EpochSupplier;
-import accord.primitives.Route;
-import accord.primitives.Timestamp;
+import javax.annotation.Nonnull;
+
+import accord.primitives.SaveStatus;
+import accord.primitives.Status.Durability;
import accord.primitives.TxnId;
import accord.primitives.Unseekables;
-import static accord.local.RedundantBefore.NO_UPPER_BOUND;
import static accord.local.RedundantBefore.PreBootstrapOrStale.FULLY;
+import static accord.local.RedundantStatus.LIVE;
import static accord.local.RedundantStatus.NOT_OWNED;
-import static accord.local.SaveStatus.Erased;
-import static accord.local.SaveStatus.Invalidated;
-import static accord.local.SaveStatus.TruncatedApply;
-import static accord.local.SaveStatus.TruncatedApplyWithOutcome;
-import static accord.local.SaveStatus.Uninitialised;
-import static accord.local.Status.Applied;
-import static accord.local.Status.Durability.Majority;
-import static accord.local.Status.Durability.UniversalOrInvalidated;
-import static accord.local.Status.PreCommitted;
+import static accord.local.RedundantStatus.SHARD_REDUNDANT;
+import static accord.primitives.SaveStatus.Erased;
+import static accord.primitives.SaveStatus.ErasedOrVestigial;
+import static accord.primitives.SaveStatus.Invalidated;
+import static accord.primitives.SaveStatus.TruncatedApply;
+import static accord.primitives.SaveStatus.TruncatedApplyWithOutcome;
+import static accord.primitives.SaveStatus.Uninitialised;
+import static accord.primitives.Status.Applied;
+import static accord.primitives.Status.Durability.Majority;
+import static accord.primitives.Status.Durability.UniversalOrInvalidated;
+import static accord.primitives.Status.PreCommitted;
import static accord.primitives.Txn.Kind.EphemeralRead;
import static accord.utils.Invariants.illegalState;
@@ -50,6 +52,7 @@ public enum Cleanup
TRUNCATE_WITH_OUTCOME(TruncatedApplyWithOutcome),
TRUNCATE(TruncatedApply),
INVALIDATE(Invalidated),
+ VESTIGIAL(ErasedOrVestigial),
ERASE(Erased);
public final SaveStatus appliesIfNot;
@@ -59,84 +62,78 @@ public enum Cleanup
this.appliesIfNot = appliesIfNot;
}
- /**
- * Durability has been achieved for the specific keys associated with this txnId that makes its metadata safe to purge
- */
- public static boolean isSafeToCleanup(DurableBefore durableBefore, TxnId txnId, Unseekables<?> participants)
+ public final Cleanup filter(SaveStatus saveStatus)
{
- return durableBefore.min(txnId, participants) == UniversalOrInvalidated;
+ return saveStatus.compareTo(appliesIfNot) >= 0 ? NO : this;
}
- public static Cleanup shouldCleanup(SafeCommandStore safeStore, Command command, EpochSupplier toEpoch, Unseekables<?> maybeFullRoute)
+ public static Cleanup shouldCleanup(SafeCommandStore safeStore, Command command)
{
- return shouldCleanup(safeStore.commandStore(), command, toEpoch, maybeFullRoute);
+ return shouldCleanup(safeStore.commandStore(), command, command.participants());
}
- public static Cleanup shouldCleanup(CommandStore commandStore, Command command, EpochSupplier toEpoch, Unseekables<?> maybeFullRoute)
+ public static Cleanup shouldCleanup(SafeCommandStore safeStore, Command command, @Nonnull StoreParticipants participants)
{
- return shouldCleanup(commandStore, command, toEpoch, maybeFullRoute, true);
+ return shouldCleanup(safeStore.commandStore(), command, participants);
}
- public static Cleanup shouldCleanup(CommandStore commandStore, Command command, EpochSupplier toEpoch, Unseekables<?> maybeFullRoute, boolean enforceInvariants)
+ public static Cleanup shouldCleanup(CommandStore commandStore, Command command, @Nonnull StoreParticipants participants)
{
- if (command.saveStatus() == Erased)
- return Cleanup.NO; // once erased we no longer have executeAt, and may consider that a transaction is owned by us that should not be (as its txnId is an earlier epoch than we adopted a range)
-
- Route<?> route = command.route();
- if (!Route.isFullRoute(route))
- {
- if (command.known().hasFullRoute()) illegalState(command.saveStatus() + " should include full route, but only found " + route);
- else if (Route.isFullRoute(maybeFullRoute)) route = Route.castToFullRoute(maybeFullRoute);
- else return Cleanup.NO;
- }
-
- Timestamp executeAt = command.executeAt();
- if (toEpoch == null || (executeAt != null && executeAt.epoch() > toEpoch.epoch()))
- toEpoch = executeAt;
+ return shouldCleanup(command.txnId(), command.saveStatus(), command.durability(), participants,
+ commandStore.redundantBefore(), commandStore.durableBefore());
+ }
- return shouldCleanup(command.txnId(), command.status(), command.durability(), toEpoch, route,
- commandStore.redundantBefore(), commandStore.durableBefore(), enforceInvariants);
+ public static Cleanup shouldCleanup(TxnId txnId, SaveStatus status, Durability durability, StoreParticipants participants, RedundantBefore redundantBefore, DurableBefore durableBefore)
+ {
+ return shouldCleanup(txnId, status, durability, participants, redundantBefore, durableBefore, true);
}
- public static Cleanup shouldCleanup(TxnId txnId, Status status, Durability durability, EpochSupplier toEpoch, Route<?> route, RedundantBefore redundantBefore, DurableBefore durableBefore)
+ public static Cleanup shouldCleanup(TxnId txnId, SaveStatus status, Durability durability, StoreParticipants participants, RedundantBefore redundantBefore, DurableBefore durableBefore, boolean enforceInvariants)
{
- return shouldCleanup(txnId, status, durability, toEpoch, route, redundantBefore, durableBefore, true);
+ return shouldCleanupInternal(txnId, status, durability, participants, redundantBefore, durableBefore, enforceInvariants).filter(status);
}
- public static Cleanup shouldCleanup(TxnId txnId, Status status, Durability durability, EpochSupplier toEpoch, Route<?> route, RedundantBefore redundantBefore, DurableBefore durableBefore, boolean enforceInvariants)
+ private static Cleanup shouldCleanupInternal(TxnId txnId, SaveStatus status, Durability durability, StoreParticipants participants, RedundantBefore redundantBefore, DurableBefore durableBefore, boolean enforceInvariants)
{
if (txnId.kind() == EphemeralRead)
- return Cleanup.NO; // TODO (required): clean-up based on timeout
+ return NO; // TODO (required): clean-up based on timeout
- if (durableBefore.min(txnId) == UniversalOrInvalidated)
+ if (status == Uninitialised)
{
- if (status.hasBeen(PreCommitted) && !status.hasBeen(Applied)) // TODO (expected): either stale or pre-bootstrap
- illegalState("Loading universally-durable command that has been PreCommitted but not Applied");
- return Cleanup.ERASE;
+ if (!redundantBefore.isAnyOnAnyEpoch(txnId, participants.touches, SHARD_REDUNDANT))
+ return NO;
+
+ // participants.touches() means e.g. we used to or will own the participant, but shard redundant means all
+ // owners of the command have applied it - if we aren't guaranteed to know it and we don't then it is
+ // "vestigial" i.e. represents some attempt to coordinate the command against us (e.g. failed propose or calculateDeps)
+ Cleanup cleanup = VESTIGIAL;
+ if (redundantBefore.isAnyOnCoordinationEpoch(txnId, participants.owns, SHARD_REDUNDANT))
+ {
+ // owns means the key is known to interact with us is shard redundant then we
+ cleanup = ERASE;
+ if (redundantBefore.isAnyOnCoordinationEpoch(txnId, participants.owns, LIVE))
+ cleanup = INVALIDATE;
+ }
+ return cleanup;
}
- if (!Route.isFullRoute(route))
+ if (!status.hasBeen(PreCommitted) && redundantBefore.isAnyOnCoordinationEpoch(txnId, participants.owns, SHARD_REDUNDANT))
+ return Cleanup.INVALIDATE;
+
+ if (!participants.hasFullRoute())
+ {
+ if (status == Invalidated && durableBefore.min(txnId) == UniversalOrInvalidated)
+ return Cleanup.ERASE;
+
return Cleanup.NO;
+ }
- // TODO (required): should we apply additionalKeysOrRanges() to calculations here?
- // TODO (required): enrich with additional epochs we know the command applies to
// We first check if the command is redundant locally, i.e. whether it has been applied to all non-faulty replicas of the local shard
// If not, we don't want to truncate its state else we may make catching up for these other replicas much harder
- RedundantStatus redundant = redundantBefore.status(txnId, toEpoch, route.participants());
+ RedundantStatus redundant = redundantBefore.status(txnId, participants.route);
if (redundant == NOT_OWNED)
- {
- // ONLY upgrade to looking to future if NOT_OWNED, as if we're owned we can safely manage the lifecycle whatever epochs it participated in
- // however, if this is a vestigial preaccept/accept in an epoch after the execution epoch we may not clean up
- redundant = redundantBefore.status(txnId, NO_UPPER_BOUND, route.participants());
- if (redundant == NOT_OWNED)
- {
- // TODO (expected): improve handling of epoch bounds
- // - we can impose additional validations here IF we receive an epoch upper bound
- // - we should be more robust to the presence/absence of executeAt
- // - be cognisant of future epochs that participated only for PreAccept/Accept, but where txn was not committed to execute in the epoch (this is why we provide null toEpoch here)
- illegalState("Command %s that is being loaded is not owned by this shard on route %s. Redundant before: %s", txnId, route, redundantBefore);
- }
- }
+ illegalState("Command " + txnId + " that is being loaded is not owned by this shard on route " + participants.route);
+
switch (redundant)
{
default: throw new AssertionError();
@@ -147,14 +144,15 @@ public enum Cleanup
case LOCALLY_REDUNDANT:
return Cleanup.NO;
case SHARD_REDUNDANT:
- if (enforceInvariants && status.hasBeen(PreCommitted) && !status.hasBeen(Applied) && redundantBefore.preBootstrapOrStale(txnId, toEpoch, route.participants()) != FULLY)
+ if (enforceInvariants && status.hasBeen(PreCommitted) && !status.hasBeen(Applied) && redundantBefore.preBootstrapOrStale(txnId, participants.owns) != FULLY)
illegalState("Loading redundant command that has been PreCommitted but not Applied.");
+ case WAS_OWNED:
}
if (!status.hasBeen(PreCommitted))
return INVALIDATE;
- Durability min = durableBefore.min(txnId, route);
+ Durability min = durableBefore.min(txnId, participants.route);
switch (min)
{
default:
@@ -162,7 +160,6 @@ public enum Cleanup
throw new AssertionError("Unexpected durability: " + min);
case NotDurable:
- // TODO (required): disambiguate INVALIDATE and TRUNCATE_WITH_OUTCOME, and impose stronger invariants on former
if (durability == Majority)
return Cleanup.TRUNCATE;
return Cleanup.TRUNCATE_WITH_OUTCOME;
diff --git a/accord-core/src/main/java/accord/local/Command.java b/accord-core/src/main/java/accord/local/Command.java
index ddc4b5a8..e7033feb 100644
--- a/accord-core/src/main/java/accord/local/Command.java
+++ b/accord-core/src/main/java/accord/local/Command.java
@@ -24,18 +24,17 @@ import java.util.List;
import java.util.Objects;
import javax.annotation.Nullable;
-import accord.api.Key;
import accord.api.Result;
import accord.api.RoutingKey;
import accord.api.VisibleForImplementation;
-import accord.local.Status.Durability;
-import accord.local.Status.Known;
+import accord.primitives.SaveStatus;
+import accord.primitives.Status;
+import accord.primitives.Status.Durability;
+import accord.primitives.Known;
import accord.local.cfk.CommandsForKey;
import accord.primitives.Ballot;
import accord.primitives.Deps;
-import accord.primitives.FullRoute;
import accord.primitives.KeyDeps;
-import accord.primitives.Keys;
import accord.primitives.PartialDeps;
import accord.primitives.PartialTxn;
import accord.primitives.Participants;
@@ -43,7 +42,7 @@ import accord.primitives.RangeDeps;
import accord.primitives.Ranges;
import accord.primitives.Routable;
import accord.primitives.Route;
-import accord.primitives.Seekables;
+import accord.primitives.RoutingKeys;
import accord.primitives.Timestamp;
import accord.primitives.Txn;
import accord.primitives.TxnId;
@@ -55,17 +54,19 @@ import accord.utils.IndexedTriConsumer;
import accord.utils.Invariants;
import accord.utils.SimpleBitSet;
+import javax.annotation.Nonnull;
+
import static accord.local.Command.AbstractCommand.validate;
-import static accord.local.SaveStatus.AcceptedInvalidate;
-import static accord.local.SaveStatus.ErasedOrInvalidOrVestigial;
-import static accord.local.SaveStatus.Uninitialised;
-import static accord.local.Status.Durability.Local;
-import static accord.local.Status.Durability.NotDurable;
-import static accord.local.Status.Durability.ShardUniversal;
-import static accord.local.Status.Durability.UniversalOrInvalidated;
-import static accord.local.Status.Invalidated;
-import static accord.local.Status.KnownExecuteAt.ExecuteAtKnown;
-import static accord.local.Status.Stable;
+import static accord.primitives.SaveStatus.AcceptedInvalidate;
+import static accord.primitives.SaveStatus.ErasedOrVestigial;
+import static accord.primitives.SaveStatus.Uninitialised;
+import static accord.primitives.Status.Durability.Local;
+import static accord.primitives.Status.Durability.NotDurable;
+import static accord.primitives.Status.Durability.ShardUniversal;
+import static accord.primitives.Status.Durability.UniversalOrInvalidated;
+import static accord.primitives.Status.Invalidated;
+import static accord.primitives.Known.KnownExecuteAt.ExecuteAtKnown;
+import static accord.primitives.Status.Stable;
import static accord.primitives.Routable.Domain.Range;
import static accord.primitives.Routables.Slice.Minimal;
import static accord.utils.Invariants.illegalArgument;
@@ -79,7 +80,7 @@ public abstract class Command implements CommonAttributes
{
private static Durability durability(Durability durability, SaveStatus status)
{
- if (durability == NotDurable && status.compareTo(SaveStatus.PreApplied) >= 0 && status.compareTo(ErasedOrInvalidOrVestigial) < 0)
+ if (durability == NotDurable && status.compareTo(SaveStatus.PreApplied) >= 0 && status.compareTo(ErasedOrVestigial) < 0)
return Local; // not necessary anywhere, but helps for logical consistency
return durability;
}
@@ -173,10 +174,10 @@ public abstract class Command implements CommonAttributes
case TruncatedApply:
case TruncatedApplyWithDeps:
case TruncatedApplyWithOutcome:
- if (txnId.kind().awaitsOnlyDeps())
+ if (txnId.awaitsOnlyDeps())
return validateCommandClass(status, TruncatedAwaitsOnlyDeps.class, klass);
case Erased:
- case ErasedOrInvalidOrVestigial:
+ case ErasedOrVestigial:
case Invalidated:
return validateCommandClass(status, Truncated.class, klass);
default:
@@ -189,18 +190,15 @@ public abstract class Command implements CommonAttributes
private final TxnId txnId;
private final SaveStatus status;
private final Durability durability;
- private final @Nullable Route<?> route;
- // TODO (expected): more consistent handling of keys for transactions that only *may* intersect a command store, as have both additionalKeysOrRanges AND forLocalEpoch to separately handle the problem
- private final @Nullable Seekables<?, ?> additionalKeysOrRanges;
+ private final @Nonnull StoreParticipants participants;
private final Ballot promised;
- private AbstractCommand(TxnId txnId, SaveStatus status, Durability durability, @Nullable Route<?> route, @Nullable Seekables<?, ?> additionalKeysOrRanges, Ballot promised)
+ private AbstractCommand(TxnId txnId, SaveStatus status, Durability durability, @Nonnull StoreParticipants participants, Ballot promised)
{
this.txnId = txnId;
this.status = validateCommandClass(txnId, status, getClass());
this.durability = durability;
- this.route = route;
- this.additionalKeysOrRanges = additionalKeysOrRanges;
+ this.participants = participants;
this.promised = promised;
}
@@ -209,8 +207,7 @@ public abstract class Command implements CommonAttributes
this.txnId = common.txnId();
this.status = validateCommandClass(txnId, status, getClass());
this.durability = common.durability();
- this.route = common.route();
- this.additionalKeysOrRanges = common.additionalKeysOrRanges();
+ this.participants = common.participants();
this.promised = promised;
}
@@ -224,7 +221,7 @@ public abstract class Command implements CommonAttributes
&& saveStatus() == command.saveStatus()
&& durability() == command.durability()
&& Objects.equals(route(), command.route())
- && Objects.equals(additionalKeysOrRanges, command.additionalKeysOrRanges())
+ && Objects.equals(participants, command.participants())
&& Objects.equals(promised(), command.promised());
}
@@ -241,9 +238,9 @@ public abstract class Command implements CommonAttributes
}
@Override
- public final Route<?> route()
+ public final StoreParticipants participants()
{
- return route;
+ return participants;
}
@Override
@@ -264,12 +261,6 @@ public abstract class Command implements CommonAttributes
return status;
}
- @Override
- public @Nullable Seekables<?, ?> additionalKeysOrRanges()
- {
- return additionalKeysOrRanges;
- }
-
public static <T extends AbstractCommand> T validate(T validate)
{
Known known = validate.known();
@@ -302,6 +293,7 @@ public abstract class Command implements CommonAttributes
default: throw new AssertionError("Unhandled KnownExecuteAt: " + known.executeAt);
case ExecuteAtErased:
case ExecuteAtUnknown:
+ Invariants.checkState(executeAt == null || executeAt.compareTo(validate.txnId()) >= 0);
break;
case ExecuteAtProposed:
case ExecuteAtKnown:
@@ -389,7 +381,7 @@ public abstract class Command implements CommonAttributes
*/
@Override
@Nullable
- public abstract Route<?> route();
+ public final Route<?> route() { return participants().route(); }
/**
* We require that this is a FullRoute for all states where isDefinitionKnown().
@@ -403,14 +395,13 @@ public abstract class Command implements CommonAttributes
* or any route will do
*/
@Override
- @Nullable
- public abstract Participants<?> participants();
+ public abstract StoreParticipants participants();
public Participants<?> maxContactable()
{
Route<?> route = route();
if (route != null) return route.withHomeKey();
- else return participants();
+ else return participants().hasTouched();
}
/**
@@ -470,7 +461,6 @@ public abstract class Command implements CommonAttributes
@Override
public abstract PartialTxn partialTxn();
- public abstract Seekables<?, ?> keysOrRanges();
@Override
public abstract @Nullable PartialDeps partialDeps();
@@ -602,18 +592,14 @@ public abstract class Command implements CommonAttributes
public static class NotDefined extends AbstractCommand
{
- final Participants<?> participants;
- NotDefined(TxnId txnId, SaveStatus status, Durability durability, @Nullable Route<?> route, @Nullable Participants<?> participants, @Nullable Seekables<?, ?> additionalKeysOrRanges, Ballot promised)
+ NotDefined(TxnId txnId, SaveStatus status, Durability durability, @Nonnull StoreParticipants participants, Ballot promised)
{
- super(txnId, status, durability, route, additionalKeysOrRanges, promised);
- this.participants = participants;
- Invariants.checkState(route == participants || route == null);
+ super(txnId, status, durability, participants, promised);
}
NotDefined(CommonAttributes common, SaveStatus status, Ballot promised)
{
super(common, status, promised);
- participants = common.participants();
}
@Override
@@ -629,14 +615,7 @@ public abstract class Command implements CommonAttributes
public static NotDefined uninitialised(TxnId txnId)
{
- return validate(new NotDefined(txnId, Uninitialised, NotDurable, null, null, null, Ballot.ZERO));
- }
-
- @Nullable
- @Override
- public Participants<?> participants()
- {
- return participants;
+ return validate(new NotDefined(txnId, Uninitialised, NotDurable, StoreParticipants.empty(txnId), Ballot.ZERO));
}
@Override
@@ -657,12 +636,6 @@ public abstract class Command implements CommonAttributes
return null;
}
- @Override
- public Seekables<?, ?> keysOrRanges()
- {
- return additionalKeysOrRanges();
- }
-
@Override
public @Nullable PartialDeps partialDeps()
{
@@ -689,9 +662,9 @@ public abstract class Command implements CommonAttributes
this.result = result;
}
- public Truncated(TxnId txnId, SaveStatus saveStatus, Durability durability, @Nullable Route<?> route, @Nullable Timestamp executeAt, @Nullable Writes writes, @Nullable Result result)
+ public Truncated(TxnId txnId, SaveStatus saveStatus, Durability durability, @Nonnull StoreParticipants participants, @Nullable Timestamp executeAt, @Nullable Writes writes, @Nullable Result result)
{
- super(txnId, saveStatus, durability, route, null, Ballot.MAX);
+ super(txnId, saveStatus, durability, participants, Ballot.MAX);
this.executeAt = executeAt;
this.writes = writes;
this.result = result;
@@ -712,22 +685,22 @@ public abstract class Command implements CommonAttributes
public static Truncated erased(Command command)
{
Durability durability = Durability.mergeAtLeast(command.durability(), UniversalOrInvalidated);
- return erased(command.txnId(), durability, command.route());
+ return erased(command.txnId(), durability, command.participants());
}
- public static Truncated erased(TxnId txnId, Status.Durability durability, Route<?> route)
+ public static Truncated erased(TxnId txnId, Status.Durability durability, StoreParticipants participants)
{
- return validate(new Truncated(txnId, SaveStatus.Erased, durability, route, null, null, null));
+ return validate(new Truncated(txnId, SaveStatus.Erased, durability, participants, null, null, null));
}
public static Truncated erasedOrInvalidOrVestigial(Command command)
{
- return erasedOrInvalidOrVestigial(command.txnId(), command.durability(), command.route());
+ return erasedOrInvalidOrVestigial(command.txnId(), command.durability(), command.participants());
}
- public static Truncated erasedOrInvalidOrVestigial(TxnId txnId, Status.Durability durability, Route<?> route)
+ public static Truncated erasedOrInvalidOrVestigial(TxnId txnId, Status.Durability durability, StoreParticipants participants)
{
- return validate(new Truncated(txnId, SaveStatus.ErasedOrInvalidOrVestigial, durability, route, null, null, null));
+ return validate(new Truncated(txnId, SaveStatus.ErasedOrVestigial, durability, participants, null, null, null));
}
public static Truncated truncatedApply(Command command)
@@ -735,43 +708,44 @@ public abstract class Command implements CommonAttributes
return truncatedApply(command, null);
}
- public static Truncated truncatedApply(Command command, @Nullable FullRoute<?> route)
+ public static Truncated truncatedApply(Command command, @Nullable StoreParticipants participants)
{
Invariants.checkArgument(command.known().executeAt.isDecidedAndKnownToExecute());
- if (route == null) route = Route.castToNonNullFullRoute(command.route());
+ if (participants == null) participants = command.participants();
+ else participants = command.participants().supplement(participants);
Durability durability = Durability.mergeAtLeast(command.durability(), ShardUniversal);
- if (command.txnId().kind().awaitsOnlyDeps())
+ if (command.txnId().awaitsOnlyDeps())
{
Timestamp executesAtLeast = command.hasBeen(Stable) ? command.executesAtLeast() : null;
- return validate(new TruncatedAwaitsOnlyDeps(command.txnId(), SaveStatus.TruncatedApply, durability, route, command.executeAt(), null, null, executesAtLeast));
+ return validate(new TruncatedAwaitsOnlyDeps(command.txnId(), SaveStatus.TruncatedApply, durability, participants, command.executeAt(), null, null, executesAtLeast));
}
- return validate(new Truncated(command.txnId(), SaveStatus.TruncatedApply, durability, route, command.executeAt(), null, null));
+ return validate(new Truncated(command.txnId(), SaveStatus.TruncatedApply, durability, participants, command.executeAt(), null, null));
}
public static Truncated truncatedApplyWithOutcome(Executed command)
{
Durability durability = Durability.mergeAtLeast(command.durability(), ShardUniversal);
- if (command.txnId().kind().awaitsOnlyDeps())
- return validate(new TruncatedAwaitsOnlyDeps(command.txnId(), SaveStatus.TruncatedApplyWithOutcome, durability, command.route(), command.executeAt(), command.writes, command.result, command.executesAtLeast()));
- return validate(new Truncated(command.txnId(), SaveStatus.TruncatedApplyWithOutcome, durability, command.route(), command.executeAt(), command.writes, command.result));
+ if (command.txnId().awaitsOnlyDeps())
+ return validate(new TruncatedAwaitsOnlyDeps(command.txnId(), SaveStatus.TruncatedApplyWithOutcome, durability, command.participants(), command.executeAt(), command.writes, command.result, command.executesAtLeast()));
+ return validate(new Truncated(command.txnId(), SaveStatus.TruncatedApplyWithOutcome, durability, command.participants(), command.executeAt(), command.writes, command.result));
}
public static Truncated truncatedApply(CommonAttributes common, SaveStatus saveStatus, Timestamp executeAt, Writes writes, Result result)
{
- Invariants.checkArgument(!common.txnId().kind().awaitsOnlyDeps());
+ Invariants.checkArgument(!common.txnId().awaitsOnlyDeps());
Durability durability = checkTruncatedApplyInvariants(common, saveStatus, executeAt);
- return validate(new Truncated(common.txnId(), saveStatus, durability, common.route(), executeAt, writes, result));
+ return validate(new Truncated(common.txnId(), saveStatus, durability, common.participants(), executeAt, writes, result));
}
public static Truncated truncatedApply(CommonAttributes common, SaveStatus saveStatus, Timestamp executeAt, Writes writes, Result result, @Nullable Timestamp dependencyExecutesAt)
{
- if (!common.txnId().kind().awaitsOnlyDeps())
+ if (!common.txnId().awaitsOnlyDeps())
{
Invariants.checkState(dependencyExecutesAt == null);
return truncatedApply(common, saveStatus, executeAt, writes, result);
}
Durability durability = checkTruncatedApplyInvariants(common, saveStatus, executeAt);
- return validate(new TruncatedAwaitsOnlyDeps(common.txnId(), saveStatus, durability, common.route(), executeAt, writes, result, dependencyExecutesAt));
+ return validate(new TruncatedAwaitsOnlyDeps(common.txnId(), saveStatus, durability, common.participants(), executeAt, writes, result, dependencyExecutesAt));
}
private static Durability checkTruncatedApplyInvariants(CommonAttributes common, SaveStatus saveStatus, Timestamp executeAt)
@@ -791,14 +765,7 @@ public abstract class Command implements CommonAttributes
{
// TODO (expected): migrate to using null for executeAt when invalidated
// TODO (expected): is UniversalOrInvalidated correct here? Should we have a lower implication pure Invalidated?
- return validate(new Truncated(txnId, SaveStatus.Invalidated, UniversalOrInvalidated, null, Timestamp.NONE, null, null));
- }
-
- @Nullable
- @Override
- public final Participants<?> participants()
- {
- return route();
+ return validate(new Truncated(txnId, SaveStatus.Invalidated, UniversalOrInvalidated, StoreParticipants.empty(txnId), Timestamp.NONE, null, null));
}
@Override
@@ -831,12 +798,6 @@ public abstract class Command implements CommonAttributes
return null;
}
- @Override
- public Seekables<?, ?> keysOrRanges()
- {
- return null;
- }
-
@Override
public @Nullable PartialDeps partialDeps()
{
@@ -846,8 +807,7 @@ public abstract class Command implements CommonAttributes
@Override
public Command updateAttributes(CommonAttributes attrs, Ballot promised)
{
- // TODO (now): invoke listeners precisely once when we adopt this state, then we can simply return `this`
- return validate(new Truncated(txnId(), saveStatus(), attrs.durability(), attrs.route(), executeAt, writes, result));
+ return validate(new Truncated(txnId(), saveStatus(), attrs.durability(), attrs.participants(), executeAt, writes, result));
}
}
@@ -865,9 +825,9 @@ public abstract class Command implements CommonAttributes
this.executesAtLeast = executesAtLeast;
}
- public TruncatedAwaitsOnlyDeps(TxnId txnId, SaveStatus saveStatus, Durability durability, @Nullable Route<?> route, @Nullable Timestamp executeAt, @Nullable Writes writes, @Nullable Result result, @Nullable Timestamp executesAtLeast)
+ public TruncatedAwaitsOnlyDeps(TxnId txnId, SaveStatus saveStatus, Durability durability, StoreParticipants participants, @Nullable Timestamp executeAt, @Nullable Writes writes, @Nullable Result result, @Nullable Timestamp executesAtLeast)
{
- super(txnId, saveStatus, durability, route, executeAt, writes, result);
+ super(txnId, saveStatus, durability, participants, executeAt, writes, result);
this.executesAtLeast = executesAtLeast;
}
@@ -916,13 +876,6 @@ public abstract class Command implements CommonAttributes
this.partialDeps = partialDeps;
}
- @Nullable
- @Override
- public final Participants<?> participants()
- {
- return route();
- }
-
@Override
public Timestamp executeAt()
{
@@ -941,14 +894,6 @@ public abstract class Command implements CommonAttributes
return partialTxn;
}
- @Override
- public Seekables<?, ?> keysOrRanges()
- {
- return partialTxn == null ? additionalKeysOrRanges()
- : additionalKeysOrRanges() == null
- ? partialTxn.keys() : ((Seekables)partialTxn.keys()).with(additionalKeysOrRanges());
- }
-
@Override
public @Nullable PartialDeps partialDeps()
{
@@ -1082,7 +1027,7 @@ public abstract class Command implements CommonAttributes
@Override
public Timestamp executesAtLeast()
{
- if (!txnId().kind().awaitsOnlyDeps()) return executeAt();
+ if (!txnId().awaitsOnlyDeps()) return executeAt();
if (status().hasBeen(Stable)) return waitingOn.executeAtLeast(executeAt());
return null;
}
@@ -1210,8 +1155,8 @@ public abstract class Command implements CommonAttributes
public static class WaitingOn
{
- private static final WaitingOn EMPTY_FOR_KEY = new WaitingOn(Keys.EMPTY, RangeDeps.NONE, KeyDeps.NONE, ImmutableBitSet.EMPTY, null);
- private static final WaitingOn EMPTY_FOR_RANGE = new WaitingOn(Keys.EMPTY, RangeDeps.NONE, KeyDeps.NONE, ImmutableBitSet.EMPTY, ImmutableBitSet.EMPTY);
+ private static final WaitingOn EMPTY_FOR_KEY = new WaitingOn(RoutingKeys.EMPTY, RangeDeps.NONE, KeyDeps.NONE, ImmutableBitSet.EMPTY, null);
+ private static final WaitingOn EMPTY_FOR_RANGE = new WaitingOn(RoutingKeys.EMPTY, RangeDeps.NONE, KeyDeps.NONE, ImmutableBitSet.EMPTY, ImmutableBitSet.EMPTY);
public static WaitingOn empty(Routable.Domain domain)
{
@@ -1220,7 +1165,7 @@ public abstract class Command implements CommonAttributes
return EMPTY_FOR_KEY;
}
- public final Keys keys;
+ public final RoutingKeys keys;
public final RangeDeps directRangeDeps;
public final KeyDeps directKeyDeps;
// waitingOn ONLY encodes both txnIds and keys
@@ -1232,7 +1177,7 @@ public abstract class Command implements CommonAttributes
this(copy.keys, copy.directRangeDeps, copy.directKeyDeps, copy.waitingOn, copy.appliedOrInvalidated);
}
- public WaitingOn(Keys keys, RangeDeps directRangeDeps, KeyDeps directKeyDeps, ImmutableBitSet waitingOn, ImmutableBitSet appliedOrInvalidated)
+ public WaitingOn(RoutingKeys keys, RangeDeps directRangeDeps, KeyDeps directKeyDeps, ImmutableBitSet waitingOn, ImmutableBitSet appliedOrInvalidated)
{
this.keys = keys;
this.directRangeDeps = directRangeDeps;
@@ -1300,7 +1245,7 @@ public abstract class Command implements CommonAttributes
return waitingOn.lastSetBit() >= txnIdCount();
}
- public Key lastWaitingOnKey()
+ public RoutingKey lastWaitingOnKey()
{
int keyIndex = waitingOn.lastSetBit() - txnIdCount();
if (keyIndex < 0) return null;
@@ -1352,7 +1297,7 @@ public abstract class Command implements CommonAttributes
public String toString()
{
List<TxnId> waitingOnTxnIds = new ArrayList<>();
- List<Key> waitingOnKeys = new ArrayList<>();
+ List<RoutingKey> waitingOnKeys = new ArrayList<>();
waitingOn.reverseForEach(waitingOnTxnIds, waitingOnKeys, this, keys, (outIds, outKeys, self, ks, i) -> {
if (i < self.txnIdCount()) outIds.add(self.txnId(i));
else outKeys.add(ks.get(i - self.txnIdCount()));
@@ -1378,7 +1323,7 @@ public abstract class Command implements CommonAttributes
public static class Update
{
- final Keys keys;
+ final RoutingKeys keys;
final RangeDeps directRangeDeps;
final KeyDeps directKeyDeps;
private SimpleBitSet waitingOn;
@@ -1401,7 +1346,7 @@ public abstract class Command implements CommonAttributes
this(committed.waitingOn);
}
- private Update(TxnId txnId, Keys keys, RangeDeps directRangeDeps, KeyDeps directKeyDeps)
+ private Update(TxnId txnId, RoutingKeys keys, RangeDeps directRangeDeps, KeyDeps directKeyDeps)
{
this.keys = keys;
this.directRangeDeps = directRangeDeps;
@@ -1416,7 +1361,8 @@ public abstract class Command implements CommonAttributes
Update update = new Update(txnId, deps.keyDeps.keys(), deps.rangeDeps, deps.directKeyDeps);
deps.rangeDeps.forEach(executionParticipants, update, Update::initialise);
deps.directKeyDeps.forEach(ranges, 0, deps.directKeyDeps.txnIdCount(), update, deps.rangeDeps, (upd, rdeps, index) -> upd.initialise(index + rdeps.txnIdCount()));
- deps.keyDeps.keys().forEach(ranges, (upd, key, index) -> upd.initialise(index + upd.txnIdCount()), update);
+ deps.keyDeps.keys()
+ .forEach(ranges, (upd, key, index) -> upd.initialise(index + upd.txnIdCount()), update);
return update;
}
@@ -1478,7 +1424,7 @@ public abstract class Command implements CommonAttributes
return i >= 0 && !waitingOn.get(i);
}
- public boolean removeWaitingOn(Key key)
+ public boolean removeWaitingOn(RoutingKey key)
{
int index = keys.indexOf(key);
// we can be a member of a CFK we aren't waiting on, if we Accept in a later epoch using a key we don't own once Committed to an earlier eopch
@@ -1494,7 +1440,7 @@ public abstract class Command implements CommonAttributes
return true;
}
- public boolean isWaitingOn(Key key)
+ public boolean isWaitingOn(RoutingKey key)
{
int index = keys.indexOf(key) + txnIdCount();
return index >= 0 && waitingOn.get(index);
@@ -1668,7 +1614,7 @@ public abstract class Command implements CommonAttributes
public String toString()
{
List<TxnId> waitingOnTxnIds = new ArrayList<>();
- List<Key> waitingOnKeys = new ArrayList<>();
+ List<RoutingKey> waitingOnKeys = new ArrayList<>();
waitingOn.reverseForEach(waitingOnTxnIds, waitingOnKeys, this, keys, (outIds, outKeys, self, ks, i) -> {
if (i < self.txnIdCount()) outIds.add(self.txnId(i));
else outKeys.add(ks.get(i - self.txnIdCount()));
diff --git a/accord-core/src/main/java/accord/local/CommandStore.java b/accord-core/src/main/java/accord/local/CommandStore.java
index 6c15cbc7..e149f1ad 100644
--- a/accord-core/src/main/java/accord/local/CommandStore.java
+++ b/accord-core/src/main/java/accord/local/CommandStore.java
@@ -18,12 +18,35 @@
package accord.local;
+import accord.api.LocalListeners;
+import accord.api.ProgressLog;
+import accord.api.DataStore;
+import accord.coordinate.CollectCalculatedDeps;
+import accord.local.Command.WaitingOn;
+
+import javax.annotation.Nullable;
+import accord.api.Agent;
+
+import accord.local.CommandStores.RangesForEpoch;
+import accord.primitives.KeyDeps;
+import accord.primitives.Participants;
+import accord.primitives.Range;
+import accord.primitives.Routables;
+import accord.primitives.RoutingKeys;
+import accord.primitives.Status;
+import accord.primitives.Unseekables;
+import accord.utils.async.AsyncChain;
+
+import accord.api.ConfigurationService.EpochReady;
+import accord.utils.DeterministicIdentitySet;
+import accord.utils.Invariants;
+import accord.utils.async.AsyncResult;
+
import java.util.AbstractMap.SimpleImmutableEntry;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.NavigableMap;
-import java.util.Objects;
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.Callable;
@@ -31,39 +54,18 @@ import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.Supplier;
-import javax.annotation.Nullable;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableSortedMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import accord.api.Agent;
-import accord.api.ConfigurationService.EpochReady;
-import accord.api.DataStore;
-import accord.api.LocalListeners;
-import accord.api.ProgressLog;
-import accord.coordinate.CollectCalculatedDeps;
-import accord.local.Command.WaitingOn;
-import accord.local.CommandStores.RangesForEpoch;
import accord.primitives.Deps;
import accord.primitives.FullRoute;
-import accord.primitives.KeyDeps;
-import accord.primitives.Keys;
-import accord.primitives.Participants;
-import accord.primitives.Range;
import accord.primitives.RangeDeps;
import accord.primitives.Ranges;
-import accord.primitives.Routables;
-import accord.primitives.Seekables;
import accord.primitives.Timestamp;
import accord.primitives.TxnId;
-import accord.primitives.Unseekables;
-import accord.utils.DeterministicIdentitySet;
-import accord.utils.Invariants;
-import accord.utils.ReducingRangeMap;
-import accord.utils.async.AsyncChain;
-import accord.utils.async.AsyncResult;
import accord.utils.async.AsyncResults;
import org.agrona.collections.Int2ObjectHashMap;
@@ -154,12 +156,12 @@ public abstract class CommandStore implements AgentExecutor
// Used in markShardStale to make sure the staleness includes in progresss bootstraps
private transient NavigableMap<TxnId, Ranges> bootstrapBeganAt = ImmutableSortedMap.of(TxnId.NONE, Ranges.EMPTY); // additive (i.e. once inserted, rolled-over until invalidated, and the floor entry contains additions)
-
private RedundantBefore redundantBefore = RedundantBefore.EMPTY;
// TODO (expected): store this only once per node
private DurableBefore durableBefore = DurableBefore.EMPTY;
private MaxConflicts maxConflicts = MaxConflicts.EMPTY;
protected RangesForEpoch rangesForEpoch;
+ protected Ranges activeRanges, leavingRanges;
/**
* safeToRead is related to RedundantBefore, but a distinct concept.
@@ -179,7 +181,7 @@ public abstract class CommandStore implements AgentExecutor
*/
private NavigableMap<Timestamp, Ranges> safeToRead = ImmutableSortedMap.of(Timestamp.NONE, Ranges.EMPTY);
private final Set<Bootstrap> bootstraps = Collections.synchronizedSet(new DeterministicIdentitySet<>());
- @Nullable private ReducingRangeMap<Timestamp> rejectBefore;
+ @Nullable private RejectBefore rejectBefore;
protected CommandStore(int id,
NodeTimeService time,
@@ -254,7 +256,7 @@ public abstract class CommandStore implements AgentExecutor
durableBefore = DurableBefore.merge(durableBefore, addDurableBefore);
}
- protected void unsafeSetRejectBefore(ReducingRangeMap<Timestamp> newRejectBefore)
+ protected void unsafeSetRejectBefore(RejectBefore newRejectBefore)
{
this.rejectBefore = newRejectBefore;
}
@@ -300,26 +302,24 @@ public abstract class CommandStore implements AgentExecutor
{
Timestamp executeAt = updated.executeAt();
if (executeAt == null) return;
- Seekables<?, ?> keysOrRanges = updated.keysOrRanges();
- if (keysOrRanges == null) return;
if (prev != null && prev.executeAt() != null && prev.executeAt().compareTo(executeAt) >= 0) return;
- setMaxConflicts(maxConflicts.update(keysOrRanges, executeAt));
+ setMaxConflicts(maxConflicts.update(updated.participants().hasTouched(), executeAt));
}
public final void markExclusiveSyncPoint(SafeCommandStore safeStore, TxnId txnId, Ranges ranges)
{
// TODO (desired): narrow ranges to those that are owned
- Invariants.checkArgument(txnId.kind() == ExclusiveSyncPoint);
- ReducingRangeMap<Timestamp> newRejectBefore = rejectBefore != null ? rejectBefore : new ReducingRangeMap<>();
- newRejectBefore = ReducingRangeMap.add(newRejectBefore, ranges, txnId, Timestamp::max);
+ Invariants.checkArgument(txnId.is(ExclusiveSyncPoint));
+ RejectBefore newRejectBefore = rejectBefore != null ? rejectBefore : new RejectBefore();
+ newRejectBefore = RejectBefore.add(newRejectBefore, ranges, txnId);
unsafeSetRejectBefore(newRejectBefore);
}
public final void markExclusiveSyncPointLocallyApplied(SafeCommandStore safeStore, TxnId txnId, Ranges ranges)
{
// TODO (desired): narrow ranges to those that are owned
- Invariants.checkArgument(txnId.kind() == ExclusiveSyncPoint);
+ Invariants.checkArgument(txnId.is(ExclusiveSyncPoint));
RedundantBefore newRedundantBefore = RedundantBefore.merge(redundantBefore, RedundantBefore.create(ranges, txnId, TxnId.NONE, TxnId.NONE, TxnId.NONE));
unsafeSetRedundantBefore(newRedundantBefore);
updatedRedundantBefore(safeStore, txnId, ranges);
@@ -328,28 +328,28 @@ public abstract class CommandStore implements AgentExecutor
/**
* We expect keys to be sliced to those owned by the replica in the coordination epoch
*/
- final Timestamp preaccept(TxnId txnId, Seekables<?, ?> keys, SafeCommandStore safeStore, boolean permitFastPath)
+ final Timestamp preaccept(TxnId txnId, Routables<?> keys, SafeCommandStore safeStore, boolean permitFastPath)
{
NodeTimeService time = safeStore.time();
- boolean isExpired = time.now() - txnId.hlc() >= safeStore.preAcceptTimeout() && !txnId.kind().isSyncPoint();
+ boolean isExpired = time.now() - txnId.hlc() >= safeStore.preAcceptTimeout() && !txnId.isSyncPoint();
if (rejectBefore != null && !isExpired)
- isExpired = null == rejectBefore.foldl(keys, (rejectIfBefore, test) -> rejectIfBefore.compareTo(test) > 0 ? null : test, txnId, Objects::isNull);
+ isExpired = rejectBefore.rejects(txnId, keys);
if (isExpired)
return time.uniqueNow(txnId).asRejected();
- if (txnId.kind() == ExclusiveSyncPoint)
+ if (txnId.is(ExclusiveSyncPoint))
return txnId;
// TODO (expected): reject if any transaction exists with a higher timestamp OR a higher epoch
// this permits us to agree fast path decisions across epoch changes
// TODO (expected): we should (perhaps) separate conflicts for reads and writes
- Timestamp minNonConflicting = maxConflicts.get(keys);
- if (permitFastPath && txnId.compareTo(minNonConflicting) >= 0 && txnId.epoch() >= time.epoch())
+ Timestamp min = TxnId.max(txnId, maxConflicts.get(keys));
+ if (permitFastPath && txnId == min && txnId.epoch() >= time.epoch())
return txnId;
- return time.uniqueNow(minNonConflicting);
+ return time.uniqueNow(min);
}
public Timestamp maxConflict(Routables<?> keysOrRanges)
@@ -471,7 +471,7 @@ public abstract class CommandStore implements AgentExecutor
Timestamp before = Timestamp.minForEpoch(epoch);
FullRoute<?> route = node.computeRoute(id, ranges);
// TODO (required): we need to ensure anyone we receive a reply from proposes newer timestamps for anything we don't see
- CollectCalculatedDeps.withCalculatedDeps(node, id, route, route, ranges, before, (deps, fail) -> {
+ CollectCalculatedDeps.withCalculatedDeps(node, id, route, route, before, (deps, fail) -> {
if (fail != null)
{
fetchMajorityDeps(coordination, node, epoch, ranges);
@@ -599,13 +599,13 @@ public abstract class CommandStore implements AgentExecutor
safeStore.setSafeToRead(purgeAndInsert(safeToRead, TxnId.NONE, ranges));
}).beginAsResult();
- return new EpochReady(epoch, DONE, DONE, DONE, DONE);
+ return new EpochReady(epoch, done, done, done, done);
};
}
public final Ranges safeToReadAt(Timestamp at)
{
- return safeToRead.floorEntry(at).getValue();
+ return safeToRead.lowerEntry(at).getValue();
}
// TODO (desired): Commands.durability() can use this to upgrade to Majority without further info
@@ -635,7 +635,7 @@ public abstract class CommandStore implements AgentExecutor
if (rejectBefore == null)
return false;
- return null != rejectBefore.foldl(participants, (rejectIfBefore, test) -> rejectIfBefore.compareTo(test) > 0 ? null : test, txnId, Objects::isNull);
+ return rejectBefore.rejects(txnId, participants);
}
public final void removeRedundantDependencies(Unseekables<?> participants, WaitingOn.Update builder)
@@ -645,7 +645,7 @@ public abstract class CommandStore implements AgentExecutor
// TODO (required): make sure we have no races on HLC around SyncPoint else this resolution may not work (we need to know the micros equivalent timestamp of the snapshot)
class KeyState
{
- Int2ObjectHashMap<Keys> partiallyBootstrapping;
+ Int2ObjectHashMap<RoutingKeys> partiallyBootstrapping;
/**
* Are the participating ranges for the txn fully covered by bootstrapping ranges for this command store
@@ -657,8 +657,8 @@ public abstract class CommandStore implements AgentExecutor
if (partiallyBootstrapping == null)
partiallyBootstrapping = new Int2ObjectHashMap<>();
- Keys prev = partiallyBootstrapping.get(txnIdx);
- Keys remaining = prev;
+ RoutingKeys prev = partiallyBootstrapping.get(txnIdx);
+ RoutingKeys remaining = prev;
if (remaining == null) remaining = builder.directKeyDeps.participatingKeys(txnIdx);
else Invariants.checkState(!remaining.isEmpty());
remaining = remaining.without(range);
@@ -775,7 +775,7 @@ public abstract class CommandStore implements AgentExecutor
// TODO (required): consider race conditions when bootstrapping into an active command store, that may have seen a higher txnId than this?
// might benefit from maintaining a per-CommandStore largest TxnId register to ensure we allocate a higher TxnId for our ExclSync,
// or from using whatever summary records we have for the range, once we maintain them
- return redundantBefore.status(minimumDependencyId, executeAt, participantsOfWaitingTxn).compareTo(RedundantStatus.PARTIALLY_PRE_BOOTSTRAP_OR_STALE) >= 0;
+ return redundantBefore.status(minimumDependencyId, participantsOfWaitingTxn).compareTo(RedundantStatus.PARTIALLY_PRE_BOOTSTRAP_OR_STALE) >= 0;
}
final void markUnsafeToRead(Ranges ranges)
diff --git a/accord-core/src/main/java/accord/local/CommandStores.java b/accord-core/src/main/java/accord/local/CommandStores.java
index 545a7843..e5f684c3 100644
--- a/accord-core/src/main/java/accord/local/CommandStores.java
+++ b/accord-core/src/main/java/accord/local/CommandStores.java
@@ -39,7 +39,6 @@ import org.slf4j.LoggerFactory;
import accord.api.Agent;
import accord.api.ConfigurationService.EpochReady;
import accord.api.DataStore;
-import accord.api.Key;
import accord.api.LocalListeners;
import accord.api.ProgressLog;
import accord.api.RoutingKey;
@@ -137,9 +136,8 @@ public abstract class CommandStores
}
}
- // TODO (required): ensure all deterministic results across runs; updating RangesForEpoch should happen once per
- // execution batch and be logged
// TODO (expected): merge with RedundantBefore, and standardise executeRanges() to treat removing stale ranges the same as adding new epoch ranges
+ // We ONLY remove ranges to keep logic manageable; likely to only merge CommandStores into a new CommandStore via some kind of Bootstrap
public static class RangesForEpoch
{
public static class Snapshot
@@ -226,35 +224,24 @@ public abstract class CommandStores
return ranges[i];
}
- /**
- * Extend a previously computed set of Ranges that included {@code fromInclusive}
- * to include {@code toInclusive}
- */
- public @Nonnull Ranges extend(Ranges extend, Timestamp fromInclusive, Timestamp toInclusive)
- {
- return extend(extend, fromInclusive.epoch(), toInclusive.epoch());
- }
-
/**
* Extend a previously computed set of Ranges that included {@code fromInclusive}
* to include ranges up to {@code toInclusive}
*/
- public @Nonnull Ranges extend(Ranges extend, long fromInclusive, long toInclusive)
+ public @Nonnull Ranges extend(Ranges extend, long curFrom, long curTo, long extendFrom, long extendTo)
{
- if (fromInclusive == toInclusive)
- return extend;
+ if (extend.isEmpty()) // this captures the case where curTo < epochs[0]
+ return allBetween(extendFrom, extendTo);
- int startIndex = 1 + floorIndex(fromInclusive);
- int endIndex = 1 + floorIndex(toInclusive);
- for (int i = startIndex ; i < endIndex; ++i)
- extend = extend.with(ranges[i]); // want to always return extend for equal ranges, so same ranges are identity-equals
+ if (extendFrom >= curFrom)
+ return extend;
- return extend;
- }
+ int startCurIndex = floorIndex(curFrom);
+ int startExtendIndex = Math.max(0, floorIndex(extendFrom));
+ if (startCurIndex <= startExtendIndex)
+ return extend;
- public @Nonnull Ranges allBetween(EpochSupplier fromInclusive, EpochSupplier toInclusive)
- {
- return allBetween(fromInclusive.epoch(), toInclusive.epoch());
+ return ranges[startExtendIndex];
}
public @Nonnull Ranges allBetween(long fromInclusive, EpochSupplier toInclusive)
@@ -267,35 +254,35 @@ public abstract class CommandStores
if (fromInclusive > toInclusive)
throw new IndexOutOfBoundsException();
- if (fromInclusive == toInclusive)
- return allAt(fromInclusive);
+ int since = floorIndex(fromInclusive);
+ if (since >= 0) return ranges[since];
- return allInternal(Math.max(0, floorIndex(fromInclusive)), 1 + floorIndex(toInclusive));
+ int to = floorIndex(toInclusive);
+ if (to >= 0) return ranges[0];
+ return Ranges.EMPTY;
}
public @Nonnull Ranges all()
{
- return allInternal(0, ranges.length);
+ return ranges[0];
}
public @Nonnull Ranges allBefore(long toExclusive)
{
- return allInternal(0, ceilIndex(toExclusive));
- }
-
- public @Nonnull Ranges allAfter(long fromInclusive)
- {
- return allInternal(Math.max(0, floorIndex(fromInclusive)), ranges.length);
+ int to = ceilIndex(toExclusive);
+ return to <= 0 ? Ranges.EMPTY : ranges[0];
}
public @Nonnull Ranges allUntil(long toInclusive)
{
- return allInternal(0, 1 + floorIndex(toInclusive));
+ int to = floorIndex(toInclusive);
+ return to < 0 ? Ranges.EMPTY : ranges[0];
}
public @Nonnull Ranges allSince(long fromInclusive)
{
- return allInternal(Math.max(0, floorIndex(fromInclusive)), epochs.length);
+ int since = floorIndex(fromInclusive);
+ return ranges[Math.max(since, 0)];
}
private int floorIndex(long epoch)
@@ -312,22 +299,6 @@ public abstract class CommandStores
return i;
}
- private @Nonnull Ranges allInternal(int startIndex, int endIndex)
- {
- if (startIndex >= endIndex) return Ranges.EMPTY;
- Ranges result = ranges[startIndex];
- for (int i = startIndex + 1 ; i < endIndex; ++i)
- result = ranges[i].with(result);
-
- return result;
- }
-
- public @Nonnull Ranges applyRanges(Timestamp executeAt)
- {
- // Note: we COULD slice to only those ranges we haven't bootstrapped, but no harm redundantly writing
- return allAt(executeAt.epoch());
- }
-
public @Nonnull Ranges currentRanges()
{
return ranges[ranges.length - 1];
@@ -339,12 +310,17 @@ public abstract class CommandStores
.collect(Collectors.joining(", "));
}
- public long latestEpochWithNewParticipants(long sinceEpoch, Routables<?> keysOrRanges)
+ public long earliestLaterEpochThatFullyCovers(long sinceEpoch, Routables<?> keysOrRanges)
+ {
+ return Math.max(sinceEpoch, epochs[0]);
+ }
+
+ public long latestEarlierEpochThatFullyCovers(long beforeEpoch, Routables<?> keysOrRanges)
{
- int i = floorIndex(sinceEpoch);
- long latest = sinceEpoch;
- Ranges existing = i < 0 ? Ranges.EMPTY : ranges[i];
- while (++i < ranges.length)
+ int i = ceilIndex(beforeEpoch);
+ long latest = beforeEpoch;
+ Ranges existing = i >= ranges.length ? Ranges.EMPTY : ranges[i];
+ while (--i >= 0)
{
if (ranges[i].without(existing).intersects(keysOrRanges))
latest = epochs[i];
@@ -612,7 +588,7 @@ public abstract class CommandStores
ShardHolder[] shards = snapshot.shards;
for (ShardHolder shard : shards)
{
- // TODO (urgent, efficiency): range map for intersecting ranges (e.g. that to be introduced for range dependencies)
+ // TODO (required, efficiency): range map for intersecting ranges (e.g. that to be introduced for range dependencies)
Ranges shardRanges = shard.ranges().allBetween(minEpoch, maxEpoch);
if (!shardRanges.intersects(keys))
continue;
@@ -759,7 +735,7 @@ public abstract class CommandStores
}
@VisibleForTesting
- public CommandStore unsafeForKey(Key key)
+ public CommandStore unsafeForKey(RoutingKey key)
{
ShardHolder[] shards = current.shards;
for (ShardHolder shard : shards)
diff --git a/accord-core/src/main/java/accord/local/Commands.java b/accord-core/src/main/java/accord/local/Commands.java
index a20d81ba..3c453d3f 100644
--- a/accord-core/src/main/java/accord/local/Commands.java
+++ b/accord-core/src/main/java/accord/local/Commands.java
@@ -22,25 +22,27 @@ import java.util.Collection;
import java.util.Collections;
import java.util.function.BiPredicate;
import java.util.function.Consumer;
+import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import accord.api.Key;
import accord.api.Result;
-import accord.coordinate.Infer;
+import accord.api.RoutingKey;
import accord.local.Command.WaitingOn;
import accord.primitives.Ballot;
import accord.primitives.Deps;
-import accord.primitives.EpochSupplier;
import accord.primitives.FullRoute;
+import accord.primitives.Known;
import accord.primitives.PartialDeps;
import accord.primitives.PartialTxn;
import accord.primitives.Participants;
import accord.primitives.Ranges;
import accord.primitives.Route;
+import accord.primitives.SaveStatus;
import accord.primitives.Seekables;
+import accord.primitives.Status;
import accord.primitives.Timestamp;
import accord.primitives.Txn;
import accord.primitives.TxnId;
@@ -53,28 +55,31 @@ import accord.utils.async.AsyncChains;
import static accord.api.ProgressLog.BlockedUntil.CanApply;
import static accord.api.ProgressLog.BlockedUntil.HasDecidedExecuteAt;
import static accord.local.Cleanup.ERASE;
+import static accord.local.Cleanup.NO;
import static accord.local.Cleanup.shouldCleanup;
import static accord.local.Command.Truncated.erased;
+import static accord.local.Command.Truncated.erasedOrInvalidOrVestigial;
import static accord.local.Command.Truncated.invalidated;
import static accord.local.Command.Truncated.truncatedApply;
import static accord.local.Command.Truncated.truncatedApplyWithOutcome;
import static accord.local.KeyHistory.TIMESTAMPS;
import static accord.local.PreLoadContext.contextFor;
import static accord.local.RedundantStatus.PRE_BOOTSTRAP_OR_STALE;
-import static accord.local.SaveStatus.Applying;
-import static accord.local.SaveStatus.Erased;
-import static accord.local.SaveStatus.TruncatedApply;
-import static accord.local.Status.Applied;
-import static accord.local.Status.Committed;
-import static accord.local.Status.Durability;
-import static accord.local.Status.Invalidated;
-import static accord.local.Status.KnownExecuteAt.ExecuteAtKnown;
-import static accord.local.Status.KnownRoute.Full;
-import static accord.local.Status.NotDefined;
-import static accord.local.Status.PreApplied;
-import static accord.local.Status.PreCommitted;
-import static accord.local.Status.Stable;
-import static accord.local.Status.Truncated;
+import static accord.primitives.SaveStatus.Applying;
+import static accord.primitives.SaveStatus.Erased;
+import static accord.primitives.SaveStatus.TruncatedApply;
+import static accord.primitives.Status.Applied;
+import static accord.primitives.Status.Committed;
+import static accord.primitives.Status.Durability;
+import static accord.primitives.Status.Invalidated;
+import static accord.primitives.Known.KnownExecuteAt.ExecuteAtKnown;
+import static accord.primitives.Known.KnownRoute.Full;
+import static accord.primitives.Status.NotDefined;
+import static accord.primitives.Status.PreApplied;
+import static accord.primitives.Status.PreCommitted;
+import static accord.primitives.Status.Stable;
+import static accord.primitives.Status.Truncated;
+import static accord.primitives.Routables.Slice.Minimal;
import static accord.primitives.Route.isFullRoute;
import static accord.utils.Invariants.illegalState;
@@ -88,18 +93,18 @@ public class Commands
public enum AcceptOutcome { Success, Redundant, RejectedBallot, Truncated }
- public static AcceptOutcome preaccept(SafeCommandStore safeStore, SafeCommand safeCommand, TxnId txnId, long acceptEpoch, PartialTxn partialTxn, FullRoute<?> route)
+ public static AcceptOutcome preaccept(SafeCommandStore safeStore, SafeCommand safeCommand, StoreParticipants participants, TxnId txnId, long acceptEpoch, PartialTxn partialTxn, FullRoute<?> route)
{
- return preacceptOrRecover(safeStore, safeCommand, txnId, acceptEpoch, partialTxn, route, Ballot.ZERO);
+ return preacceptOrRecover(safeStore, safeCommand, participants, txnId, acceptEpoch, partialTxn, route, Ballot.ZERO);
}
- public static AcceptOutcome recover(SafeCommandStore safeStore, SafeCommand safeCommand, TxnId txnId, PartialTxn partialTxn, FullRoute<?> route, Ballot ballot)
+ public static AcceptOutcome recover(SafeCommandStore safeStore, SafeCommand safeCommand, StoreParticipants participants, TxnId txnId, PartialTxn partialTxn, FullRoute<?> route, Ballot ballot)
{
// for recovery we only ever propose either the original epoch or an Accept that we witness; otherwise we invalidate
- return preacceptOrRecover(safeStore, safeCommand, txnId, txnId.epoch(), partialTxn, route, ballot);
+ return preacceptOrRecover(safeStore, safeCommand, participants, txnId, txnId.epoch(), partialTxn, route, ballot);
}
- private static AcceptOutcome preacceptOrRecover(SafeCommandStore safeStore, SafeCommand safeCommand, TxnId txnId, long acceptEpoch, PartialTxn partialTxn, FullRoute<?> route, Ballot ballot)
+ private static AcceptOutcome preacceptOrRecover(SafeCommandStore safeStore, SafeCommand safeCommand, StoreParticipants participants, TxnId txnId, long acceptEpoch, PartialTxn partialTxn, FullRoute<?> route, Ballot ballot)
{
final Command command = safeCommand.current();
@@ -126,13 +131,9 @@ public class Commands
return ballot.equals(Ballot.ZERO) ? AcceptOutcome.Redundant : AcceptOutcome.Success;
}
- Ranges coordinateRanges = coordinateRanges(safeStore, txnId);
- Ranges acceptRanges = acceptRanges(safeStore, txnId, acceptEpoch, coordinateRanges);
- Invariants.checkState(!acceptRanges.isEmpty());
- Invariants.checkState(validate(SaveStatus.PreAccepted, command, acceptRanges, route, partialTxn, null, null));
+ Invariants.checkState(validate(SaveStatus.PreAccepted, command, participants, route, partialTxn, null, null));
- // FIXME: this should go into a consumer method
- CommonAttributes attrs = set(SaveStatus.PreAccepted, command, command, acceptRanges, ballot, route, partialTxn, null);
+ CommonAttributes attrs = set(safeStore, SaveStatus.PreAccepted, command, command, participants, ballot, partialTxn, null);
if (command.executeAt() == null)
{
// unlike in the Accord paper, we partition shards within a node, so that to ensure a total order we must either:
@@ -141,7 +142,7 @@ public class Commands
// if we are performing recovery (i.e. non-zero ballot), do not permit a fast path decision as we want to
// invalidate any transactions that were not completed by their initial coordinator
// TODO (desired): limit preaccept to keys we include, to avoid inflating unnecessary state
- Timestamp executeAt = safeStore.commandStore().preaccept(txnId, attrs.partialTxn().keys(), safeStore, ballot.equals(Ballot.ZERO));
+ Timestamp executeAt = safeStore.commandStore().preaccept(txnId, route, safeStore, ballot.equals(Ballot.ZERO));
safeCommand.preaccept(safeStore, attrs, executeAt, ballot);
}
else
@@ -175,24 +176,11 @@ public class Commands
return true;
}
- public static AcceptOutcome accept(SafeCommandStore safeStore, TxnId txnId, Ballot ballot, Route<?> route, Seekables<?, ?> keysOrRanges, Timestamp executeAt, PartialDeps partialDeps)
+ public static AcceptOutcome accept(SafeCommandStore safeStore, SafeCommand safeCommand, StoreParticipants participants, TxnId txnId, Ballot ballot, Route<?> route, Timestamp executeAt, PartialDeps partialDeps)
{
- SafeCommand safeCommand = safeStore.get(txnId, executeAt, route);
final Command command = safeCommand.current();
if (command.hasBeen(PreCommitted))
{
- if (command.is(Invalidated))
- {
- logger.trace("{}: skipping accept - command is invalidated", txnId);
- return AcceptOutcome.RejectedBallot;
- }
-
- if (command.is(Truncated))
- {
- logger.trace("{}: skipping accept - command is truncated", txnId);
- return AcceptOutcome.Truncated;
- }
-
logger.trace("{}: skipping accept - already committed ({})", txnId, command.status());
return AcceptOutcome.Redundant;
}
@@ -203,19 +191,14 @@ public class Commands
return AcceptOutcome.RejectedBallot;
}
- Ranges coordinateRanges = coordinateRanges(safeStore, txnId);
- Ranges acceptRanges = acceptRanges(safeStore, txnId, executeAt, coordinateRanges);
- Invariants.checkState(!acceptRanges.isEmpty());
-
- Invariants.checkState(validate(SaveStatus.Accepted, command, acceptRanges, route, null, partialDeps, null));
+ Invariants.checkState(validate(SaveStatus.Accepted, command, participants, route, null, partialDeps, null));
// TODO (desired, clarity/efficiency): we don't need to set the route here, and perhaps we don't even need to
// distributed partialDeps at all, since all we gain is not waiting for these transactions to commit during
// recovery. We probably don't want to directly persist a Route in any other circumstances, either, to ease persistence.
- CommonAttributes attrs = set(SaveStatus.Accepted, command, command, acceptRanges, ballot, route, null, partialDeps);
+ CommonAttributes attrs = set(safeStore, SaveStatus.Accepted, command, command, participants, ballot, null, partialDeps);
- keysOrRanges = keysOrRanges.slice(acceptRanges);
- safeCommand.accept(safeStore, keysOrRanges, attrs, executeAt, ballot);
+ safeCommand.accept(safeStore, route, attrs, executeAt, ballot);
safeStore.notifyListeners(safeCommand, command);
return AcceptOutcome.Success;
@@ -227,20 +210,8 @@ public class Commands
final Command command = safeCommand.current();
if (command.hasBeen(PreCommitted))
{
- if (command.is(Invalidated))
- {
- logger.trace("{}: skipping accept invalidated - already invalidated ({})", command.txnId(), command.status());
- return AcceptOutcome.Redundant;
- }
-
- if (command.is(Truncated))
- {
- logger.trace("{}: skipping accept invalidated - already truncated ({})", command.txnId(), command.status());
- return AcceptOutcome.Truncated;
- }
-
- logger.trace("{}: skipping accept invalidated - already committed ({})", command.txnId(), command.status());
- return AcceptOutcome.RejectedBallot;
+ logger.trace("{}: skipping accept invalidated - already committed or invalidated ({})", command.txnId(), command.status());
+ return AcceptOutcome.Redundant;
}
if (command.promised().compareTo(ballot) > 0)
@@ -260,7 +231,7 @@ public class Commands
// relies on mutual exclusion for each key
- public static CommitOutcome commit(SafeCommandStore safeStore, SafeCommand safeCommand, SaveStatus newStatus, Ballot ballot, TxnId txnId, Route<?> route, @Nullable PartialTxn partialTxn, Timestamp executeAt, PartialDeps partialDeps)
+ public static CommitOutcome commit(SafeCommandStore safeStore, SafeCommand safeCommand, StoreParticipants participants, SaveStatus newStatus, Ballot ballot, TxnId txnId, Route<?> route, @Nullable PartialTxn partialTxn, Timestamp executeAt, PartialDeps partialDeps)
{
final Command command = safeCommand.current();
SaveStatus curStatus = command.saveStatus();
@@ -292,13 +263,10 @@ public class Commands
}
}
- Ranges coordinateRanges = coordinateRanges(safeStore, txnId);
- Ranges acceptRanges = acceptRanges(safeStore, txnId, executeAt, coordinateRanges);
-
- if (!validate(newStatus, command, acceptRanges, route, partialTxn, partialDeps, null))
+ if (!validate(newStatus, command, participants, route, partialTxn, partialDeps, null))
return CommitOutcome.Insufficient;
- CommonAttributes attrs = set(newStatus, command, command, acceptRanges, ballot, route, partialTxn, partialDeps);
+ CommonAttributes attrs = set(safeStore, newStatus, command, command, participants, ballot, partialTxn, partialDeps);
logger.trace("{}: committed with executeAt: {}, deps: {}", txnId, executeAt, partialDeps);
final Command.Committed committed;
@@ -322,9 +290,9 @@ public class Commands
}
// relies on mutual exclusion for each key
- public static CommitOutcome precommit(SafeCommandStore safeStore, SafeCommand safeCommand, TxnId txnId, Timestamp executeAt, Route<?> route)
+ public static CommitOutcome precommit(SafeCommandStore safeStore, SafeCommand safeCommand, StoreParticipants participants, TxnId txnId, Timestamp executeAt)
{
- Invariants.checkState(Route.isFullRoute(route));
+ Invariants.checkState(Route.isFullRoute(participants.route));
final Command command = safeCommand.current();
if (command.hasBeen(PreCommitted))
{
@@ -343,30 +311,28 @@ public class Commands
}
}
- CommonAttributes attrs = command;
- if (command.route() == null || !Route.isFullRoute(command.route()))
- attrs = updateRoute(command, route);
-
+ CommonAttributes attrs = updateParticipants(command, participants);
safeCommand.precommit(safeStore, attrs, executeAt);
safeStore.notifyListeners(safeCommand, command);
logger.trace("{}: precommitted with executeAt: {}", txnId, executeAt);
return CommitOutcome.Success;
}
- public static void createBootstrapCompleteMarkerTransaction(SafeCommandStore safeStore, TxnId localSyncId, Seekables<?, ?> keys)
+ public static void createBootstrapCompleteMarkerTransaction(SafeCommandStore safeStore, TxnId localSyncId, Participants<?> keys)
{
SafeCommand safeCommand = safeStore.get(localSyncId);
final Command command = safeCommand.current();
Invariants.checkState(!command.hasBeen(Committed));
FullRoute<?> route = keys.toRoute(keys.get(0).someIntersectingRoutingKey(null));
- Ranges coordinateRanges = coordinateRanges(safeStore, localSyncId);
+ Ranges coordinateRanges = safeStore.coordinateRanges(localSyncId);
+ StoreParticipants participants = StoreParticipants.update(safeStore, route, localSyncId.epoch(), localSyncId, localSyncId.epoch());
// TODO (desired, consider): in the case of sync points, the coordinator is unlikely to be a home shard, do we mind this? should document at least
- Txn emptyTxn = safeStore.agent().emptySystemTxn(localSyncId.kind(), keys);
+ Txn emptyTxn = safeStore.agent().emptySystemTxn(localSyncId.kind(), localSyncId.domain());
PartialDeps none = Deps.NONE.intersecting(route);
PartialTxn partialTxn = emptyTxn.slice(coordinateRanges, true);
- Invariants.checkState(validate(SaveStatus.Stable, command, coordinateRanges, route, partialTxn, none, null));
- CommonAttributes newAttributes = set(SaveStatus.Stable, command, command, coordinateRanges, Ballot.ZERO, route, partialTxn, none);
+ Invariants.checkState(validate(SaveStatus.Stable, command, participants, route, partialTxn, none, null));
+ CommonAttributes newAttributes = set(safeStore, SaveStatus.Stable, command, command, participants, Ballot.ZERO, partialTxn, none);
safeCommand.stable(safeStore, newAttributes, Ballot.ZERO, localSyncId, WaitingOn.empty(emptyTxn.keys().domain()));
safeStore.notifyListeners(safeCommand, command);
}
@@ -380,11 +346,16 @@ public class Commands
// TODO (required): by creating synthetic TxnId in future epochs we may not be evictable
// but for ephemeral reads we want parallel eviction - or preferably no durability - anyway
+ // REVISIT
txnId = txnId.withEpoch(executeAtEpoch);
- Ranges coordinateRanges = coordinateRanges(safeStore, txnId);
- Invariants.checkState(validate(SaveStatus.Stable, command, coordinateRanges, route, partialTxn, partialDeps, null));
- CommonAttributes attrs = set(SaveStatus.Stable, command, command, coordinateRanges, Ballot.ZERO, route, partialTxn, partialDeps);
+ // BREAKING CHANGE NOTE: if in future we support a CommandStore adopting additional ranges (rather than only shedding them)
+ // then we need to revisit how we execute transactions that awaitsOnlyDeps, as they may need additional
+ // information to execute in the eventual execution epoch (that they didn't know they needed when they were made stable)
+
+ StoreParticipants participants = StoreParticipants.update(safeStore, route, txnId.epoch(), txnId, txnId.epoch());
+ Invariants.checkState(validate(SaveStatus.Stable, command, participants, route, partialTxn, partialDeps, null));
+ CommonAttributes attrs = set(safeStore, SaveStatus.Stable, command, command, participants, Ballot.ZERO, partialTxn, partialDeps);
safeCommand.stable(safeStore, attrs, Ballot.ZERO, txnId, initialiseWaitingOn(safeStore, txnId, attrs, txnId, route));
maybeExecute(safeStore, safeCommand, false, true);
}
@@ -397,7 +368,7 @@ public class Commands
return;
// NOTE: if this is ever made a non-empty txn this will introduce a potential bug where the txn is registered against CommandsForKeys
- Txn emptyTxn = safeStore.agent().emptySystemTxn(localSyncId.kind(), keys);
+ Txn emptyTxn = safeStore.agent().emptySystemTxn(localSyncId.kind(), localSyncId.domain());
safeCommand.preapplied(safeStore, command, command.executeAt(), command.waitingOn(), emptyTxn.execute(localSyncId, localSyncId, null), emptyTxn.result(localSyncId, localSyncId, null));
maybeExecute(safeStore, safeCommand, true, false);
}
@@ -431,7 +402,7 @@ public class Commands
public enum ApplyOutcome { Success, Redundant, Insufficient }
- public static ApplyOutcome apply(SafeCommandStore safeStore, SafeCommand safeCommand, TxnId txnId, Route<?> route, Timestamp executeAt, @Nullable PartialDeps partialDeps, @Nullable PartialTxn partialTxn, Writes writes, Result result)
+ public static ApplyOutcome apply(SafeCommandStore safeStore, SafeCommand safeCommand, StoreParticipants participants, TxnId txnId, Route<?> route, Timestamp executeAt, @Nullable PartialDeps partialDeps, @Nullable PartialTxn partialTxn, Writes writes, Result result)
{
Command command = safeCommand.current();
if (command.hasBeen(PreApplied) && executeAt.equals(command.executeAt()))
@@ -446,13 +417,10 @@ public class Commands
safeStore.agent().onInconsistentTimestamp(command, command.executeAt(), executeAt);
}
- Ranges coordinateRanges = coordinateRanges(safeStore, txnId);
- Ranges acceptRanges = acceptRanges(safeStore, txnId, executeAt, coordinateRanges);
-
- if (!validate(SaveStatus.PreApplied, command, acceptRanges, route, partialTxn, partialDeps, safeStore))
+ if (!validate(SaveStatus.PreApplied, command, participants, route, partialTxn, partialDeps, safeStore))
return ApplyOutcome.Insufficient; // TODO (expected, consider): this should probably be an assertion failure if !TrySet
- CommonAttributes attrs = set(SaveStatus.PreApplied, command, command, acceptRanges, null, route, partialTxn, partialDeps);
+ CommonAttributes attrs = set(safeStore, SaveStatus.PreApplied, command, command, participants, null, partialTxn, partialDeps);
WaitingOn waitingOn = !command.hasBeen(Stable) ? initialiseWaitingOn(safeStore, txnId, attrs, executeAt, attrs.route()) : command.asCommitted().waitingOn();
@@ -512,29 +480,6 @@ public class Commands
safeStore.notifyListeners(safeCommand, original);
}
- /**
- * The ranges for which we participate in the consensus decision of when a transaction executes
- */
- private static Ranges coordinateRanges(SafeCommandStore safeStore, TxnId txnId)
- {
- return safeStore.ranges().coordinates(txnId);
- }
-
- private static Ranges acceptRanges(SafeCommandStore safeStore, TxnId txnId, long untilEpoch, Ranges coordinateRanges)
- {
- return safeStore.ranges().extend(coordinateRanges, txnId.epoch(), untilEpoch);
- }
-
- private static Ranges acceptRanges(SafeCommandStore safeStore, TxnId txnId, Timestamp executeAt, Ranges coordinateRanges)
- {
- return safeStore.ranges().extend(coordinateRanges, txnId, executeAt);
- }
-
- private static Ranges executeRanges(SafeCommandStore safeStore, Timestamp executeAt)
- {
- return safeStore.ranges().allAt(executeAt.epoch());
- }
-
/**
* The ranges for which we participate in the execution of a transaction, excluding those ranges
* for transactions below a SyncPoint where we adopted the range, and that will be obtained from peers,
@@ -542,7 +487,7 @@ public class Commands
*/
private static Ranges applyRanges(SafeCommandStore safeStore, Timestamp executeAt)
{
- return safeStore.ranges().applyRanges(executeAt);
+ return safeStore.ranges().allAt(executeAt.epoch());
}
public static AsyncChain<Void> applyChain(SafeCommandStore safeStore, PreLoadContext context, TxnId txnId)
@@ -576,10 +521,9 @@ public class Commands
public static AsyncChain<Void> applyWrites(SafeCommandStore safeStore, PreLoadContext context, Command command)
{
CommandStore unsafeStore = safeStore.commandStore();
- Ranges executeRanges = executeRanges(safeStore, command.executeAt());
Command.Executed executed = command.asExecuted();
- boolean intersects = executed.writes().keys.intersects(executeRanges);
- if (intersects)
+ Participants<?> executes = executed.participants().executes(safeStore, command.txnId(), command.executeAt());
+ if (!executes.isEmpty())
return command.writes().apply(safeStore, applyRanges(safeStore, command.executeAt()), command.partialTxn())
.flatMap(unused -> unsafeStore.submit(context, ss -> {
postApply(ss, command.txnId());
@@ -589,12 +533,12 @@ public class Commands
return AsyncChains.success(null);
}
- public static void apply(SafeCommandStore safeStore, Command command)
+ private static void apply(SafeCommandStore safeStore, Command.Executed command, Participants<?> executes)
{
CommandStore unsafeStore = safeStore.commandStore();
TxnId txnId = command.txnId();
// TODO (expected): there is some coupling going on here - concept of TIMESTAMPS only needed if implementation tracks on apply
- PreLoadContext context = contextFor(command.txnId(), command.partialTxn().keys(), TIMESTAMPS);
+ PreLoadContext context = contextFor(command.txnId(), executes, TIMESTAMPS);
// this is sometimes called from a listener update, which will not have the keys in context
if (safeStore.canExecuteWith(context))
{
@@ -652,28 +596,25 @@ public class Commands
return true;
case PreApplied:
- Ranges executeRanges = executeRanges(safeStore, command.executeAt());
+ Participants<?> executes = command.participants().executes(safeStore, command.txnId(), command.executeAt());
Command.Executed executed = command.asExecuted();
- boolean intersects = executed.writes().keys.intersects(executeRanges);
- if (intersects)
+ if (executes.isEmpty())
{
- // TODO (now): we should set applying within apply to avoid applying multiple times
- safeCommand.applying(safeStore);
+ // TODO (desirable, performance): Mark no-ops in CFK so we can notify later transactions immediately
+ logger.trace("{}: applying no-op", command.txnId());
+ safeCommand.applied(safeStore);
safeStore.notifyListeners(safeCommand, command);
- logger.trace("{}: applying", command.txnId());
- apply(safeStore, executed);
- return true;
}
else
{
- // TODO (desirable, performance): This could be performed immediately upon Committed
- // but: if we later support transitive dependency elision this could be dangerous
- logger.trace("{}: applying no-op", command.txnId());
- safeCommand.applied(safeStore);
+ // TODO (now): we should set applying within apply to avoid applying multiple times
+ safeCommand.applying(safeStore);
safeStore.notifyListeners(safeCommand, command);
- return true;
+ logger.trace("{}: applying", command.txnId());
+ apply(safeStore, executed, executes);
}
+ return true;
default:
throw illegalState("Unexpected status: " + command.status());
}
@@ -681,12 +622,13 @@ public class Commands
protected static WaitingOn initialiseWaitingOn(SafeCommandStore safeStore, TxnId waitingId, CommonAttributes waiting, Timestamp waitingExecuteAt, Route<?> route)
{
- if (waitingId.kind().awaitsOnlyDeps())
+ if (waitingId.awaitsOnlyDeps())
waitingExecuteAt = Timestamp.maxForEpoch(waitingId.epoch());
Ranges ranges = safeStore.ranges().allAt(waitingExecuteAt);
- PartialDeps deps = waiting.partialDeps();
+ ranges = safeStore.commandStore().redundantBefore().removePreBootstrap(waitingId, ranges);
+ PartialDeps deps = waiting.partialDeps();
WaitingOn.Update update = WaitingOn.Update.initialise(waitingId, route, ranges, deps);
return updateWaitingOn(safeStore, waiting, waitingExecuteAt, update, route.participants()).build();
}
@@ -718,7 +660,7 @@ public class Commands
Command dependency = dependencySafeCommand.current();
Invariants.checkState(dependency.hasBeen(PreCommitted));
TxnId dependencyId = dependency.txnId();
- if (waitingId.kind().awaitsOnlyDeps() && dependency.known().executeAt == ExecuteAtKnown && dependency.executeAt().compareTo(waitingId) > 0)
+ if (waitingId.awaitsOnlyDeps() && dependency.known().executeAt == ExecuteAtKnown && dependency.executeAt().compareTo(waitingId) > 0)
waitingOn.updateExecuteAtLeast(dependency.executeAt());
if (dependency.hasBeen(Truncated))
@@ -729,8 +671,8 @@ public class Commands
case TruncatedApply:
case TruncatedApplyWithOutcome:
case TruncatedApplyWithDeps:
- Invariants.checkState(dependency.executeAt().compareTo(waitingExecuteAt) < 0 || waitingId.kind().awaitsOnlyDeps() || !dependency.txnId().kind().witnesses(waitingId));
- case ErasedOrInvalidOrVestigial:
+ Invariants.checkState(dependency.executeAt().compareTo(waitingExecuteAt) < 0 || waitingId.awaitsOnlyDeps() || !dependency.txnId().kind().witnesses(waitingId));
+ case ErasedOrVestigial:
case Erased:
logger.trace("{}: {} is truncated. Stop listening and removing from waiting on commit set.", waitingId, dependencyId);
break;
@@ -739,7 +681,7 @@ public class Commands
}
return waitingOn.setAppliedOrInvalidated(dependencyId);
}
- else if (dependency.executeAt().compareTo(waitingExecuteAt) > 0 && !waitingId.kind().awaitsOnlyDeps())
+ else if (dependency.executeAt().compareTo(waitingExecuteAt) > 0 && !waitingId.awaitsOnlyDeps())
{
// dependency cannot be a predecessor if it executes later
logger.trace("{}: {} executes after us. Removing from waiting on apply set.", waitingId, dependencyId);
@@ -752,7 +694,7 @@ public class Commands
}
else if (waitingOn.isWaitingOn(dependencyId))
{
- safeStore.registerListener(dependencySafeCommand, SaveStatus.Applied, Full, waitingId);
+ safeStore.registerListener(dependencySafeCommand, SaveStatus.Applied, waitingId);
return false;
}
else if (safeStore.isFullyPreBootstrapOrStale(dependency, waiting.partialDeps().participants(dependency.txnId())))
@@ -793,7 +735,7 @@ public class Commands
}
}
- public static void removeWaitingOnKeyAndMaybeExecute(SafeCommandStore safeStore, SafeCommand safeCommand, Key key)
+ public static void removeWaitingOnKeyAndMaybeExecute(SafeCommandStore safeStore, SafeCommand safeCommand, RoutingKey key)
{
if (safeCommand.current().hasBeen(Applied))
return;
@@ -818,23 +760,23 @@ public class Commands
setTruncatedApplyOrErasedVestigial(safeStore, safeCommand, null, null);
}
- public static void setTruncatedApplyOrErasedVestigial(SafeCommandStore safeStore, SafeCommand safeCommand, @Nullable Timestamp executeAt, Route<?> maybeFullRoute)
+ public static void setTruncatedApplyOrErasedVestigial(SafeCommandStore safeStore, SafeCommand safeCommand, StoreParticipants participants, @Nullable Timestamp executeAt)
{
+ // TODO (expected): tighten up and declare invariants and reasoning here
Command command = safeCommand.current();
if (command.saveStatus().compareTo(TruncatedApply) >= 0) return;
- FullRoute<?> route = Route.tryCastToFullRoute(maybeFullRoute);
- if (route == null) route = Route.tryCastToFullRoute(command.route());
+ participants = command.participants().supplement(participants);
if (executeAt == null) executeAt = command.executeAtIfKnown();
- if (route == null || executeAt == null)
+ if (participants.route == null || executeAt == null)
{
safeCommand.update(safeStore, Command.Truncated.erasedOrInvalidOrVestigial(command));
- if (route != null && !safeStore.ranges().allAt(command.txnId()).contains(route.homeKey()))
+ if (participants.route != null && !safeStore.coordinateRanges(command.txnId()).contains(participants.route.homeKey()))
safeStore.progressLog().clear(command.txnId());
}
else
{
- CommonAttributes attributes = command.mutable().route(route);
- if (!safeCommand.txnId().kind().awaitsOnlyDeps())
+ CommonAttributes attributes = command.mutable().updateParticipants(participants);
+ if (!safeCommand.txnId().awaitsOnlyDeps())
{
safeCommand.update(safeStore, truncatedApply(attributes, TruncatedApply, executeAt, null, null));
}
@@ -850,13 +792,13 @@ public class Commands
public static void setErased(SafeCommandStore safeStore, SafeCommand safeCommand)
{
- purge(safeStore, safeCommand, null, null, ERASE, true);
+ purge(safeStore, safeCommand, null, ERASE, true);
}
/**
* Purge all or part of the metadata for a Commmand
*/
- public static Command purge(SafeCommandStore safeStore, SafeCommand safeCommand, @Nullable Seekables<?, ?> keysOrRanges, @Nullable Unseekables<?> maybeFullRoute, Cleanup cleanup, boolean notifyListeners)
+ public static Command purge(SafeCommandStore safeStore, SafeCommand safeCommand, StoreParticipants participants, Cleanup cleanup, boolean notifyListeners)
{
final Command command = safeCommand.current();
@@ -866,10 +808,10 @@ public class Commands
// TODO (desired): consider if there are better invariants we can impose for undecided transactions, to verify they aren't later committed (should be detected already, but more is better)
// note that our invariant here is imperfectly applied to keep the code cleaner: we don't verify that the caller was safe to invoke if we don't already have a route in the command and we're only PreCommitted
Invariants.checkState(command.hasBeen(Applied) || !command.hasBeen(PreCommitted)
- || (command.route() == null || Infer.safeToCleanup(safeStore, command, command.route(), command.executeAt()) || safeStore.isFullyPreBootstrapOrStale(command, command.route().participants()))
+ || (command.route() == null || validateSafeToCleanup(safeStore, command, participants) || safeStore.isFullyPreBootstrapOrStale(command, command.route().participants()))
, "Command %s could not be truncated", command);
- Command result = purge(command, maybeFullRoute, cleanup);
+ Command result = purge(command, participants, cleanup);
safeCommand.update(safeStore, result);
safeStore.progressLog().clear(safeCommand.txnId());
if (notifyListeners)
@@ -877,7 +819,7 @@ public class Commands
return result;
}
- public static Command purge(Command command, @Nullable Unseekables<?> maybeFullRoute, Cleanup cleanup)
+ public static Command purge(Command command, StoreParticipants participants, Cleanup cleanup)
{
Command result;
switch (cleanup)
@@ -895,9 +837,14 @@ public class Commands
break;
case TRUNCATE:
- Invariants.checkState(command.saveStatus().compareTo(TruncatedApply) < 0, "%s should not have been in %s but is %s", command, TruncatedApply, command.saveStatus());
- Invariants.checkState(command.hasBeen(PreApplied),"%s should have been %s", command, PreApplied);
- result = truncatedApply(command, Route.tryCastToFullRoute(maybeFullRoute));
+ Invariants.checkState(command.saveStatus().compareTo(TruncatedApply) < 0);
+ Invariants.checkState(command.hasBeen(PreApplied));
+ result = truncatedApply(command, participants);
+ break;
+
+ case VESTIGIAL:
+ Invariants.checkState(command.saveStatus().compareTo(Erased) < 0);
+ result = erasedOrInvalidOrVestigial(command);
break;
case ERASE:
@@ -908,23 +855,41 @@ public class Commands
return result;
}
- public static boolean maybeCleanup(SafeCommandStore safeStore, SafeCommand safeCommand, Command command, EpochSupplier toEpoch, Unseekables<?> maybeFullRoute)
+ private static boolean validateSafeToCleanup(SafeCommandStore safeStore, Command command, StoreParticipants participants)
{
- Cleanup cleanup = shouldCleanup(safeStore, command, toEpoch, maybeFullRoute);
- if (command.saveStatus().compareTo(cleanup.appliesIfNot) >= 0)
- return false;
-
- purge(safeStore, safeCommand, null, maybeFullRoute, cleanup, true);
- return true;
+ TxnId txnId = command.txnId();
+ participants = command.participants().supplement(participants);
+ RedundantStatus status = safeStore.commandStore().redundantBefore().status(txnId, participants.owns());
+ switch (status)
+ {
+ default: throw new AssertionError("Unhandled RedundantStatus: " + status);
+ case LIVE:
+ case REDUNDANT_PRE_BOOTSTRAP_OR_STALE:
+ case PARTIALLY_PRE_BOOTSTRAP_OR_STALE:
+ return false;
+ case LOCALLY_REDUNDANT:
+ case SHARD_REDUNDANT:
+ Invariants.checkState(!command.hasBeen(PreCommitted));
+ case WAS_OWNED:
+ case PRE_BOOTSTRAP_OR_STALE:
+ case NOT_OWNED:
+ return true;
+ }
}
- // TODO (now): either ignore this message if we don't have a route, or else require FullRoute requiring route, or else require FullRoute
- public static Command setDurability(SafeCommandStore safeStore, SafeCommand safeCommand, Durability durability, @Nullable Route<?> route, @Nullable Timestamp executeAt)
+ public static boolean maybeCleanup(SafeCommandStore safeStore, SafeCommand safeCommand, Command command, @Nonnull StoreParticipants participants)
{
- return setDurability(safeStore, safeCommand, durability, route, executeAt, null);
+ Cleanup cleanup = shouldCleanup(safeStore, command, participants);
+ if (cleanup == NO)
+ return false;
+
+ Invariants.checkState(command.saveStatus().compareTo(cleanup.appliesIfNot) < 0);
+ purge(safeStore, safeCommand, participants, cleanup, true);
+ return true;
}
- public static Command setDurability(SafeCommandStore safeStore, SafeCommand safeCommand, Durability durability, @Nullable Route<?> route, @Nullable Timestamp executeAt, @Nullable EpochSupplier toEpoch)
+ // TODO (expected): either ignore this message if we don't have a route, or else require FullRoute requiring route, or else require FullRoute
+ public static Command setDurability(SafeCommandStore safeStore, SafeCommand safeCommand, StoreParticipants participants, Durability durability, @Nullable Timestamp executeAt)
{
final Command command = safeCommand.current();
if (command.is(Truncated))
@@ -933,13 +898,13 @@ public class Commands
if (command.durability().compareTo(durability) >= 0)
return command;
- CommonAttributes attrs = route == null ? command : updateRoute(command, route);
+ CommonAttributes attrs = updateParticipants(command, participants);
if (executeAt != null && command.status().hasBeen(Committed) && !command.executeAt().equals(executeAt))
safeStore.agent().onInconsistentTimestamp(command, command.asCommitted().executeAt(), executeAt);
attrs = attrs.mutable().durability(durability);
Command updated = safeCommand.updateAttributes(safeStore, attrs);
- if (maybeCleanup(safeStore, safeCommand, command, toEpoch, route))
+ if (maybeCleanup(safeStore, safeCommand, command, participants))
updated = safeCommand.current();
safeStore.notifyListeners(safeCommand, command);
@@ -972,7 +937,7 @@ public class Commands
depSafe = safeStore.ifInitialised(loadDepId);
if (depSafe == null)
{
- RedundantStatus redundantStatus = safeStore.commandStore().redundantBefore().status(waitingId, waiting.executeAt(), waiting.partialDeps().participants(loadDepId));
+ RedundantStatus redundantStatus = safeStore.commandStore().redundantBefore().status(waitingId, waiting.partialDeps().participants(loadDepId));
switch (redundantStatus)
{
default: throw new AssertionError("Unexpected redundant status: " + redundantStatus);
@@ -1033,7 +998,7 @@ public class Commands
switch (depStatus.known.executeAt)
{
case ExecuteAtKnown:
- if (waitingId.kind().awaitsOnlyDeps() || dep.executeAt().compareTo(waiting.executeAt()) < 0)
+ if (waitingId.awaitsOnlyDeps() || dep.executeAt().compareTo(waiting.executeAt()) < 0)
break;
case NoExecuteAt:
@@ -1047,7 +1012,8 @@ public class Commands
{
// TODO (desired): slightly costly to invert a large partialDeps collection
Participants<?> participants = waiting.partialDeps().participants(dep.txnId());
- RedundantStatus redundantStatus = safeStore.commandStore().redundantBefore().status(dep.txnId(), waiting.executeAt(), participants);
+ participants = waiting.participants().dependencyExecutesAtLeast(safeStore, participants, waitingId, waiting.executeAt());
+ RedundantStatus redundantStatus = safeStore.commandStore().redundantBefore().status(dep.txnId(), participants);
switch (redundantStatus)
{
default: throw new AssertionError("Unknown redundant status: " + redundantStatus);
@@ -1055,7 +1021,7 @@ public class Commands
case LIVE:
case PARTIALLY_PRE_BOOTSTRAP_OR_STALE:
if (logger.isTraceEnabled()) logger.trace("{} blocked on {} until ReadyToExclude", waitingId, dep.txnId());
- safeStore.registerListener(depSafe, HasDecidedExecuteAt.minSaveStatus, Full, waitingId);
+ safeStore.registerListener(depSafe, HasDecidedExecuteAt.minSaveStatus, waitingId);
safeStore.progressLog().waiting(HasDecidedExecuteAt, safeStore, depSafe, null, participants);
return;
@@ -1078,7 +1044,7 @@ public class Commands
case CleaningUp: throw illegalState("Invalid LocalExecution (should already be handled): " + depStatus.execution);
case NotReady:
- safeStore.registerListener(depSafe, HasDecidedExecuteAt.minSaveStatus, Full, waitingId);
+ safeStore.registerListener(depSafe, HasDecidedExecuteAt.minSaveStatus, waitingId);
safeStore.progressLog().waiting(HasDecidedExecuteAt, safeStore, depSafe, dep.route(), null);
return;
@@ -1088,13 +1054,13 @@ public class Commands
safeStore.progressLog().waiting(CanApply, safeStore, depSafe, dep.route(), null);
case Applying:
- safeStore.registerListener(depSafe, SaveStatus.Applied, Full, waitingId);
+ safeStore.registerListener(depSafe, SaveStatus.Applied, waitingId);
return;
case WaitingToApply:
if (dep.asCommitted().isWaitingOnDependency())
{
- safeStore.registerListener(depSafe, SaveStatus.Applied, Full, waitingId);
+ safeStore.registerListener(depSafe, SaveStatus.Applied, waitingId);
return;
}
else
@@ -1104,7 +1070,7 @@ public class Commands
{
default: throw illegalState("Invalid child status after attempt to execute: " + depSafe.current().saveStatus());
case Applying:
- safeStore.registerListener(depSafe, SaveStatus.Applied, Full, waitingId);
+ safeStore.registerListener(depSafe, SaveStatus.Applied, waitingId);
return;
case Applied:
@@ -1140,8 +1106,8 @@ public class Commands
WaitingOn.Update update = new WaitingOn.Update(current.waitingOn);
TxnId minWaitingOnTxnId = update.minWaitingOnTxnId();
- if (minWaitingOnTxnId != null && commandStore.hasLocallyRedundantDependencies(update.minWaitingOnTxnId(), current.executeAt(), current.route().participants()))
- safeStore.commandStore().removeRedundantDependencies(current.route().participants(), update);
+ if (minWaitingOnTxnId != null && commandStore.hasLocallyRedundantDependencies(update.minWaitingOnTxnId(), current.executeAt(), current.participants().owns))
+ safeStore.commandStore().removeRedundantDependencies(current.participants().owns, update);
// if we are a range transaction, being redundant for this transaction does not imply we are redundant for all transactions
if (redundant != null)
@@ -1149,19 +1115,6 @@ public class Commands
return safeCommand.updateWaitingOn(update);
}
- @SuppressWarnings({"unchecked", "rawtypes"})
- public static void ensureHomeIsMonitoring(SafeCommandStore safeStore, SafeCommand safeCommand, Command command, Unseekables<?> maybeRoute)
- {
- if (command.known().isDefinitionKnown() || command.hasBeen(Truncated))
- return;
-
- Route<?> route = Route.tryCastToRoute(maybeRoute);
- if (route == null)
- return;
-
- safeCommand.updateAttributes(safeStore, command.mutable().route(Route.merge((Route)route, command.route())));
- }
-
/**
* A key nominated to represent the "home" shard - only members of the home shard may be nominated to recover
* a transaction, to reduce the cluster-wide overhead of ensuring progress. A transaction that has only been
@@ -1172,32 +1125,44 @@ public class Commands
* For recovery purposes the "home shard" is as of txnId.epoch until Committed, and executeAt.epoch once Executed
*/
@SuppressWarnings({"unchecked", "rawtypes"})
+ public static CommonAttributes updateParticipants(Command command, StoreParticipants participants)
+ {
+ participants = command.participants().supplement(participants);
+ if (command.participants() == participants)
+ return command;
+
+ return command.mutable().updateParticipants(participants);
+ }
+
public static CommonAttributes updateRoute(Command command, Route<?> route)
{
- if (command.route() == null || !Route.isFullRoute(command.route()))
- return command.mutable().route(Route.merge((Route)route, command.route()));
+ StoreParticipants participants = command.participants().supplement(route);
+ if (command.participants() == participants)
+ return command;
- return command;
+ return command.mutable().updateParticipants(participants);
}
- public static Command updateRouteOrParticipants(SafeCommandStore safeStore, SafeCommand safeCommand, Unseekables<?> participants)
+ public static Command updateParticipants(SafeCommandStore safeStore, SafeCommand safeCommand, StoreParticipants participants)
{
Command current = safeCommand.current();
if (current.saveStatus().compareTo(Erased) >= 0)
return current;
- CommonAttributes updated = current;
- if (Route.isRoute(participants))
- {
- updated = updateRoute(current, Route.castToRoute(participants));
- }
- else
- {
- Route<?> existingRoute = current.route();
- if (existingRoute != null && !existingRoute.containsAll(participants))
- updated = current.mutable().route(existingRoute.with((Unseekables) participants));
- }
+ CommonAttributes updated = updateParticipants(current, participants);
+ if (current == updated)
+ return current;
+
+ return safeCommand.updateAttributes(safeStore, updated);
+ }
+
+ public static Command updateRoute(SafeCommandStore safeStore, SafeCommand safeCommand, Route<?> route)
+ {
+ Command current = safeCommand.current();
+ if (current.hasBeen(Truncated))
+ return current;
+ CommonAttributes updated = updateRoute(current, route);
if (current == updated)
return current;
@@ -1205,21 +1170,23 @@ public class Commands
}
@SuppressWarnings({"unchecked", "rawtypes"})
- private static CommonAttributes set(SaveStatus newStatus, Command cur, CommonAttributes upd,
- Ranges acceptRanges, @Nullable Ballot newAcceptedOrCommittedBallot,
- Route<?> route, @Nullable PartialTxn partialTxn, @Nullable PartialDeps partialDeps)
+ private static CommonAttributes set(SafeCommandStore safeStore, SaveStatus newStatus, Command cur, CommonAttributes upd,
+ StoreParticipants participants, @Nullable Ballot newAcceptedOrCommittedBallot,
+ @Nullable PartialTxn partialTxn, @Nullable PartialDeps partialDeps)
{
- Status.Known haveKnown = cur.saveStatus().known;
- Status.Known expectKnown = newStatus.known;
+ Known haveKnown = cur.saveStatus().known;
+ Known expectKnown = newStatus.known;
- if (Route.isFullRoute(upd.route())) route = upd.route();
- else upd = upd.mutable().route(route = Route.merge(upd.route(), (Route)route));
+ participants = participants.supplement(cur.participants());
+ if (cur.participants() != participants)
+ {
+ upd = upd.mutable().setParticipants(participants);
+ Invariants.checkState(participants.touches == participants.owns || safeStore.ranges().all().containsAll(participants.touches));
+ }
- Route<?> scope;
if (partialTxn != null && expectKnown.definition.isKnown())
{
- scope = route.slice(acceptRanges);
- partialTxn = partialTxn.intersecting(scope, true);
+ partialTxn = partialTxn.intersecting(participants.owns, true);
if (haveKnown.definition.isKnown())
upd = upd.mutable().partialTxn(upd.partialTxn().with(partialTxn));
else
@@ -1227,54 +1194,47 @@ public class Commands
}
if (partialDeps != null && expectKnown.deps.hasProposedOrDecidedDeps() && (haveKnown.deps != expectKnown.deps || (newAcceptedOrCommittedBallot != null && !newAcceptedOrCommittedBallot.equals(cur.acceptedOrCommitted()))))
- {
- scope = route.slice(acceptRanges);
- upd = upd.mutable().partialDeps(partialDeps.intersecting(scope));
- }
+ upd = upd.mutable().partialDeps(partialDeps.intersecting(participants.touches));
return upd;
}
- private static boolean validate(SaveStatus newStatus, Command cur,
- Ranges acceptRanges,
+ private static boolean validate(SaveStatus newStatus, Command cur, StoreParticipants participants,
Route<?> addRoute,
@Nullable PartialTxn addPartialTxn,
@Nullable PartialDeps newPartialDeps,
@Nullable SafeCommandStore permitStaleMissing)
{
- Status.Known haveKnown = cur.saveStatus().known;
- Status.Known expectKnown = newStatus.known;
+ Known haveKnown = cur.saveStatus().known;
+ Known expectKnown = newStatus.known;
- FullRoute<?> fullRoute;
- if (isFullRoute(cur.route())) fullRoute = Route.castToFullRoute(cur.route());
- else if (isFullRoute(addRoute)) fullRoute = Route.castToFullRoute(addRoute);
- else return expectKnown.route != Full; // we cannot validate further without a full route, so in cases we don't expect a full route we just assume we have received the right information
+ Invariants.checkState(addRoute == participants.route);
+ if (expectKnown.route == Full && !isFullRoute(cur.route()) && !isFullRoute(addRoute))
+ return false;
- Participants<?> acceptScope = null;
if (expectKnown.definition.isKnown())
{
- acceptScope = fullRoute.slice(acceptRanges);
- if (haveKnown.definition.isKnown())
+ if (cur.txnId().isSystemTxn())
+ {
+ if (cur.partialTxn() == null && addPartialTxn == null)
+ return false;
+ }
+ else if (haveKnown.definition.isKnown())
{
// TODO (desired): avoid converting to participants before subtracting
- Participants<?> extraScope = acceptScope.without(cur.partialTxn().keys().toParticipants());
+ Participants<?> extraScope = participants.owns.without(cur.partialTxn().keys().toParticipants());
if (!containsAll(addPartialTxn, PartialTxn::covers, extraScope, permitStaleMissing))
return false;
}
else
{
- if (!containsAll(addPartialTxn, PartialTxn::covers, acceptScope, permitStaleMissing))
+ if (!containsAll(addPartialTxn, PartialTxn::covers, participants.owns, permitStaleMissing))
return false;
}
}
if (haveKnown.deps != expectKnown.deps && expectKnown.deps.hasProposedOrDecidedDeps())
- {
- if (acceptScope == null)
- acceptScope = fullRoute.slice(acceptRanges);
-
- return containsAll(newPartialDeps, PartialDeps::covers, acceptScope, permitStaleMissing);
- }
+ return containsAll(newPartialDeps, PartialDeps::covers, participants.touches, permitStaleMissing);
return true;
}
diff --git a/accord-core/src/main/java/accord/local/CommonAttributes.java b/accord-core/src/main/java/accord/local/CommonAttributes.java
index 4612266f..e45664c2 100644
--- a/accord-core/src/main/java/accord/local/CommonAttributes.java
+++ b/accord-core/src/main/java/accord/local/CommonAttributes.java
@@ -18,13 +18,10 @@
package accord.local;
-import javax.annotation.Nullable;
-
import accord.primitives.PartialDeps;
import accord.primitives.PartialTxn;
-import accord.primitives.Participants;
import accord.primitives.Route;
-import accord.primitives.Seekables;
+import accord.primitives.Status;
import accord.primitives.TxnId;
import accord.utils.Invariants;
@@ -32,11 +29,9 @@ public interface CommonAttributes
{
TxnId txnId();
Status.Durability durability();
- Route<?> route();
- // if we don't know a route, we may know some participants; if we know a route, this must return the same object as route()
- Participants<?> participants();
+ StoreParticipants participants();
+ default Route<?> route() { return participants().route(); }
PartialTxn partialTxn();
- @Nullable Seekables<?, ?> additionalKeysOrRanges();
PartialDeps partialDeps();
default Mutable mutable()
@@ -48,11 +43,9 @@ public interface CommonAttributes
{
private TxnId txnId;
private Status.Durability durability;
- private Route<?> route;
- private Participants<?> participants; // if route != null, route == participants
+ private StoreParticipants participants;
private PartialTxn partialTxn;
private PartialDeps partialDeps;
- private Seekables<?, ?> additionalKeysOrRanges;
public Mutable(TxnId txnId)
{
@@ -63,12 +56,9 @@ public interface CommonAttributes
{
this.txnId = attributes.txnId();
this.durability = attributes.durability();
- this.route = attributes.route();
- this.participants = attributes.participants();
- Invariants.checkState(participants == route || route == null);
+ this.participants = Invariants.nonNull(attributes.participants());
this.partialTxn = attributes.partialTxn();
this.partialDeps = attributes.partialDeps();
- this.additionalKeysOrRanges = attributes.additionalKeysOrRanges();
}
@Override
@@ -102,27 +92,19 @@ public interface CommonAttributes
}
@Override
- public Route<?> route()
+ public StoreParticipants participants()
{
- return route;
+ return participants;
}
- public Mutable route(Route<?> route)
+ public Mutable updateParticipants(StoreParticipants participants)
{
- this.route = route;
- this.participants = route;
+ this.participants = this.participants == null ? participants : participants.supplement(this.participants);
return this;
}
- @Override
- public Participants<?> participants()
+ public Mutable setParticipants(StoreParticipants participants)
{
- return participants;
- }
-
- public Mutable participants(Participants<?> participants)
- {
- Invariants.checkState(route == null);
this.participants = participants;
return this;
}
@@ -135,27 +117,7 @@ public interface CommonAttributes
public Mutable partialTxn(PartialTxn partialTxn)
{
- PartialTxn prev = this.partialTxn;
this.partialTxn = partialTxn;
- if (prev != null || additionalKeysOrRanges != null)
- {
- Seekables<?, ?> removed = prev == null ? null : ((Seekables) prev.keys()).without(partialTxn.keys());
- if (prev != null && !removed.isEmpty())
- {
- if (additionalKeysOrRanges == null) additionalKeysOrRanges = removed;
- else additionalKeysOrRanges = ((Seekables)additionalKeysOrRanges).without(partialTxn.keys()).with(removed);
- }
- else if (additionalKeysOrRanges != null)
- {
- additionalKeysOrRanges = ((Seekables)additionalKeysOrRanges).without(partialTxn.keys());
- }
- }
- return this;
- }
-
- public Mutable removePartialTxn()
- {
- this.partialTxn = null;
return this;
}
@@ -170,16 +132,5 @@ public interface CommonAttributes
this.partialDeps = partialDeps;
return this;
}
-
- public Seekables<?, ?> additionalKeysOrRanges()
- {
- return additionalKeysOrRanges;
- }
-
- public Mutable additionalKeysOrRanges(Seekables<?, ?> additionalKeysOrRanges)
- {
- this.additionalKeysOrRanges = additionalKeysOrRanges;
- return this;
- }
}
}
diff --git a/accord-core/src/main/java/accord/local/DurableBefore.java b/accord-core/src/main/java/accord/local/DurableBefore.java
index aab40559..4d437ab5 100644
--- a/accord-core/src/main/java/accord/local/DurableBefore.java
+++ b/accord-core/src/main/java/accord/local/DurableBefore.java
@@ -23,7 +23,7 @@ import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import accord.api.RoutingKey;
-import accord.local.Status.Durability;
+import accord.primitives.Status.Durability;
import accord.primitives.AbstractRanges;
import accord.primitives.Participants;
import accord.primitives.TxnId;
@@ -32,9 +32,9 @@ import accord.utils.Invariants;
import accord.utils.ReducingIntervalMap;
import accord.utils.ReducingRangeMap;
-import static accord.local.Status.Durability.MajorityOrInvalidated;
-import static accord.local.Status.Durability.NotDurable;
-import static accord.local.Status.Durability.UniversalOrInvalidated;
+import static accord.primitives.Status.Durability.MajorityOrInvalidated;
+import static accord.primitives.Status.Durability.NotDurable;
+import static accord.primitives.Status.Durability.UniversalOrInvalidated;
public class DurableBefore extends ReducingRangeMap<DurableBefore.Entry>
{
diff --git a/accord-core/src/main/java/accord/local/MaxConflicts.java b/accord-core/src/main/java/accord/local/MaxConflicts.java
index c31a1bfa..5ff321a8 100644
--- a/accord-core/src/main/java/accord/local/MaxConflicts.java
+++ b/accord-core/src/main/java/accord/local/MaxConflicts.java
@@ -23,8 +23,8 @@ import javax.annotation.Nonnull;
import accord.api.RoutingKey;
import accord.primitives.AbstractRanges;
import accord.primitives.Routables;
-import accord.primitives.Seekables;
import accord.primitives.Timestamp;
+import accord.primitives.Unseekables;
import accord.utils.ReducingIntervalMap;
import accord.utils.ReducingRangeMap;
@@ -43,7 +43,7 @@ public class MaxConflicts extends ReducingRangeMap<Timestamp>
super(inclusiveEnds, starts, values);
}
- public Timestamp get(Seekables<?, ?> keysOrRanges)
+ public Timestamp get(Unseekables<?> keysOrRanges)
{
return foldl(keysOrRanges, Timestamp::max, Timestamp.NONE);
}
@@ -53,7 +53,7 @@ public class MaxConflicts extends ReducingRangeMap<Timestamp>
return foldl(keysOrRanges, Timestamp::max, Timestamp.NONE);
}
- MaxConflicts update(Seekables<?, ?> keysOrRanges, Timestamp maxConflict)
+ MaxConflicts update(Unseekables<?> keysOrRanges, Timestamp maxConflict)
{
return merge(this, create(keysOrRanges, maxConflict));
}
@@ -66,7 +66,7 @@ public class MaxConflicts extends ReducingRangeMap<Timestamp>
return create(ranges, maxConflict, MaxConflicts.Builder::new);
}
- public static MaxConflicts create(Seekables<?, ?> keysOrRanges, @Nonnull Timestamp maxConflict)
+ public static MaxConflicts create(Unseekables<?> keysOrRanges, @Nonnull Timestamp maxConflict)
{
if (keysOrRanges.isEmpty())
return MaxConflicts.EMPTY;
diff --git a/accord-core/src/main/java/accord/local/Node.java b/accord-core/src/main/java/accord/local/Node.java
index f008edf1..5f61d01c 100644
--- a/accord-core/src/main/java/accord/local/Node.java
+++ b/accord-core/src/main/java/accord/local/Node.java
@@ -45,7 +45,6 @@ import accord.api.Agent;
import accord.api.ConfigurationService;
import accord.api.ConfigurationService.EpochReady;
import accord.api.DataStore;
-import accord.api.Key;
import accord.api.LocalConfig;
import accord.api.LocalListeners;
import accord.api.MessageSink;
@@ -77,7 +76,6 @@ import accord.primitives.Ranges;
import accord.primitives.Routable.Domain;
import accord.primitives.Routables;
import accord.primitives.Route;
-import accord.primitives.Seekables;
import accord.primitives.Timestamp;
import accord.primitives.Txn;
import accord.primitives.TxnId;
@@ -89,6 +87,8 @@ import accord.utils.DeterministicSet;
import accord.utils.Invariants;
import accord.utils.MapReduceConsume;
import accord.utils.RandomSource;
+import accord.utils.SortedList;
+import accord.utils.SortedListMap;
import accord.utils.async.AsyncChain;
import accord.utils.async.AsyncExecutor;
import accord.utils.async.AsyncResult;
@@ -514,6 +514,10 @@ public class Node implements ConfigurationService.Listener, NodeTimeService
if (collection instanceof List) return;
if (collection instanceof NavigableSet
|| collection instanceof LinkedHashSet
+ || collection instanceof SortedList
+ || collection instanceof SortedListMap
+ || collection instanceof SortedListMap.SetView
+ || collection instanceof SortedListMap.CollectionView
|| "java.util.LinkedHashMap.LinkedKeySet".equals(collection.getClass().getCanonicalName())
|| collection instanceof DeterministicSet) return;
throw new IllegalArgumentException("Attempted to use a collection that is unsafe for iteration: " + collection.getClass());
@@ -603,19 +607,19 @@ public class Node implements ConfigurationService.Listener, NodeTimeService
}
}
- public FullRoute<?> computeRoute(TxnId txnId, Seekables<?, ?> keysOrRanges)
+ public FullRoute<?> computeRoute(TxnId txnId, Routables<?> keysOrRanges)
{
return computeRoute(txnId.epoch(), keysOrRanges);
}
- public FullRoute<?> computeRoute(long epoch, Seekables<?, ?> keysOrRanges)
+ public FullRoute<?> computeRoute(long epoch, Routables<?> keysOrRanges)
{
Invariants.checkArgument(!keysOrRanges.isEmpty(), "Attempted to compute a route from empty keys or ranges");
RoutingKey homeKey = selectHomeKey(epoch, keysOrRanges);
return keysOrRanges.toRoute(homeKey);
}
- private RoutingKey selectHomeKey(long epoch, Seekables<?, ?> keysOrRanges)
+ private RoutingKey selectHomeKey(long epoch, Routables<?> keysOrRanges)
{
Ranges owned = topology().localForEpoch(epoch).ranges();
int i = (int)keysOrRanges.findNextIntersection(0, owned, 0);
@@ -731,7 +735,7 @@ public class Node implements ConfigurationService.Listener, NodeTimeService
}
@VisibleForTesting
- public CommandStore unsafeForKey(Key key)
+ public CommandStore unsafeForKey(RoutingKey key)
{
return commandStores.unsafeForKey(key);
}
diff --git a/accord-core/src/main/java/accord/local/PreLoadContext.java b/accord-core/src/main/java/accord/local/PreLoadContext.java
index d5153a79..adbcf774 100644
--- a/accord-core/src/main/java/accord/local/PreLoadContext.java
+++ b/accord-core/src/main/java/accord/local/PreLoadContext.java
@@ -18,13 +18,13 @@
package accord.local;
-import accord.api.Key;
+import accord.api.RoutingKey;
import accord.local.cfk.CommandsForKey;
-import accord.primitives.Keys;
-import accord.primitives.Seekables;
+import accord.primitives.RoutingKeys;
import accord.primitives.TxnId;
import com.google.common.collect.Iterators;
+import accord.primitives.Unseekables;
import accord.utils.Invariants;
import net.nicoulaj.compilecommand.annotations.Inline;
@@ -64,7 +64,7 @@ public interface PreLoadContext
Collection<TxnId> additional = additionalTxnIds();
if (primaryTxnId == null) return additional;
if (additional.isEmpty()) return Collections.singleton(primaryTxnId);
- return new AbstractCollection<TxnId>()
+ return new AbstractCollection<>()
{
@Override
public Iterator<TxnId> iterator()
@@ -97,7 +97,7 @@ public interface PreLoadContext
* potentially large scans, and for register we do not need to load into memory, we can perform a blind write.
*/
// TODO (required): specify epochs for which we should load, so we can narrow to owned keys
- default Seekables<?, ?> keys() { return Keys.EMPTY; }
+ default Unseekables<?> keys() { return RoutingKeys.EMPTY; }
default KeyHistory keyHistory() { return KeyHistory.NONE; }
@@ -135,7 +135,7 @@ public interface PreLoadContext
}
}
- static PreLoadContext contextFor(TxnId primary, Collection<TxnId> additional, Seekables<?, ?> keys, KeyHistory keyHistory)
+ static PreLoadContext contextFor(TxnId primary, Collection<TxnId> additional, Unseekables<?> keys, KeyHistory keyHistory)
{
Invariants.checkState(!additional.contains(primary));
return new Standard(primary, additional, keys, keyHistory);
@@ -145,10 +145,10 @@ public interface PreLoadContext
{
private final TxnId primary;
private final Collection<TxnId> additional;
- private final Seekables<?, ?> keys;
+ private final Unseekables<?> keys;
private final KeyHistory keyHistory;
- public Standard(TxnId primary, Collection<TxnId> additional, Seekables<?, ?> keys, KeyHistory keyHistory)
+ public Standard(TxnId primary, Collection<TxnId> additional, Unseekables<?> keys, KeyHistory keyHistory)
{
this.primary = primary;
this.additional = additional;
@@ -180,10 +180,7 @@ public interface PreLoadContext
}
@Override
- public Seekables<?, ?> keys()
- {
- return keys;
- }
+ public Unseekables<?> keys() { return keys; }
@Override
public KeyHistory keyHistory()
@@ -192,57 +189,57 @@ public interface PreLoadContext
}
}
- static PreLoadContext contextFor(TxnId primary, Collection<TxnId> additional, Seekables<?, ?> keys)
+ static PreLoadContext contextFor(TxnId primary, Collection<TxnId> additional, Unseekables<?> keys)
{
return contextFor(primary, additional, keys, KeyHistory.NONE);
}
static PreLoadContext contextFor(TxnId primary, TxnId additional)
{
- return contextFor(primary, Collections.singletonList(additional), Keys.EMPTY);
+ return contextFor(primary, Collections.singletonList(additional), RoutingKeys.EMPTY);
}
- static PreLoadContext contextFor(TxnId primary, TxnId additional, Seekables<?, ?> keys)
+ static PreLoadContext contextFor(TxnId primary, TxnId additional, Unseekables<?> keys)
{
return contextFor(primary, Collections.singletonList(additional), keys);
}
- static PreLoadContext contextFor(TxnId txnId, Seekables<?, ?> keysOrRanges, KeyHistory keyHistory)
+ static PreLoadContext contextFor(TxnId txnId, Unseekables<?> keysOrRanges, KeyHistory keyHistory)
{
return contextFor(txnId, Collections.emptyList(), keysOrRanges, keyHistory);
}
- static PreLoadContext contextFor(TxnId txnId, Seekables<?, ?> keysOrRanges)
+ static PreLoadContext contextFor(TxnId txnId, Unseekables<?> keysOrRanges)
{
return contextFor(txnId, keysOrRanges, KeyHistory.NONE);
}
static PreLoadContext contextFor(TxnId txnId)
{
- return contextFor(txnId, Keys.EMPTY);
+ return contextFor(txnId, RoutingKeys.EMPTY);
}
static PreLoadContext contextFor(TxnId primary, Collection<TxnId> additional)
{
- return contextFor(primary, additional, Keys.EMPTY);
+ return contextFor(primary, additional, RoutingKeys.EMPTY);
}
- static PreLoadContext contextFor(Key key, KeyHistory keyHistory)
+ static PreLoadContext contextFor(RoutingKey key, KeyHistory keyHistory)
{
- return contextFor(null, Collections.emptyList(), Keys.of(key), keyHistory);
+ return contextFor(null, Collections.emptyList(), RoutingKeys.of(key), keyHistory);
}
- static PreLoadContext contextFor(Key key)
+ static PreLoadContext contextFor(RoutingKey key)
{
return contextFor(key, KeyHistory.NONE);
}
- static PreLoadContext contextFor(Collection<TxnId> ids, Seekables<?, ?> keys)
+ static PreLoadContext contextFor(Collection<TxnId> ids, Unseekables<?> keys)
{
return contextFor(null, ids, keys);
}
- static PreLoadContext contextFor(Seekables<?, ?> keys)
+ static PreLoadContext contextFor(Unseekables<?> keys)
{
return contextFor(null, Collections.emptyList(), keys);
}
@@ -252,5 +249,5 @@ public interface PreLoadContext
return EMPTY_PRELOADCONTEXT;
}
- PreLoadContext EMPTY_PRELOADCONTEXT = contextFor(null, Collections.emptyList(), Keys.EMPTY);
+ PreLoadContext EMPTY_PRELOADCONTEXT = contextFor(null, Collections.emptyList(), RoutingKeys.EMPTY);
}
diff --git a/accord-core/src/main/java/accord/local/RedundantBefore.java b/accord-core/src/main/java/accord/local/RedundantBefore.java
index a39cdf98..8da8a050 100644
--- a/accord-core/src/main/java/accord/local/RedundantBefore.java
+++ b/accord-core/src/main/java/accord/local/RedundantBefore.java
@@ -23,15 +23,17 @@ import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import accord.api.RoutingKey;
+import accord.api.VisibleForImplementation;
+import accord.primitives.AbstractRanges;
import accord.primitives.Deps;
import accord.primitives.EpochSupplier;
import accord.primitives.Participants;
import accord.primitives.Range;
import accord.primitives.Ranges;
-import accord.primitives.RoutableKey;
-import accord.primitives.Seekables;
+import accord.primitives.Routables;
import accord.primitives.Timestamp;
import accord.primitives.TxnId;
+import accord.primitives.Unseekables;
import accord.utils.Invariants;
import accord.utils.ReducingIntervalMap;
import accord.utils.ReducingRangeMap;
@@ -44,12 +46,11 @@ import static accord.local.RedundantStatus.LOCALLY_REDUNDANT;
import static accord.local.RedundantStatus.NOT_OWNED;
import static accord.local.RedundantStatus.PRE_BOOTSTRAP_OR_STALE;
import static accord.local.RedundantStatus.SHARD_REDUNDANT;
+import static accord.local.RedundantStatus.WAS_OWNED;
import static accord.utils.Invariants.illegalState;
public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
{
- public static final EpochSupplier NO_UPPER_BOUND = () -> Long.MAX_VALUE;
-
public static class SerializerSupport
{
public static RedundantBefore create(boolean inclusiveEnds, RoutingKey[] ends, Entry[] values)
@@ -60,12 +61,13 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
public enum PreBootstrapOrStale { NOT_OWNED, FULLY, PARTIALLY, POST_BOOTSTRAP }
+ // TODO (expected): rationalise the various bounds we maintain; make merge idempotent and apply any filtering by superseding bounds on access
public static class Entry
{
// TODO (desired): we don't need to maintain this now, can migrate to ReducingRangeMap.foldWithBounds
public final Range range;
// start inclusive, end exclusive
- public final long startEpoch, endEpoch;
+ public final long startOwnershipEpoch, endOwnershipEpoch;
/**
* Represents the maximum TxnId we know to have fully executed until locally for the range in question.
@@ -90,8 +92,8 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
public final @Nonnull TxnId locallyDecidedAndAppliedOrInvalidatedBefore;
/**
- * Represents the maximum TxnId we know to have fully executed until across all healthy replicas for the range in question.
- * Unless we are stale or pre-bootstrap, in which case no such guarantees can be made.
+ * Represents the maximum TxnId we know to have fully executed until across all healthy non-bootstrapping replicas
+ * for the range in question, including ourselves.
*/
public final @Nonnull TxnId shardAppliedOrInvalidatedBefore;
@@ -101,6 +103,12 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
*/
public final @Nonnull TxnId gcBefore;
+ /**
+ * Represents the maximum TxnId we know to have fully executed until across all healthy non-bootstrapping replicas
+ * for the range in question.
+ */
+ public final @Nonnull TxnId shardOnlyAppliedOrInvalidatedBefore;
+
/**
* bootstrappedAt defines the txnId bounds we expect to maintain data for locally.
*
@@ -122,20 +130,21 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
*/
public final @Nullable Timestamp staleUntilAtLeast;
- public Entry(Range range, long startEpoch, long endEpoch, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId gcBefore, @Nonnull TxnId bootstrappedAt, @Nullable Timestamp staleUntilAtLeast)
+ public Entry(Range range, long startOwnershipEpoch, long endOwnershipEpoch, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId shardOnlyAppliedOrInvalidatedBefore, @Nonnull TxnId gcBefore, @Nonnull TxnId bootstrappedAt, @Nullable Timestamp staleUntilAtLeast)
{
- this(range, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, gcBefore, bootstrappedAt, staleUntilAtLeast);
+ this(range, startOwnershipEpoch, endOwnershipEpoch, locallyAppliedOrInvalidatedBefore, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, shardOnlyAppliedOrInvalidatedBefore, gcBefore, bootstrappedAt, staleUntilAtLeast);
}
- public Entry(Range range, long startEpoch, long endEpoch, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId locallyDecidedAndAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId gcBefore, @Nonnull TxnId bootstrappedAt, @Nullable Timestamp staleUntilAtLeast)
+ public Entry(Range range, long startOwnershipEpoch, long endOwnershipEpoch, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId locallyDecidedAndAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId shardOnlyAppliedOrInvalidatedBefore, @Nonnull TxnId gcBefore, @Nonnull TxnId bootstrappedAt, @Nullable Timestamp staleUntilAtLeast)
{
this.range = range;
- this.startEpoch = startEpoch;
- this.endEpoch = endEpoch;
+ this.startOwnershipEpoch = startOwnershipEpoch;
+ this.endOwnershipEpoch = endOwnershipEpoch;
this.locallyAppliedOrInvalidatedBefore = locallyAppliedOrInvalidatedBefore;
this.locallyDecidedAndAppliedOrInvalidatedBefore = locallyDecidedAndAppliedOrInvalidatedBefore;
this.shardAppliedOrInvalidatedBefore = shardAppliedOrInvalidatedBefore;
this.gcBefore = gcBefore;
+ this.shardOnlyAppliedOrInvalidatedBefore = shardOnlyAppliedOrInvalidatedBefore;
this.bootstrappedAt = bootstrappedAt;
this.staleUntilAtLeast = staleUntilAtLeast;
Invariants.checkArgument(locallyAppliedOrInvalidatedBefore.equals(TxnId.NONE) || locallyAppliedOrInvalidatedBefore.domain().isRange());
@@ -152,32 +161,34 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
private static Entry merge(Range range, Entry cur, Entry add)
{
- if (cur.startEpoch > add.endEpoch)
+ // TODO (required): we shouldn't be trying to merge non-intersecting epochs
+ if (cur.startOwnershipEpoch > add.endOwnershipEpoch)
return cur;
- if (add.startEpoch > cur.endEpoch)
+ if (add.startOwnershipEpoch > cur.endOwnershipEpoch)
return add;
- long startEpoch = Long.max(cur.startEpoch, add.startEpoch);
- long endEpoch = Long.min(cur.endEpoch, add.endEpoch);
+ long startEpoch = Long.max(cur.startOwnershipEpoch, add.startOwnershipEpoch);
+ long endEpoch = Long.min(cur.endOwnershipEpoch, add.endOwnershipEpoch);
int cl = cur.locallyAppliedOrInvalidatedBefore.compareTo(add.locallyAppliedOrInvalidatedBefore);
int cd = cur.locallyDecidedAndAppliedOrInvalidatedBefore.compareTo(add.locallyDecidedAndAppliedOrInvalidatedBefore);
- int cs = cur.shardAppliedOrInvalidatedBefore.compareTo(add.shardAppliedOrInvalidatedBefore);
+ int cs = cur.shardOnlyAppliedOrInvalidatedBefore.compareTo(add.shardOnlyAppliedOrInvalidatedBefore);
int cg = cur.gcBefore.compareTo(add.gcBefore);
int cb = cur.bootstrappedAt.compareTo(add.bootstrappedAt);
int csu = compareStaleUntilAtLeast(cur.staleUntilAtLeast, add.staleUntilAtLeast);
- if (range.equals(cur.range) && startEpoch == cur.startEpoch && endEpoch == cur.endEpoch && cl >= 0 && cb >= 0 && cs >= 0 && csu >= 0)
+ if (range.equals(cur.range) && startEpoch == cur.startOwnershipEpoch && endEpoch == cur.endOwnershipEpoch && cl >= 0 && cb >= 0 && cs >= 0 && csu >= 0)
return cur;
- if (range.equals(add.range) && startEpoch == add.startEpoch && endEpoch == add.endEpoch && cl <= 0 && cb <= 0 && cs <= 0 && csu <= 0)
+ if (range.equals(add.range) && startEpoch == add.startOwnershipEpoch && endEpoch == add.endOwnershipEpoch && cl <= 0 && cb <= 0 && cs <= 0 && csu <= 0)
return add;
TxnId locallyAppliedOrInvalidatedBefore = cl >= 0 ? cur.locallyAppliedOrInvalidatedBefore : add.locallyAppliedOrInvalidatedBefore;
TxnId locallyDecidedAndAppliedOrInvalidatedBefore = cd >= 0 ? cur.locallyDecidedAndAppliedOrInvalidatedBefore : add.locallyDecidedAndAppliedOrInvalidatedBefore;
- TxnId shardAppliedOrInvalidatedBefore = cs >= 0 ? cur.shardAppliedOrInvalidatedBefore : add.shardAppliedOrInvalidatedBefore;
+ TxnId shardOnlyAppliedOrInvalidatedBefore = cs >= 0 ? cur.shardOnlyAppliedOrInvalidatedBefore : add.shardOnlyAppliedOrInvalidatedBefore;
TxnId gcBefore = cg >= 0 ? cur.gcBefore : add.gcBefore;
TxnId bootstrappedAt = cb >= 0 ? cur.bootstrappedAt : add.bootstrappedAt;
Timestamp staleUntilAtLeast = csu >= 0 ? cur.staleUntilAtLeast : add.staleUntilAtLeast;
+ TxnId shardAppliedOrInvalidatedBefore = TxnId.min(shardOnlyAppliedOrInvalidatedBefore, locallyAppliedOrInvalidatedBefore);
// if a NEW redundantBefore predates our current bootstrappedAt, we should not update it to avoid erroneously
// treating transactions prior as locally redundant when they may simply have not applied yet, since we may
@@ -193,26 +204,72 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
if (staleUntilAtLeast != null && bootstrappedAt.compareTo(staleUntilAtLeast) >= 0)
staleUntilAtLeast = null;
- return new Entry(range, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, locallyDecidedAndAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, gcBefore, bootstrappedAt, staleUntilAtLeast);
+ return new Entry(range, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, locallyDecidedAndAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, shardOnlyAppliedOrInvalidatedBefore, gcBefore, bootstrappedAt, staleUntilAtLeast);
}
- static @Nonnull RedundantStatus getAndMerge(Entry entry, @Nonnull RedundantStatus prev, TxnId txnId, EpochSupplier executeAt)
+ public Entry withShardAppliedOrInvalidatedBeforeAtLeast(TxnId newShardAppliedOrInvalidatedBefore)
{
- if (entry == null || entry.outOfBounds(txnId, executeAt))
+ if (newShardAppliedOrInvalidatedBefore.compareTo(locallyAppliedOrInvalidatedBefore) >= 0)
+ newShardAppliedOrInvalidatedBefore = locallyAppliedOrInvalidatedBefore;
+
+ if (newShardAppliedOrInvalidatedBefore.compareTo(shardAppliedOrInvalidatedBefore) <= 0)
+ return this;
+
+ return new Entry(range, startOwnershipEpoch, endOwnershipEpoch, locallyAppliedOrInvalidatedBefore, locallyDecidedAndAppliedOrInvalidatedBefore, newShardAppliedOrInvalidatedBefore, shardOnlyAppliedOrInvalidatedBefore, gcBefore, bootstrappedAt, staleUntilAtLeast);
+ }
+
+ @VisibleForImplementation
+ public Entry withEpochs(long start, long end)
+ {
+ return new Entry(range, start, end, locallyAppliedOrInvalidatedBefore, locallyDecidedAndAppliedOrInvalidatedBefore, shardOnlyAppliedOrInvalidatedBefore, gcBefore, bootstrappedAt, staleUntilAtLeast);
+ }
+
+ static @Nonnull RedundantStatus getAndMerge(Entry entry, @Nonnull RedundantStatus prev, TxnId txnId, Object ignore)
+ {
+ if (entry == null)
return prev;
return prev.merge(entry.get(txnId));
}
+ static @Nonnull Boolean isAnyOnCoordinationEpoch(Entry entry, @Nonnull Boolean prev, TxnId txnId, RedundantStatus status)
+ {
+ if (entry == null || prev)
+ return prev;
+
+ long epoch = txnId.epoch();
+ if (entry.startOwnershipEpoch > epoch || entry.endOwnershipEpoch <= epoch)
+ return false;
+
+ return entry.getIgnoringOwnership(txnId) == status;
+ }
+
+ static @Nonnull Boolean isAnyOnAnyEpoch(Entry entry, @Nonnull Boolean prev, TxnId txnId, RedundantStatus status)
+ {
+ if (entry == null || prev)
+ return prev;
+
+ long epoch = txnId.epoch();
+ if (entry.startOwnershipEpoch > epoch || entry.endOwnershipEpoch <= epoch)
+ return false;
+
+ return entry.getIgnoringOwnership(txnId) == status;
+ }
+
static RedundantStatus get(Entry entry, TxnId txnId, EpochSupplier executeAt)
{
- if (entry == null || entry.outOfBounds(txnId, executeAt))
+ if (entry == null)
return NOT_OWNED;
+
return entry.get(txnId);
}
- static PreBootstrapOrStale getAndMerge(Entry entry, @Nonnull PreBootstrapOrStale prev, TxnId txnId, EpochSupplier executeAt)
+ static PreBootstrapOrStale getAndMerge(Entry entry, @Nonnull PreBootstrapOrStale prev, TxnId txnId, Object ignore)
{
- if (prev == PARTIALLY || entry == null || entry.outOfBounds(txnId, executeAt))
+ if (prev == PARTIALLY || entry == null)
+ return prev;
+
+ // TODO (required): consider all call-sites and confirm the answers when wasOwned and willBeOwned are reasonable
+ if (entry.wasOwned(txnId) && entry.isComplete())
return prev;
boolean isPreBootstrapOrStale = entry.staleUntilAtLeast != null || entry.bootstrappedAt.compareTo(txnId) > 0;
@@ -222,7 +279,7 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
static <T extends Deps> Deps.AbstractBuilder<T> collectDep(Entry entry, @Nonnull Deps.AbstractBuilder<T> prev, @Nonnull EpochSupplier minEpoch, @Nonnull EpochSupplier executeAt)
{
- if (entry == null || entry.outOfBounds(minEpoch, executeAt))
+ if (entry == null)
return prev;
if (entry.shardAppliedOrInvalidatedBefore.compareTo(Timestamp.NONE) > 0)
@@ -264,18 +321,41 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
return notRedundant;
}
+ static Ranges removePreBootstrap(Entry entry, @Nonnull Ranges notPreBootstrap, TxnId txnId, Object ignore)
+ {
+ if (entry == null)
+ return notPreBootstrap;
+
+ if (txnId.compareTo(entry.bootstrappedAt) < 0)
+ return notPreBootstrap.without(Ranges.of(entry.range));
+
+ return notPreBootstrap;
+ }
+
RedundantStatus get(TxnId txnId)
+ {
+ if (wasOwned(txnId) && isComplete())
+ return WAS_OWNED;
+ return getIgnoringOwnership(txnId);
+ }
+
+ RedundantStatus getIgnoringOwnership(TxnId txnId)
{
// we have to first check bootstrappedAt, since we are not locally redundant for the covered range
// if the txnId is partially pre-bootstrap (since we may not have applied it for this range)
if (staleUntilAtLeast != null || bootstrappedAt.compareTo(txnId) > 0)
return PRE_BOOTSTRAP_OR_STALE;
+
if (locallyAppliedOrInvalidatedBefore.compareTo(txnId) > 0)
{
if (shardAppliedOrInvalidatedBefore.compareTo(txnId) > 0)
return SHARD_REDUNDANT;
return LOCALLY_REDUNDANT;
}
+
+ if (txnId.epoch() < startOwnershipEpoch)
+ return PRE_BOOTSTRAP_OR_STALE;
+
return LIVE;
}
@@ -301,22 +381,36 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
return TxnId.max(locallyAppliedOrInvalidatedBefore, bootstrappedAt);
}
- // TODO (required, consider): this admits the range of epochs that cross the two timestamps, which matches our
- // behaviour elsewhere but we probably want to only interact with the two point epochs in which we participate,
- // but note hasRedundantDependencies which really does want to scan a range
private boolean outOfBounds(EpochSupplier lb, EpochSupplier ub)
{
- return ub.epoch() < startEpoch || lb.epoch() >= endEpoch;
+ return ub.epoch() < startOwnershipEpoch || lb.epoch() >= endOwnershipEpoch;
+ }
+
+ private boolean wasOwned(EpochSupplier lb)
+ {
+ return lb.epoch() >= endOwnershipEpoch;
+ }
+
+ // TODO (required): do we still need this, or can we stick to just the explicit endOwnershipEpoch
+ private boolean isComplete()
+ {
+ // TODO (required): carefully consider whether we should ALSO expect some local property to be met here
+ return endOwnershipEpoch <= shardAppliedOrInvalidatedBefore.epoch();
}
private boolean outOfBounds(Timestamp lb)
{
- return lb.epoch() >= endEpoch;
+ return lb.epoch() >= endOwnershipEpoch;
}
- Entry withRange(Range range)
+ Entry withEpochs(int startEpoch, int endEpoch)
{
- return new Entry(range, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, gcBefore, bootstrappedAt, staleUntilAtLeast);
+ return new Entry(range, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, shardOnlyAppliedOrInvalidatedBefore, gcBefore, bootstrappedAt, staleUntilAtLeast);
+ }
+
+ public Entry withRange(Range range)
+ {
+ return new Entry(range, startOwnershipEpoch, endOwnershipEpoch, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, shardOnlyAppliedOrInvalidatedBefore, gcBefore, bootstrappedAt, staleUntilAtLeast);
}
public boolean equals(Object that)
@@ -331,9 +425,10 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
public boolean equalsIgnoreRange(Entry that)
{
- return this.startEpoch == that.startEpoch
- && this.endEpoch == that.endEpoch
+ return this.startOwnershipEpoch == that.startOwnershipEpoch
+ && this.endOwnershipEpoch == that.endOwnershipEpoch
&& this.locallyAppliedOrInvalidatedBefore.equals(that.locallyAppliedOrInvalidatedBefore)
+ && this.locallyDecidedAndAppliedOrInvalidatedBefore.equals(that.locallyDecidedAndAppliedOrInvalidatedBefore)
&& this.shardAppliedOrInvalidatedBefore.equals(that.shardAppliedOrInvalidatedBefore)
&& this.bootstrappedAt.equals(that.bootstrappedAt)
&& Objects.equals(this.staleUntilAtLeast, that.staleUntilAtLeast);
@@ -343,8 +438,8 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
public String toString()
{
return "("
- + (startEpoch == Long.MIN_VALUE ? "-\u221E" : Long.toString(startEpoch)) + ","
- + (endEpoch == Long.MAX_VALUE ? "\u221E" : Long.toString(endEpoch)) + ","
+ + (startOwnershipEpoch == Long.MIN_VALUE ? "-\u221E" : Long.toString(startOwnershipEpoch)) + ","
+ + (endOwnershipEpoch == Long.MAX_VALUE ? "\u221E" : Long.toString(endOwnershipEpoch)) + ","
+ (locallyAppliedOrInvalidatedBefore.compareTo(bootstrappedAt) >= 0 ? locallyAppliedOrInvalidatedBefore + ")" : bootstrappedAt + "*)");
}
}
@@ -387,27 +482,28 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
return Ranges.ofSortedAndDeoverlapped(staleRanges).mergeTouching();
}
- public static RedundantBefore create(Ranges ranges, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId gcBefore, @Nonnull TxnId bootstrappedAt)
+ public static RedundantBefore create(AbstractRanges ranges, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId gcBefore, @Nonnull TxnId bootstrappedAt)
{
return create(ranges, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, gcBefore, bootstrappedAt, null);
}
- public static RedundantBefore create(Ranges ranges, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId gcBefore, @Nonnull TxnId bootstrappedAt, @Nullable Timestamp staleUntilAtLeast)
+ public static RedundantBefore create(AbstractRanges ranges, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId gcBefore, @Nonnull TxnId bootstrappedAt, @Nullable Timestamp staleUntilAtLeast)
{
return create(ranges, Long.MIN_VALUE, Long.MAX_VALUE, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, gcBefore, bootstrappedAt, staleUntilAtLeast);
}
- public static RedundantBefore create(Ranges ranges, long startEpoch, long endEpoch, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId gcBefore, @Nonnull TxnId bootstrappedAt)
+ public static RedundantBefore create(AbstractRanges ranges, long startEpoch, long endEpoch, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId gcBefore, @Nonnull TxnId bootstrappedAt)
{
return create(ranges, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, gcBefore, bootstrappedAt, null);
}
- public static RedundantBefore create(Ranges ranges, long startEpoch, long endEpoch, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId gcBefore, @Nonnull TxnId bootstrappedAt, @Nullable Timestamp staleUntilAtLeast)
+ public static RedundantBefore create(AbstractRanges ranges, long startEpoch, long endEpoch, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardOnlyAppliedOrInvalidatedBefore, @Nonnull TxnId gcBefore, @Nonnull TxnId bootstrappedAt, @Nullable Timestamp staleUntilAtLeast)
{
if (ranges.isEmpty())
return new RedundantBefore();
- Entry entry = new Entry(null, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, gcBefore, bootstrappedAt, staleUntilAtLeast);
+ TxnId shardAppliedOrInvalidatedBefore = TxnId.min(locallyAppliedOrInvalidatedBefore, shardOnlyAppliedOrInvalidatedBefore);
+ Entry entry = new Entry(null, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, shardOnlyAppliedOrInvalidatedBefore, gcBefore, bootstrappedAt, staleUntilAtLeast);
Builder builder = new Builder(ranges.get(0).endInclusive(), ranges.size() * 2);
for (int i = 0 ; i < ranges.size() ; ++i)
{
@@ -429,40 +525,32 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
return Entry.get(entry, txnId, executeAt);
}
- public TxnId shardRedundantBefore(RoutableKey key)
- {
- Entry entry = get(key);
- if (entry == null)
- return TxnId.NONE;
- return entry.shardAppliedOrInvalidatedBefore;
+ public RedundantStatus status(TxnId txnId, Participants<?> participants)
+ { // TODO (required): consider how the use of txnId for executeAt affects exclusive sync points for cleanup
+ // may want to issue synthetic sync points for local evaluation in later epochs
+ return foldl(participants, Entry::getAndMerge, NOT_OWNED, txnId, null, ignore -> false);
}
- public TxnId locallyRedundantBefore(RoutableKey key)
+ public boolean isAnyOnCoordinationEpoch(TxnId txnId, Unseekables<?> participants, RedundantStatus status)
{
- Entry entry = get(key);
- if (entry == null)
- return TxnId.NONE;
- return entry.locallyAppliedOrInvalidatedBefore;
+ return foldl(participants, Entry::isAnyOnCoordinationEpoch, false, txnId, status, isDone -> isDone);
}
- public RedundantStatus status(TxnId txnId, EpochSupplier executeAt, Participants<?> participants)
- { // TODO (required): consider how the use of txnId for executeAt affects exclusive sync points for cleanup
- // may want to issue synthetic sync points for local evaluation in later epochs
- if (executeAt == null) executeAt = txnId;
- return foldl(participants, Entry::getAndMerge, NOT_OWNED, txnId, executeAt, ignore -> false);
+ public boolean isAnyOnAnyEpoch(TxnId txnId, Unseekables<?> participants, RedundantStatus status)
+ {
+ return foldl(participants, Entry::isAnyOnAnyEpoch, false, txnId, status, isDone -> isDone);
}
/**
* RedundantStatus.REDUNDANT overrides PRE_BOOTSTRAP; to avoid complicating that state machine,
* for cases where we care independently about the overall pre-bootstrap state we have a separate mechanism
*/
- public PreBootstrapOrStale preBootstrapOrStale(TxnId txnId, @Nullable EpochSupplier executeAt, Participants<?> participants)
+ public PreBootstrapOrStale preBootstrapOrStale(TxnId txnId, Participants<?> participants)
{
- if (executeAt == null) executeAt = txnId;
- return foldl(participants, Entry::getAndMerge, PreBootstrapOrStale.NOT_OWNED, txnId, executeAt, r -> r == PARTIALLY);
+ return foldl(participants, Entry::getAndMerge, PreBootstrapOrStale.NOT_OWNED, txnId, null, r -> r == PARTIALLY);
}
- public <T extends Deps> Deps.AbstractBuilder<T> collectDeps(Seekables<?, ?> participants, Deps.AbstractBuilder<T> builder, EpochSupplier minEpoch, EpochSupplier executeAt)
+ public <T extends Deps> Deps.AbstractBuilder<T> collectDeps(Routables<?> participants, Deps.AbstractBuilder<T> builder, EpochSupplier minEpoch, EpochSupplier executeAt)
{
return foldl(participants, Entry::collectDep, builder, minEpoch, executeAt, ignore -> false);
}
@@ -481,6 +569,14 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
return foldl(ranges, Entry::removeShardRedundant, ranges, txnId, executeAt, r -> false);
}
+ /**
+ * Subtract any ranges we consider stale or pre-bootstrap
+ */
+ public Ranges removePreBootstrap(TxnId txnId, Ranges ranges)
+ {
+ return foldl(ranges, Entry::removePreBootstrap, ranges, txnId, null, r -> false);
+ }
+
/**
* Subtract any ranges we consider stale or pre-bootstrap
*/
@@ -520,7 +616,7 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
if (v.range.start().equals(start) && v.range.end().equals(end))
return v;
- return new Entry(v.range.newRange(start, end), v.startEpoch, v.endEpoch, v.locallyAppliedOrInvalidatedBefore, v.shardAppliedOrInvalidatedBefore, v.gcBefore, v.bootstrappedAt, v.staleUntilAtLeast);
+ return new Entry(v.range.newRange(start, end), v.startOwnershipEpoch, v.endOwnershipEpoch, v.locallyAppliedOrInvalidatedBefore, v.shardAppliedOrInvalidatedBefore, v.shardOnlyAppliedOrInvalidatedBefore, v.gcBefore, v.bootstrappedAt, v.staleUntilAtLeast);
}
@Override
@@ -539,7 +635,7 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
return new Entry(a.range.newRange(
a.range.start().compareTo(b.range.start()) <= 0 ? a.range.start() : b.range.start(),
a.range.end().compareTo(b.range.end()) >= 0 ? a.range.end() : b.range.end()
- ), a.startEpoch, a.endEpoch, a.locallyAppliedOrInvalidatedBefore, a.shardAppliedOrInvalidatedBefore, a.gcBefore, a.bootstrappedAt, a.staleUntilAtLeast);
+ ), a.startOwnershipEpoch, a.endOwnershipEpoch, a.locallyAppliedOrInvalidatedBefore, a.shardAppliedOrInvalidatedBefore, a.shardOnlyAppliedOrInvalidatedBefore, a.gcBefore, a.bootstrappedAt, a.staleUntilAtLeast);
}
@Override
diff --git a/accord-core/src/main/java/accord/local/RedundantStatus.java b/accord-core/src/main/java/accord/local/RedundantStatus.java
index 3afa8c11..e3338cd8 100644
--- a/accord-core/src/main/java/accord/local/RedundantStatus.java
+++ b/accord-core/src/main/java/accord/local/RedundantStatus.java
@@ -29,6 +29,11 @@ public enum RedundantStatus
*/
NOT_OWNED,
+ /**
+ * None of the relevant ranges are owned by the command store anymore
+ */
+ WAS_OWNED,
+
/**
* Some of the relevant ranges are owned by the command store and valid for execution
*/
@@ -79,14 +84,24 @@ public enum RedundantStatus
{
NOT_OWNED.merge = new EnumMap<>(RedundantStatus.class);
NOT_OWNED.merge.put(NOT_OWNED, NOT_OWNED);
+ NOT_OWNED.merge.put(WAS_OWNED, WAS_OWNED);
NOT_OWNED.merge.put(LIVE, LIVE);
NOT_OWNED.merge.put(PARTIALLY_PRE_BOOTSTRAP_OR_STALE, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
NOT_OWNED.merge.put(PRE_BOOTSTRAP_OR_STALE, PRE_BOOTSTRAP_OR_STALE);
NOT_OWNED.merge.put(REDUNDANT_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
NOT_OWNED.merge.put(LOCALLY_REDUNDANT, LOCALLY_REDUNDANT);
NOT_OWNED.merge.put(SHARD_REDUNDANT, SHARD_REDUNDANT);
+ WAS_OWNED.merge = new EnumMap<>(RedundantStatus.class);
+ WAS_OWNED.merge.put(NOT_OWNED, NOT_OWNED);
+ WAS_OWNED.merge.put(LIVE, LIVE);
+ WAS_OWNED.merge.put(PARTIALLY_PRE_BOOTSTRAP_OR_STALE, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
+ WAS_OWNED.merge.put(PRE_BOOTSTRAP_OR_STALE, PRE_BOOTSTRAP_OR_STALE);
+ WAS_OWNED.merge.put(REDUNDANT_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+ WAS_OWNED.merge.put(LOCALLY_REDUNDANT, LOCALLY_REDUNDANT);
+ WAS_OWNED.merge.put(SHARD_REDUNDANT, SHARD_REDUNDANT);
LIVE.merge = new EnumMap<>(RedundantStatus.class);
LIVE.merge.put(NOT_OWNED, LIVE);
+ LIVE.merge.put(WAS_OWNED, LIVE);
LIVE.merge.put(LIVE, LIVE);
LIVE.merge.put(PARTIALLY_PRE_BOOTSTRAP_OR_STALE, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
LIVE.merge.put(PRE_BOOTSTRAP_OR_STALE, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
@@ -95,6 +110,7 @@ public enum RedundantStatus
LIVE.merge.put(SHARD_REDUNDANT, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
PARTIALLY_PRE_BOOTSTRAP_OR_STALE.merge = new EnumMap<>(RedundantStatus.class);
PARTIALLY_PRE_BOOTSTRAP_OR_STALE.merge.put(NOT_OWNED, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
+ PARTIALLY_PRE_BOOTSTRAP_OR_STALE.merge.put(WAS_OWNED, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
PARTIALLY_PRE_BOOTSTRAP_OR_STALE.merge.put(LIVE, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
PARTIALLY_PRE_BOOTSTRAP_OR_STALE.merge.put(PARTIALLY_PRE_BOOTSTRAP_OR_STALE, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
PARTIALLY_PRE_BOOTSTRAP_OR_STALE.merge.put(PRE_BOOTSTRAP_OR_STALE, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
@@ -103,6 +119,7 @@ public enum RedundantStatus
PARTIALLY_PRE_BOOTSTRAP_OR_STALE.merge.put(SHARD_REDUNDANT, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
PRE_BOOTSTRAP_OR_STALE.merge = new EnumMap<>(RedundantStatus.class);
PRE_BOOTSTRAP_OR_STALE.merge.put(NOT_OWNED, PRE_BOOTSTRAP_OR_STALE);
+ PRE_BOOTSTRAP_OR_STALE.merge.put(WAS_OWNED, PRE_BOOTSTRAP_OR_STALE);
PRE_BOOTSTRAP_OR_STALE.merge.put(LIVE, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
PRE_BOOTSTRAP_OR_STALE.merge.put(PARTIALLY_PRE_BOOTSTRAP_OR_STALE, PARTIALLY_PRE_BOOTSTRAP_OR_STALE);
PRE_BOOTSTRAP_OR_STALE.merge.put(PRE_BOOTSTRAP_OR_STALE, PRE_BOOTSTRAP_OR_STALE);
@@ -111,6 +128,7 @@ public enum RedundantStatus
PRE_BOOTSTRAP_OR_STALE.merge.put(SHARD_REDUNDANT, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
REDUNDANT_PRE_BOOTSTRAP_OR_STALE.merge = new EnumMap<>(RedundantStatus.class);
REDUNDANT_PRE_BOOTSTRAP_OR_STALE.merge.put(NOT_OWNED, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
+ REDUNDANT_PRE_BOOTSTRAP_OR_STALE.merge.put(WAS_OWNED, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
REDUNDANT_PRE_BOOTSTRAP_OR_STALE.merge.put(LIVE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE); // TODO (expected): should this be an invalid combination?
REDUNDANT_PRE_BOOTSTRAP_OR_STALE.merge.put(PARTIALLY_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
REDUNDANT_PRE_BOOTSTRAP_OR_STALE.merge.put(PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
@@ -119,6 +137,7 @@ public enum RedundantStatus
REDUNDANT_PRE_BOOTSTRAP_OR_STALE.merge.put(SHARD_REDUNDANT, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
LOCALLY_REDUNDANT.merge = new EnumMap<>(RedundantStatus.class);
LOCALLY_REDUNDANT.merge.put(NOT_OWNED, LOCALLY_REDUNDANT);
+ LOCALLY_REDUNDANT.merge.put(WAS_OWNED, LOCALLY_REDUNDANT);
LOCALLY_REDUNDANT.merge.put(LIVE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE); // TODO (expected): should this be an invalid combination?
LOCALLY_REDUNDANT.merge.put(PARTIALLY_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
LOCALLY_REDUNDANT.merge.put(PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
@@ -127,6 +146,7 @@ public enum RedundantStatus
LOCALLY_REDUNDANT.merge.put(SHARD_REDUNDANT, LOCALLY_REDUNDANT);
SHARD_REDUNDANT.merge = new EnumMap<>(RedundantStatus.class);
SHARD_REDUNDANT.merge.put(NOT_OWNED, SHARD_REDUNDANT);
+ SHARD_REDUNDANT.merge.put(WAS_OWNED, SHARD_REDUNDANT);
SHARD_REDUNDANT.merge.put(LIVE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE); // TODO (expected): should this be an invalid combination?
SHARD_REDUNDANT.merge.put(PARTIALLY_PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
SHARD_REDUNDANT.merge.put(PRE_BOOTSTRAP_OR_STALE, REDUNDANT_PRE_BOOTSTRAP_OR_STALE);
diff --git a/accord-core/src/main/java/accord/local/RejectBefore.java b/accord-core/src/main/java/accord/local/RejectBefore.java
new file mode 100644
index 00000000..7912c7b3
--- /dev/null
+++ b/accord-core/src/main/java/accord/local/RejectBefore.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package accord.local;
+
+import java.util.Objects;
+
+import accord.api.RoutingKey;
+import accord.primitives.AbstractRanges;
+import accord.primitives.Routables;
+import accord.primitives.TxnId;
+import accord.utils.ReducingIntervalMap;
+import accord.utils.ReducingRangeMap;
+
+public class RejectBefore extends ReducingRangeMap<TxnId>
+{
+ public static class SerializerSupport
+ {
+ public static RejectBefore create(boolean inclusiveEnds, RoutingKey[] ends, TxnId[] values)
+ {
+ return new RejectBefore(inclusiveEnds, ends, values);
+ }
+ }
+
+ public RejectBefore()
+ {
+ super();
+ }
+
+ protected RejectBefore(boolean inclusiveEnds, RoutingKey[] starts, TxnId[] values)
+ {
+ super(inclusiveEnds, starts, values);
+ }
+
+ public static RejectBefore add(RejectBefore existing, AbstractRanges ranges, TxnId value)
+ {
+ RejectBefore add = create(ranges, value);
+ return merge(existing, add);
+ }
+
+ public boolean rejects(TxnId txnId, Routables<?> participants)
+ {
+ return null == foldl(participants, (rejectIfBefore, test) -> rejects(rejectIfBefore, test) ? null : test, txnId, Objects::isNull);
+ }
+
+ private static boolean rejects(TxnId rejectIfBefore, TxnId test)
+ {
+ return rejectIfBefore.compareTo(test) > 0;
+ }
+
+ public static RejectBefore merge(RejectBefore historyLeft, RejectBefore historyRight)
+ {
+ return ReducingIntervalMap.merge(historyLeft, historyRight, TxnId::max, Builder::new);
+ }
+
+ public static RejectBefore create(AbstractRanges ranges, TxnId value)
+ {
+ if (value == null)
+ throw new IllegalArgumentException("value is null");
+
+ if (ranges.isEmpty())
+ return new RejectBefore();
+
+ return create(ranges, value, Builder::new);
+ }
+
+ static class Builder extends AbstractBoundariesBuilder<RoutingKey, TxnId, RejectBefore>
+ {
+ protected Builder(boolean inclusiveEnds, int capacity)
+ {
+ super(inclusiveEnds, capacity);
+ }
+
+ @Override
+ protected RejectBefore buildInternal()
+ {
+ return new RejectBefore(inclusiveEnds, starts.toArray(new RoutingKey[0]), values.toArray(new TxnId[0]));
+ }
+ }
+}
diff --git a/accord-core/src/main/java/accord/local/SafeCommand.java b/accord-core/src/main/java/accord/local/SafeCommand.java
index b89b8d94..c27f0c9b 100644
--- a/accord-core/src/main/java/accord/local/SafeCommand.java
+++ b/accord-core/src/main/java/accord/local/SafeCommand.java
@@ -21,9 +21,10 @@ package accord.local;
import accord.api.Result;
import accord.local.Command.Truncated;
import accord.primitives.Ballot;
-import accord.primitives.Seekables;
+import accord.primitives.Status;
import accord.primitives.Timestamp;
import accord.primitives.TxnId;
+import accord.primitives.Unseekables;
import accord.primitives.Writes;
import accord.utils.Invariants;
@@ -101,34 +102,13 @@ public abstract class SafeCommand
return incidentalUpdate(current().updatePromised(promised));
}
- public Command.Accepted accept(SafeCommandStore safeStore, Seekables<?, ?> keysOrRanges, CommonAttributes attrs, Timestamp executeAt, Ballot ballot)
+ public Command.Accepted accept(SafeCommandStore safeStore, Unseekables<?> keysOrRanges, CommonAttributes attrs, Timestamp executeAt, Ballot ballot)
{
Command current = current();
- attrs = updateKeysOrRanges(attrs, keysOrRanges);
Command.Accepted updated = Command.accept(current, attrs, executeAt, ballot);
return update(safeStore, updated);
}
- private static CommonAttributes updateKeysOrRanges(CommonAttributes attrs, Seekables<?, ?> keysOrRanges)
- {
- if (attrs.partialTxn() != null && attrs.partialTxn().keys().containsAll(keysOrRanges))
- return attrs;
-
- if (attrs.additionalKeysOrRanges() != null && attrs.additionalKeysOrRanges().containsAll(keysOrRanges))
- return attrs;
-
- if (attrs.partialTxn() != null)
- keysOrRanges = ((Seekables)keysOrRanges).without(attrs.partialTxn().keys());
-
- if (attrs.additionalKeysOrRanges() != null)
- keysOrRanges = ((Seekables)keysOrRanges).with(attrs.additionalKeysOrRanges());
-
- if (!keysOrRanges.isEmpty())
- attrs = attrs.mutable().additionalKeysOrRanges(keysOrRanges);
-
- return attrs;
- }
-
public Command acceptInvalidated(SafeCommandStore safeStore, Ballot ballot)
{
return update(safeStore, Command.acceptInvalidated(current(), ballot));
diff --git a/accord-core/src/main/java/accord/local/SafeCommandStore.java b/accord-core/src/main/java/accord/local/SafeCommandStore.java
index cc51f7c8..50c6410c 100644
--- a/accord-core/src/main/java/accord/local/SafeCommandStore.java
+++ b/accord-core/src/main/java/accord/local/SafeCommandStore.java
@@ -19,41 +19,38 @@
package accord.local;
import java.util.NavigableMap;
+import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import accord.api.Agent;
import accord.api.DataStore;
-import accord.api.Key;
import accord.api.LocalListeners;
import accord.api.ProgressLog;
import accord.api.RoutingKey;
-import accord.impl.ErasedSafeCommand;
import accord.local.cfk.CommandsForKey;
import accord.local.cfk.SafeCommandsForKey;
+import accord.primitives.AbstractUnseekableKeys;
import accord.primitives.Deps;
-import accord.primitives.EpochSupplier;
-import accord.primitives.Keys;
import accord.primitives.Participants;
import accord.primitives.Ranges;
import accord.primitives.Routable.Domain;
import accord.primitives.Routables;
-import accord.primitives.Seekable;
-import accord.primitives.Seekables;
+import accord.primitives.RoutingKeys;
+import accord.primitives.SaveStatus;
+import accord.primitives.Status;
import accord.primitives.Timestamp;
import accord.primitives.Txn.Kind;
import accord.primitives.Txn.Kind.Kinds;
import accord.primitives.TxnId;
+import accord.primitives.Unseekable;
import accord.primitives.Unseekables;
import accord.utils.Invariants;
import static accord.local.Cleanup.NO;
import static accord.local.KeyHistory.COMMANDS;
import static accord.local.RedundantBefore.PreBootstrapOrStale.FULLY;
-import static accord.local.SaveStatus.Applied;
-import static accord.local.SaveStatus.Erased;
-import static accord.local.SaveStatus.ErasedOrInvalidOrVestigial;
import static accord.primitives.Routables.Slice.Minimal;
-import static accord.primitives.Route.isFullRoute;
+import static accord.primitives.SaveStatus.Applied;
/**
* A CommandStore with exclusive access; a reference to this should not be retained outside of the scope of the method
@@ -65,7 +62,7 @@ public abstract class SafeCommandStore
{
public interface CommandFunction<P1, I, O>
{
- O apply(P1 p1, Seekable keyOrRange, TxnId txnId, Timestamp executeAt, I in);
+ O apply(P1 p1, Unseekable keyOrRange, TxnId txnId, Timestamp executeAt, I in);
}
public enum TestStartedAt
@@ -90,19 +87,7 @@ public abstract class SafeCommandStore
Command command = safeCommand.current();
if (command.saveStatus().isUninitialised())
return null;
- return maybeTruncate(safeCommand, command, txnId, null);
- }
-
- public SafeCommand get(TxnId txnId, RoutingKey unseekable)
- {
- SafeCommand safeCommand = get(txnId);
- Command command = safeCommand.current();
- if (command.saveStatus().isUninitialised())
- {
- if (commandStore().durableBefore().isUniversal(txnId, unseekable))
- return new ErasedSafeCommand(txnId, ErasedOrInvalidOrVestigial);
- }
- return maybeTruncate(safeCommand, command, txnId, null);
+ return maybeCleanup(safeCommand);
}
// decidedExecuteAt == null if not yet PreCommitted
@@ -125,21 +110,21 @@ public abstract class SafeCommandStore
*
* TODO (desired): Introduce static types that permit us to propagate this information safely.
*/
- public SafeCommand get(TxnId txnId, EpochSupplier toEpoch, Unseekables<?> unseekables)
+ public SafeCommand get(TxnId txnId, StoreParticipants participants)
{
SafeCommand safeCommand = get(txnId);
- Command command = safeCommand.current();
- if (command.saveStatus().isUninitialised())
- {
- if (Cleanup.isSafeToCleanup(commandStore().durableBefore(), txnId, unseekables))
- return new ErasedSafeCommand(txnId, isFullRoute(unseekables) ? Erased : ErasedOrInvalidOrVestigial);
- }
- return maybeTruncate(safeCommand, command, toEpoch, unseekables);
+ Commands.maybeCleanup(this, safeCommand, safeCommand.current(), participants);
+ return safeCommand;
+ }
+
+ protected SafeCommand maybeCleanup(SafeCommand safeCommand)
+ {
+ return maybeCleanup(safeCommand, safeCommand.current(), StoreParticipants.empty(safeCommand.txnId()));
}
- protected SafeCommand maybeTruncate(SafeCommand safeCommand, Command command, @Nullable EpochSupplier toEpoch, @Nullable Unseekables<?> maybeFullRoute)
+ protected SafeCommand maybeCleanup(SafeCommand safeCommand, Command command, @Nonnull StoreParticipants participants)
{
- Commands.maybeCleanup(this, safeCommand, command, toEpoch, maybeFullRoute);
+ Commands.maybeCleanup(this, safeCommand, command, participants);
return safeCommand;
}
@@ -154,13 +139,13 @@ public abstract class SafeCommandStore
SafeCommand safeCommand = getInternalIfLoadedAndInitialised(txnId);
if (safeCommand == null)
return null;
- return maybeTruncate(safeCommand, safeCommand.current(), txnId, null);
+ return maybeCleanup(safeCommand, safeCommand.current(), StoreParticipants.empty(txnId));
}
protected SafeCommand get(TxnId txnId)
{
SafeCommand safeCommand = getInternal(txnId);
- return maybeTruncate(safeCommand, safeCommand.current(), null, null);
+ return maybeCleanup(safeCommand, safeCommand.current(), StoreParticipants.empty(txnId));
}
public SafeCommand unsafeGet(TxnId txnId)
@@ -168,7 +153,7 @@ public abstract class SafeCommandStore
return get(txnId);
}
- protected SafeCommandsForKey maybeTruncate(SafeCommandsForKey safeCfk)
+ protected SafeCommandsForKey maybeCleanup(SafeCommandsForKey safeCfk)
{
RedundantBefore.Entry entry = commandStore().redundantBefore().get(safeCfk.key().toUnseekable());
if (entry != null)
@@ -182,18 +167,18 @@ public abstract class SafeCommandStore
*
* This permits efficient operation when a transaction involved in processing another transaction happens to be in memory.
*/
- public final SafeCommandsForKey ifLoadedAndInitialised(Key key)
+ public final SafeCommandsForKey ifLoadedAndInitialised(RoutingKey key)
{
SafeCommandsForKey safeCfk = getInternalIfLoadedAndInitialised(key);
if (safeCfk == null)
return null;
- return maybeTruncate(safeCfk);
+ return maybeCleanup(safeCfk);
}
- public SafeCommandsForKey get(Key key)
+ public SafeCommandsForKey get(RoutingKey key)
{
SafeCommandsForKey safeCfk = getInternal(key);
- return maybeTruncate(safeCfk);
+ return maybeCleanup(safeCfk);
}
public long preAcceptTimeout()
@@ -203,8 +188,8 @@ public abstract class SafeCommandStore
protected abstract SafeCommand getInternal(TxnId txnId);
protected abstract SafeCommand getInternalIfLoadedAndInitialised(TxnId txnId);
- protected abstract SafeCommandsForKey getInternal(Key key);
- protected abstract SafeCommandsForKey getInternalIfLoadedAndInitialised(Key key);
+ protected abstract SafeCommandsForKey getInternal(RoutingKey key);
+ protected abstract SafeCommandsForKey getInternalIfLoadedAndInitialised(RoutingKey key);
public abstract boolean canExecuteWith(PreLoadContext context);
protected void update(Command prev, Command updated)
@@ -244,7 +229,7 @@ public abstract class SafeCommandStore
return;
TxnId txnId = updated.txnId();
- if (!txnId.kind().isGloballyVisible())
+ if (!txnId.isVisible())
return;
commandStore().updateMaxConflicts(prev, updated);
@@ -298,20 +283,17 @@ public abstract class SafeCommandStore
private static void updateManagedCommandsForKey(SafeCommandStore safeStore, Command prev, Command next)
{
TxnId txnId = next.txnId();
- Keys keys = (Keys)next.keysOrRanges();
- if (keys == null || next.hasBeen(Status.Truncated)) keys = (Keys)prev.keysOrRanges();
- if (keys == null)
- return;
+ StoreParticipants participants = next.participants().supplement(prev.participants());
+ Participants<?> update = next.hasBeen(Status.Committed) ? participants.hasTouched : participants.touches;
- // TODO (required): additionalKeysOrRanges may not be being handled entirely correctly here, though it may not matter.
- // Once committed without a given key, we should be effectively erasing the command from that CFK
- PreLoadContext context = PreLoadContext.contextFor(txnId, keys, COMMANDS);
+ // TODO (expected): we don't want to insert any dependencies for those we only touch; we just need to record them as decided/applied for execution
+ PreLoadContext context = PreLoadContext.contextFor(txnId, update, COMMANDS);
// TODO (expected): execute immediately for any keys we already have loaded, and save only those we haven't for later
if (safeStore.canExecuteWith(context))
{
- for (Key key : keys)
+ for (RoutingKey key : (AbstractUnseekableKeys)update)
{
- safeStore.get(key).update(safeStore, next);
+ safeStore.get(key).update(safeStore, next, update != participants.touches && !participants.touches(key));
}
}
else
@@ -325,7 +307,7 @@ public abstract class SafeCommandStore
private static void updateUnmanagedExecutionCommandsForKey(SafeCommandStore safeStore, Command next)
{
TxnId txnId = next.txnId();
- Keys keys = next.asCommitted().waitingOn.keys;
+ RoutingKeys keys = next.asCommitted().waitingOn.keys;
// TODO (required): consider how execution works for transactions that await future deps and where the command store inherits additional keys in execution epoch
Ranges ranges = safeStore.ranges().allAt(next.executeAt());
PreLoadContext context = PreLoadContext.contextFor(txnId, keys, COMMANDS);
@@ -352,13 +334,13 @@ public abstract class SafeCommandStore
* Within each key or range visits all visible txnids needed for the given scope in ascending order of queried timestamp.
* TODO (expected): no need for slice in most (all?) cases
*/
- public abstract <P1, T> T mapReduceActive(Seekables<?, ?> keys, Ranges slice, @Nullable Timestamp withLowerTxnId, Kinds kinds, CommandFunction<P1, T, T> map, P1 p1, T initialValue);
+ public abstract <P1, T> T mapReduceActive(Unseekables<?> keys, @Nullable Timestamp withLowerTxnId, Kinds kinds, CommandFunction<P1, T, T> map, P1 p1, T initialValue);
/**
* Visits keys first and then ranges, both in ascending order.
* Within each key or range visits all unevicted txnids needed for the given scope in ascending order of queried timestamp.
*/
- public abstract <P1, T> T mapReduceFull(Seekables<?, ?> keys, Ranges slice,
+ public abstract <P1, T> T mapReduceFull(Unseekables<?> keys,
TxnId testTxnId,
Kinds testKind,
TestStartedAt testStartedAt,
@@ -373,19 +355,34 @@ public abstract class SafeCommandStore
public abstract NodeTimeService time();
public abstract CommandStores.RangesForEpoch ranges();
- public boolean isTruncated(Command command)
+ public Ranges futureRanges(TxnId txnId)
+ {
+ return ranges().allBefore(txnId.epoch());
+ }
+
+ public Ranges coordinateRanges(TxnId txnId)
{
- return Cleanup.shouldCleanup(this, command, null, null) != NO;
+ return ranges().allAt(txnId.epoch());
+ }
+
+ public Ranges ranges(TxnId txnId, Timestamp executeAt)
+ {
+ return ranges(txnId, executeAt.epoch());
+ }
+
+ public Ranges ranges(TxnId txnId, long untilLocalEpoch)
+ {
+ return ranges().allBetween(txnId.epoch(), untilLocalEpoch);
}
// if we have to re-bootstrap (due to failed bootstrap or catching up on a range) then we may
// have dangling redundant commands; these can safely be executed locally because we are a timestamp store
final boolean isFullyPreBootstrapOrStale(Command command, Participants<?> forKeys)
{
- return commandStore().redundantBefore().preBootstrapOrStale(command.txnId(), command.executeAtOrTxnId(), forKeys) == FULLY;
+ return commandStore().redundantBefore().preBootstrapOrStale(command.txnId(), forKeys) == FULLY;
}
- public void registerListener(SafeCommand listeningTo, SaveStatus await, Status.KnownRoute route, TxnId waiting)
+ public void registerListener(SafeCommand listeningTo, SaveStatus await, TxnId waiting)
{
Invariants.checkState(listeningTo.current().saveStatus().compareTo(await) < 0);
Invariants.checkState(!CommandsForKey.managesExecution(listeningTo.txnId()));
@@ -394,8 +391,9 @@ public abstract class SafeCommandStore
public LocalListeners.Registered registerAndInvoke(TxnId txnId, RoutingKey someKey, LocalListeners.ComplexListener listener)
{
+ StoreParticipants participants = StoreParticipants.read(this, Participants.singleton(txnId.domain(), someKey), txnId);
LocalListeners.Registered registered = register(txnId, listener);
- if (!listener.notify(this, get(txnId, someKey)))
+ if (!listener.notify(this, get(txnId, participants)))
registered.cancel();
return registered;
}
@@ -409,5 +407,4 @@ public abstract class SafeCommandStore
{
commandStore().listeners.notify(this, safeCommand, prev);
}
-
}
diff --git a/accord-core/src/main/java/accord/local/Status.java b/accord-core/src/main/java/accord/local/Status.java
deleted file mode 100644
index a39c4829..00000000
--- a/accord-core/src/main/java/accord/local/Status.java
+++ /dev/null
@@ -1,995 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package accord.local;
-
-import accord.messages.BeginRecovery;
-import accord.primitives.Ballot;
-import accord.primitives.Timestamp;
-import accord.primitives.TxnId;
-import accord.utils.Invariants;
-
-import java.util.List;
-import java.util.Objects;
-import java.util.function.Function;
-import java.util.function.Predicate;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
-
-import static accord.local.Status.Definition.*;
-import static accord.local.Status.Known.*;
-import static accord.local.Status.KnownDeps.*;
-import static accord.local.Status.KnownExecuteAt.*;
-import static accord.local.Status.KnownRoute.Covering;
-import static accord.local.Status.KnownRoute.Full;
-import static accord.local.Status.KnownRoute.Maybe;
-import static accord.local.Status.Outcome.*;
-import static accord.local.Status.Phase.*;
-
-public enum Status
-{
- NotDefined (None, Nothing),
- PreAccepted (PreAccept, DefinitionAndRoute),
- AcceptedInvalidate(Accept, Maybe, DefinitionUnknown, ExecuteAtUnknown, DepsUnknown, Unknown), // may or may not have witnessed
- Accepted (Accept, Covering, DefinitionUnknown, ExecuteAtProposed, DepsProposed, Unknown), // may or may not have witnessed
-
- /**
- * PreCommitted is a peculiar state, half-way between Accepted and Committed.
- * We know the transaction is Committed and its execution timestamp, but we do
- * not know its dependencies, and we may still have state leftover from the Accept round
- * that is necessary for recovery.
- *
- * So, for execution of other transactions we may treat a PreCommitted transaction as Committed,
- * using the timestamp to update our dependency set to rule it out as a dependency.
- * But we do not have enough information to execute the transaction, and when recovery calculates
- * {@link BeginRecovery#acceptedStartedBeforeWithoutWitnessing},
- * {@link BeginRecovery#hasCommittedExecutesAfterWithoutWitnessing},
- * and {@link BeginRecovery#committedStartedBeforeAndWitnessed} we may not have the dependencies
- * to calculate the result. For these operations we treat ourselves as whatever Accepted status
- * we may have previously taken, using any proposed dependencies to compute the result.
- *
- * This state exists primarily to permit us to efficiently separate work between different home shards.
- * Take a transaction A that reaches the Committed status and commits to all of its home shard A*'s replicas,
- * but fails to commit to all shards. A takes an execution time later than its TxnId, and in the process
- * adopts a dependency on a transaction B that is coordinated by its home shard B*, that has itself taken
- * a dependency upon A. Importantly, B commits a lower executeAt than A and so will execute first, and once A*
- * commits B, A will remove it from its dependencies. However, there is insufficient information on A*
- * to commit B since it does not know A*'s dependencies, and B* will not process B until A* executes A.
- * To solve this problem we simply permit the executeAt we discover for B to be propagated to A* without
- * its dependencies. Though this does complicate the state machine a little.
- */
- PreCommitted (Accept, Full, DefinitionUnknown, ExecuteAtKnown, DepsUnknown, Unknown),
-
- Committed (Commit, Full, DefinitionKnown, ExecuteAtKnown, DepsCommitted,Unknown),
- Stable (Execute, Full, DefinitionKnown, ExecuteAtKnown, DepsKnown, Unknown),
- PreApplied (Persist, Full, DefinitionKnown, ExecuteAtKnown, DepsKnown, Outcome.Apply),
- Applied (Persist, Full, DefinitionKnown, ExecuteAtKnown, DepsKnown, Outcome.Apply),
- Truncated (Cleanup, Maybe, DefinitionErased, ExecuteAtErased, DepsErased, Outcome.Erased),
- Invalidated (Persist, Maybe, NoOp, NoExecuteAt, NoDeps, Outcome.Invalidated),
- ;
-
- /**
- * Represents the phase of a transaction from the perspective of coordination
- * None: the transaction is not currently being processed by us (it may be known to us, but only transitively)
- * PreAccept: the transaction is being disseminated and is seeking an execution order
- * Accept: the transaction did not achieve 1RT consensus and is making durable its execution order
- * Commit: the transaction's execution time has been durably decided, and dependencies are being disseminated
- * Execute: the transaction's execution dependencies have been durably disseminated, and the transaction is waiting to execute
- * Persist: the transaction has executed, and its outcome is being persisted
- * Cleanup: the transaction has completed, and state used for processing it is being reclaimed
- */
- public enum Phase
- {
- None(false),
- PreAccept(false),
- Accept(true),
- Commit(true),
- Execute(false),
- Persist(false),
- Cleanup(false);
-
- public final boolean tieBreakWithBallot;
-
- Phase(boolean tieBreakWithBallot)
- {
- this.tieBreakWithBallot = tieBreakWithBallot;
- }
- }
-
- /**
- * A vector of various facets of knowledge about, or required for, processing a transaction.
- * Each property is treated independently, there is no precedence relationship between them.
- * Each property's values are however ordered with respect to each other.
- *
- * This information does not need to be consistent with
- */
- public static class Known
- {
- public static final Known Nothing = new Known(Maybe, DefinitionUnknown, ExecuteAtUnknown, DepsUnknown, Unknown);
- public static final Known DefinitionAndRoute = new Known(Full, DefinitionKnown, ExecuteAtUnknown, DepsUnknown, Unknown);
- public static final Known Apply = new Known(Full, DefinitionUnknown, ExecuteAtKnown, DepsKnown, Outcome.Apply);
- public static final Known Invalidated = new Known(Maybe, DefinitionUnknown, ExecuteAtUnknown, DepsUnknown, Outcome.Invalidated);
-
- public final KnownRoute route;
- public final Definition definition;
- public final KnownExecuteAt executeAt;
- public final KnownDeps deps;
- public final Outcome outcome;
-
- public Known(Known copy)
- {
- this.route = copy.route;
- this.definition = copy.definition;
- this.executeAt = copy.executeAt;
- this.deps = copy.deps;
- this.outcome = copy.outcome;
- }
-
- public Known(KnownRoute route, Definition definition, KnownExecuteAt executeAt, KnownDeps deps, Outcome outcome)
- {
- this.route = route;
- this.definition = definition;
- this.executeAt = executeAt;
- this.deps = deps;
- this.outcome = outcome;
- checkInvariants();
- }
-
- public Known atLeast(Known that)
- {
- Invariants.checkArgument(compatibleWith(that));
- KnownRoute maxRoute = route.atLeast(that.route);
- Definition maxDefinition = definition.atLeast(that.definition);
- KnownExecuteAt maxExecuteAt = executeAt.atLeast(that.executeAt);
- KnownDeps maxDeps = deps.atLeast(that.deps);
- Outcome maxOutcome = outcome.atLeast(that.outcome);
- return selectOrCreate(that, maxRoute, maxDefinition, maxExecuteAt, maxDeps, maxOutcome);
- }
-
- public Known min(Known that)
- {
- Invariants.checkArgument(compatibleWith(that));
- KnownRoute minRoute = min(route, that.route);
- Definition minDefinition = min(definition, that.definition);
- KnownExecuteAt minExecuteAt = min(executeAt, that.executeAt);
- KnownDeps minDeps = min(deps, that.deps);
- Outcome minOutcome = min(outcome, that.outcome);
- return selectOrCreate(that, minRoute, minDefinition, minExecuteAt, minDeps, minOutcome);
- }
-
- static <E extends Enum<E>> E min(E a, E b)
- {
- return a.compareTo(b) <= 0 ? a : b;
- }
-
- public Known reduce(Known that)
- {
- Invariants.checkArgument(compatibleWith(that));
- KnownRoute maxRoute = route.reduce(that.route);
- Definition minDefinition = definition.reduce(that.definition);
- KnownExecuteAt maxExecuteAt = executeAt.reduce(that.executeAt);
- KnownDeps minDeps = deps.reduce(that.deps);
- Outcome maxOutcome = outcome.reduce(that.outcome);
- return selectOrCreate(that, maxRoute, minDefinition, maxExecuteAt, minDeps, maxOutcome);
- }
-
- public Known validForAll()
- {
- KnownRoute maxRoute = route.validForAll();
- Definition minDefinition = definition.validForAll();
- KnownExecuteAt maxExecuteAt = executeAt.validForAll();
- KnownDeps minDeps = deps.validForAll();
- Outcome maxOutcome = outcome.validForAll();
- return selectOrCreate(maxRoute, minDefinition, maxExecuteAt, minDeps, maxOutcome);
- }
-
- boolean compatibleWith(Known that)
- {
- return executeAt.compatibleWith(that.executeAt)
- && deps.compatibleWith(that.deps)
- && outcome.compatibleWith(that.outcome);
- }
-
- @Nonnull
- private Known selectOrCreate(Known with, KnownRoute maxRoute, Definition maxDefinition, KnownExecuteAt maxExecuteAt, KnownDeps maxDeps, Outcome maxOutcome)
- {
- if (maxRoute == with.route && maxDefinition == with.definition && maxExecuteAt == with.executeAt && maxDeps == with.deps && maxOutcome == with.outcome)
- return with;
- return selectOrCreate(maxRoute, maxDefinition, maxExecuteAt, maxDeps, maxOutcome);
- }
-
- @Nonnull
- private Known selectOrCreate(KnownRoute maxRoute, Definition maxDefinition, KnownExecuteAt maxExecuteAt, KnownDeps maxDeps, Outcome maxOutcome)
- {
- if (maxRoute == route && maxDefinition == definition && maxExecuteAt == executeAt && maxDeps == deps && maxOutcome == outcome)
- return this;
- return new Known(maxRoute, maxDefinition, maxExecuteAt, maxDeps, maxOutcome);
- }
-
- public boolean isSatisfiedBy(Known that)
- {
- return this.definition.compareTo(that.definition) <= 0
- && this.executeAt.compareTo(that.executeAt) <= 0
- && this.deps.compareTo(that.deps) <= 0
- && this.outcome.isSatisfiedBy(that.outcome);
- }
-
- /**
- * The logical epoch on which the specified knowledge is best sought or sent.
- * i.e., if we include an outcome then the execution epoch
- */
- public LogicalEpoch epoch()
- {
- if (outcome.isOrWasApply())
- return LogicalEpoch.Execution;
-
- return LogicalEpoch.Coordination;
- }
-
- public long fetchEpoch(TxnId txnId, @Nullable Timestamp executeAt)
- {
- if (executeAt == null)
- return txnId.epoch();
-
- if (outcome.isOrWasApply() && !executeAt.equals(Timestamp.NONE))
- return executeAt.epoch();
-
- return txnId.epoch();
- }
-
- public Known with(Outcome newOutcome)
- {
- if (outcome == newOutcome)
- return this;
- return new Known(route, definition, executeAt, deps, newOutcome);
- }
-
- public Known with(KnownDeps newDeps)
- {
- if (deps == newDeps)
- return this;
- return new Known(route, definition, executeAt, newDeps, outcome);
- }
-
- /**
- * Convert this Known to one that represents the knowledge that can be propagated to another replica without
- * further information available to that replica, e.g. we cannot apply without also knowing the definition.
- */
- public Known propagates()
- {
- return propagatesSaveStatus().known;
- }
-
- public SaveStatus propagatesSaveStatus()
- {
- if (isInvalidated())
- return SaveStatus.Invalidated;
-
- if (route != Full)
- return SaveStatus.NotDefined;
-
- if (definition == DefinitionUnknown)
- {
- if (executeAt.isDecidedAndKnownToExecute())
- return SaveStatus.PreCommitted;
- return SaveStatus.NotDefined;
- }
-
- KnownExecuteAt executeAt = this.executeAt;
- if (!executeAt.isDecidedAndKnownToExecute())
- return SaveStatus.PreAccepted;
-
- // cannot propagate proposed deps; and cannot propagate known deps without executeAt
- KnownDeps deps = this.deps;
- if (!deps.hasDecidedDeps())
- return SaveStatus.PreCommittedWithDefinition;
-
- switch (outcome)
- {
- default: throw new AssertionError("Unhandled outcome: " + outcome);
- case Unknown:
- case WasApply:
- case Erased:
- return SaveStatus.Stable;
-
- case Apply:
- return SaveStatus.PreApplied;
- }
- }
-
- public Status propagatesStatus()
- {
- switch (outcome)
- {
- default: throw new AssertionError();
- case Invalidated:
- return Status.Invalidated;
-
- case Apply:
- case WasApply:
- if (executeAt.isDecidedAndKnownToExecute() && definition.isKnown() && deps.hasDecidedDeps())
- return PreApplied;
-
- case Erased:
- case Unknown:
- if (executeAt.isDecidedAndKnownToExecute() && definition.isKnown() && deps.hasDecidedDeps())
- return Stable;
-
- if (executeAt.isDecidedAndKnownToExecute() && hasFullRoute())
- return PreCommitted;
-
- if (definition.isKnown())
- return PreAccepted;
-
- if (deps.hasDecidedDeps())
- throw new IllegalStateException();
- }
-
- return outcome == WasApply ? Truncated : NotDefined;
- }
-
- public boolean isDefinitionKnown()
- {
- return definition.isKnown();
- }
-
- // TODO (expected): this isn't exactly correct, though it is probably fine:
- // outcome.isDecided includes Erased and Invalidated; Invalidated is definitely fine because we only use this to infer invalidation
- // whereas for Erased, we are inaccurate in the safer direction (saying it is Decided to avoid inferring Invalidated when we don't know),
- // but it would be good to tidy this up - either by clarifying the semantics or the naming
- public boolean hasDefinitionBeenKnown()
- {
- return definition.isKnown() || outcome.isDecided();
- }
-
- /**
- * The command may have an incomplete route when this is false
- */
- public boolean hasFullRoute()
- {
- return definition.isKnown() || outcome.isOrWasApply();
- }
-
- public boolean isTruncated()
- {
- switch (outcome)
- {
- default: throw new AssertionError("Unhandled outcome: " + outcome);
- case Invalidated:
- case Unknown:
- return false;
- case Apply:
- // since Apply is universal, we can
- return deps == DepsErased;
- case Erased:
- case WasApply:
- return true;
- }
- }
-
- public boolean canProposeInvalidation()
- {
- return deps.canProposeInvalidation() && executeAt.canProposeInvalidation() && outcome.canProposeInvalidation();
- }
-
- public Known subtract(Known subtract)
- {
- if (!subtract.isSatisfiedBy(this))
- return Known.Nothing;
-
- Definition newDefinition = subtract.definition.compareTo(definition) >= 0 ? DefinitionUnknown : definition;
- KnownExecuteAt newExecuteAt = subtract.executeAt.compareTo(executeAt) >= 0 ? ExecuteAtUnknown : executeAt;
- KnownDeps newDeps = subtract.deps.compareTo(deps) >= 0 ? DepsUnknown : deps;
- Outcome newOutcome = outcome.subtract(subtract.outcome);
- return new Known(route, newDefinition, newExecuteAt, newDeps, newOutcome);
- }
-
- public boolean isDecided()
- {
- return executeAt.isDecided() || outcome.isDecided();
- }
-
- public boolean isDecidedToExecute()
- {
- return executeAt.isDecidedAndKnownToExecute() || outcome.isOrWasApply();
- }
-
- public String toString()
- {
- return Stream.of(definition.isKnown() ? "Definition" : null,
- executeAt.isDecidedAndKnownToExecute() ? "ExecuteAt" : null,
- deps.hasDecidedDeps() ? "Deps" : null,
- outcome.isDecided() ? outcome.toString() : null
- ).filter(Objects::nonNull).collect(Collectors.joining(",", "[", "]"));
- }
-
- @Override
- public boolean equals(Object o)
- {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
- Known that = (Known) o;
- return route == that.route && definition == that.definition && executeAt == that.executeAt && deps == that.deps && outcome == that.outcome;
- }
-
- @Override
- public int hashCode()
- {
- throw new UnsupportedOperationException();
- }
-
- public boolean hasDefinition()
- {
- return definition.isKnown();
- }
-
- public boolean hasDecidedDeps()
- {
- return deps.hasDecidedDeps();
- }
-
- public boolean isInvalidated()
- {
- return outcome.isInvalidated();
- }
-
- public void checkInvariants()
- {
- if (outcome.isInvalidated()) Invariants.checkState( deps != DepsKnown && executeAt != ExecuteAtKnown);
- else if (outcome.isOrWasApply()) Invariants.checkState(deps != NoDeps && executeAt != NoExecuteAt);
- Invariants.checkState(!isDefinitionKnown() || hasFullRoute());
- }
- }
-
- public enum KnownRoute
- {
- /**
- * A route may or may not be known, but it may not cover (or even intersect) this shard.
- * The route should be relied upon only if it is a FullRoute.
- */
- Maybe,
-
- /**
- * A route is known that covers the ranges this shard participates in.
- * Note that if the status is less than Committed, this may not be the final set of owned ranges,
- * and the route may not cover whatever this is decided as.
- *
- * This status primarily exists to communicate semantically to the reader.
- */
- Covering,
-
- /**
- * The full route is known. <i>Generally</i> this coincides with knowing the Definition.
- */
- Full
- ;
-
- public KnownRoute reduce(KnownRoute that)
- {
- if (this == that) return this;
- if (this == Full || that == Full) return Full;
- return Maybe;
- }
-
- public KnownRoute validForAll()
- {
- return this == Covering ? Maybe : this;
- }
-
- public KnownRoute atLeast(KnownRoute that)
- {
- return this.compareTo(that) >= 0 ? this : that;
- }
- }
-
- public enum KnownExecuteAt
- {
- /**
- * No decision is known to have been reached. If executeAt is not null, it represents either when
- * the transaction was witnessed, or some earlier ExecuteAtProposed that was invalidated by AcceptedInvalidate
- */
- ExecuteAtUnknown,
-
- /**
- * A decision to execute the transaction is known to have been proposed, and the associated executeAt timestamp
- */
- ExecuteAtProposed,
-
- /**
- * A decision to execute or invalidate the transaction is known to have been reached and since been cleaned up
- */
- ExecuteAtErased,
-
- /**
- * A decision to execute the transaction is known to have been reached, and the associated executeAt timestamp
- */
- ExecuteAtKnown,
-
- /**
- * A decision to invalidate the transaction is known to have been reached
- */
- NoExecuteAt
- ;
-
- /**
- * Is known to have agreed to execute or not; but the decision is not known (maybe erased)
- */
- public boolean isDecided()
- {
- return compareTo(ExecuteAtErased) >= 0;
- }
-
- /**
- * Is known to have agreed to execute or not; but the decision is not known (maybe erased)
- */
- public boolean hasDecision()
- {
- return compareTo(ExecuteAtKnown) >= 0;
- }
-
- /**
- * Is known to execute, and when.
- */
- public boolean isDecidedAndKnownToExecute()
- {
- return this == ExecuteAtKnown;
- }
-
- public KnownExecuteAt atLeast(KnownExecuteAt that)
- {
- return compareTo(that) >= 0 ? this : that;
- }
-
- public KnownExecuteAt reduce(KnownExecuteAt that)
- {
- return atLeast(that);
- }
-
- public KnownExecuteAt validForAll()
- {
- return compareTo(ExecuteAtErased) <= 0 ? ExecuteAtUnknown : this;
- }
-
- public boolean canProposeInvalidation()
- {
- return this == ExecuteAtUnknown;
- }
-
- public boolean compatibleWith(KnownExecuteAt that)
- {
- if (this == that) return true;
- int c = compareTo(that);
- KnownExecuteAt max = c >= 0 ? this : that;
- KnownExecuteAt min = c <= 0 ? this : that;
- return max != NoExecuteAt || min != ExecuteAtKnown;
- }
- }
-
- public enum KnownDeps
- {
- /**
- * No decision is known to have been reached
- */
- DepsUnknown(PreAccept),
-
- /**
- * A decision to execute the transaction at a given timestamp is known to have been proposed,
- * and the associated dependencies for the shard(s) in question are known for the coordination epoch (txnId.epoch) only.
- *
- * Proposed means Accepted at some replica, but not necessarily a majority and so not committed
- */
- DepsProposed(Accept),
-
- /**
- * A decision to execute the transaction at a given timestamp with certain dependencies is known to have been proposed,
- * and some associated dependencies for the shard(s) in question have been committed.
- *
- * However, the dependencies are only known to committed at a replica, not necessarily a majority, i.e. not stable
- */
- DepsCommitted(Commit),
-
- /**
- * A decision to execute or invalidate the transaction is known to have been reached, and any associated
- * dependencies for the shard(s) in question have been cleaned up.
- */
- DepsErased(Cleanup),
-
- /**
- * A decision to execute the transaction is known to have been reached, and the associated dependencies
- * for the shard(s) in question are known for the coordination and execution epochs, and are stable.
- */
- DepsKnown(Execute),
-
- /**
- * A decision to invalidate the transaction is known to have been reached
- */
- NoDeps(Persist);
-
- public final Phase phase;
-
- KnownDeps(Phase phase)
- {
- this.phase = phase;
- }
-
- public boolean hasProposedDeps()
- {
- return this == DepsProposed;
- }
-
- public boolean hasDecidedDeps()
- {
- return this == DepsKnown;
- }
-
- public boolean canProposeInvalidation()
- {
- return this == DepsUnknown;
- }
-
- public boolean hasProposedOrDecidedDeps()
- {
- switch (this)
- {
- default: throw new AssertionError("Unhandled KnownDeps: " + this);
- case DepsCommitted:
- case DepsProposed:
- case DepsKnown:
- return true;
- case NoDeps:
- case DepsErased:
- case DepsUnknown:
- return false;
- }
- }
-
- public boolean hasCommittedOrDecidedDeps()
- {
- return this == DepsCommitted || this == DepsKnown;
- }
-
- public KnownDeps atLeast(KnownDeps that)
- {
- return compareTo(that) >= 0 ? this : that;
- }
-
- public KnownDeps reduce(KnownDeps that)
- {
- return compareTo(that) <= 0 ? this : that;
- }
-
- public KnownDeps validForAll()
- {
- return this == NoDeps ? NoDeps : DepsUnknown;
- }
-
- public boolean compatibleWith(KnownDeps that)
- {
- if (this == that) return true;
- int c = compareTo(that);
- KnownDeps max = c >= 0 ? this : that;
- KnownDeps min = c <= 0 ? this : that;
- return max != NoDeps || (min != DepsCommitted && min != DepsKnown);
- }
- }
-
- /**
- * Whether the transaction's definition is known.
- */
- public enum Definition
- {
- /**
- * The definition is not known
- */
- DefinitionUnknown,
-
- /**
- * The definition was perhaps known previously, but has since been erased
- */
- DefinitionErased,
-
- /**
- * The definition is irrelevant, as the transaction has been invalidated and may be treated as a no-op
- */
- NoOp,
-
- /**
- * The definition is known
- *
- * TODO (expected, clarity): distinguish between known for coordination epoch and known for commit/execute
- */
- DefinitionKnown;
-
- public boolean isKnown()
- {
- return this == DefinitionKnown;
- }
-
- public boolean isOrWasKnown()
- {
- return this == DefinitionKnown;
- }
-
- public Definition atLeast(Definition that)
- {
- return compareTo(that) >= 0 ? this : that;
- }
-
- // combine info about two shards into a composite representation
- public Definition reduce(Definition that)
- {
- return compareTo(that) <= 0 ? this : that;
- }
-
- public Definition validForAll()
- {
... 13827 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org