You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by be...@apache.org on 2023/08/15 15:51:30 UTC
[cassandra-accord] branch trunk updated: State Eviction (#50)
This is an automated email from the ASF dual-hosted git repository.
benedict pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git
The following commit(s) were added to refs/heads/trunk by this push:
new 7c15f3a6 State Eviction (#50)
7c15f3a6 is described below
commit 7c15f3a6203939bc6cb398e538df1ca3557cbe03
Author: Benedict Elliott Smith <be...@apache.org>
AuthorDate: Tue Aug 15 16:51:22 2023 +0100
State Eviction (#50)
Permit the state machine to erase transactions that are known to be applied across the cluster.
---------
Co-authored-by: dcapwell <dc...@gmail.com>
Co-authored-by: Aleksey Yeschenko <al...@apache.org>
Co-authored-by: Ariel Weisberg <aw...@apple.com>
---
.../main/java/accord/api/ConfigurationService.java | 30 +-
.../src/main/java/accord/api/DataStore.java | 9 +-
.../src/main/java/accord/api/ProgressLog.java | 51 +-
accord-core/src/main/java/accord/api/Read.java | 3 -
.../src/main/java/accord/api/Scheduler.java | 22 +
.../src/main/java/accord/coordinate/CheckOn.java | 269 -----
.../main/java/accord/coordinate/CheckShards.java | 49 +-
.../coordinate/CoordinateGloballyDurable.java | 86 ++
.../java/accord/coordinate/CoordinateNoOp.java | 12 +-
.../accord/coordinate/CoordinatePreAccept.java | 48 +-
.../accord/coordinate/CoordinateShardDurable.java | 114 ++
.../accord/coordinate/CoordinateSyncPoint.java | 54 +-
.../accord/coordinate/CoordinateTransaction.java | 1 +
.../java/accord/coordinate/CoordinationFailed.java | 3 +-
.../src/main/java/accord/coordinate/Execute.java | 25 +-
.../java/accord/coordinate/FetchCoordinator.java | 14 +-
.../src/main/java/accord/coordinate/FetchData.java | 448 +++++++-
.../src/main/java/accord/coordinate/FindRoute.java | 11 +-
.../{FindHomeKey.java => FindSomeRoute.java} | 39 +-
.../src/main/java/accord/coordinate/Infer.java | 207 ++++
.../java/accord/coordinate/InformHomeOfTxn.java | 16 +-
.../main/java/accord/coordinate/Invalidate.java | 124 ++-
.../main/java/accord/coordinate/MaybeRecover.java | 64 +-
.../src/main/java/accord/coordinate/Persist.java | 55 +-
.../src/main/java/accord/coordinate/Propose.java | 27 +-
.../java/accord/coordinate/ProposeAndExecute.java | 1 +
.../java/accord/coordinate/ReadCoordinator.java | 49 +-
.../src/main/java/accord/coordinate/Recover.java | 54 +-
.../java/accord/coordinate/RecoverWithRoute.java | 91 +-
...rWithHomeKey.java => RecoverWithSomeRoute.java} | 29 +-
.../Truncated.java} | 22 +-
.../coordinate/tracking/AbstractTracker.java | 2 +
.../accord/coordinate/tracking/AppliedTracker.java | 101 ++
.../coordinate/tracking/FastPathTracker.java | 5 +
.../coordinate/tracking/InvalidationTracker.java | 32 +-
.../accord/impl/AbstractConfigurationService.java | 19 +-
.../java/accord/impl/AbstractFetchCoordinator.java | 86 +-
.../java/accord/impl/AbstractSafeCommandStore.java | 44 +-
.../main/java/accord/impl/CommandTimeseries.java | 21 +-
.../src/main/java/accord/impl/CommandsForKey.java | 82 +-
.../impl/CoordinateDurabilityScheduling.java | 356 +++++++
...morySafeCommand.java => ErasedSafeCommand.java} | 61 +-
.../java/accord/impl/InMemoryCommandStore.java | 254 ++---
.../java/accord/impl/InMemoryCommandStores.java | 1 -
.../main/java/accord/impl/InMemorySafeCommand.java | 6 +
.../main/java/accord/impl/SafeCommandsForKey.java | 23 +-
.../main/java/accord/impl/SimpleProgressLog.java | 467 +++------
.../src/main/java/accord/local/Bootstrap.java | 22 +-
.../src/main/java/accord/local/Command.java | 764 ++++++++++----
.../src/main/java/accord/local/CommandStore.java | 321 ++++--
.../src/main/java/accord/local/CommandStores.java | 229 ++--
.../src/main/java/accord/local/Commands.java | 1093 +++++++++++---------
.../main/java/accord/local/CommonAttributes.java | 43 +-
.../src/main/java/accord/local/DurableBefore.java | 236 +++++
.../src/main/java/accord/local/Listeners.java | 6 +-
.../src/main/java/accord/local/LocalBarrier.java | 120 ---
accord-core/src/main/java/accord/local/Node.java | 153 ++-
.../main/java/accord/local/NodeTimeService.java | 24 +
.../main/java/accord/local/RedundantBefore.java | 243 +++++
.../main/java/accord/local/RedundantStatus.java | 87 ++
.../src/main/java/accord/local/SafeCommand.java | 35 +-
.../main/java/accord/local/SafeCommandStore.java | 196 +++-
.../src/main/java/accord/local/SaveStatus.java | 214 +++-
.../main/java/accord/local/ShardDistributor.java | 53 +-
accord-core/src/main/java/accord/local/Status.java | 196 +++-
.../src/main/java/accord/messages/Accept.java | 21 +-
.../src/main/java/accord/messages/Apply.java | 105 +-
.../java/accord/messages/BeginInvalidation.java | 31 +-
.../main/java/accord/messages/BeginRecovery.java | 24 +-
.../src/main/java/accord/messages/CheckStatus.java | 286 +++--
.../src/main/java/accord/messages/Commit.java | 45 +-
.../src/main/java/accord/messages/GetDeps.java | 3 +-
.../main/java/accord/messages/InformDurable.java | 9 +-
.../java/accord/messages/InformHomeDurable.java | 23 +-
.../main/java/accord/messages/InformOfTxnId.java | 22 +-
.../src/main/java/accord/messages/MessageType.java | 59 +-
.../src/main/java/accord/messages/PreAccept.java | 34 +-
.../java/accord/messages/QueryDurableBefore.java | 87 ++
.../src/main/java/accord/messages/ReadData.java | 20 +-
.../src/main/java/accord/messages/ReadTxnData.java | 57 +-
.../java/accord/messages/SetGloballyDurable.java | 75 ++
.../main/java/accord/messages/SetShardDurable.java | 80 ++
.../src/main/java/accord/messages/SimpleReply.java | 5 +
.../src/main/java/accord/messages/TxnRequest.java | 8 +-
.../main/java/accord/messages/WaitAndReadData.java | 42 +-
.../main/java/accord/messages/WaitOnCommit.java | 26 +-
.../{ReadTxnData.java => WaitUntilApplied.java} | 193 ++--
.../main/java/accord/primitives/AbstractKeys.java | 91 +-
.../java/accord/primitives/AbstractRanges.java | 242 ++++-
.../accord/primitives/AbstractUnseekableKeys.java | 10 +-
.../src/main/java/accord/primitives/Ballot.java | 1 +
.../src/main/java/accord/primitives/Deps.java | 34 +-
.../EpochSupplier.java} | 9 +-
.../main/java/accord/primitives/FullKeyRoute.java | 12 +-
.../java/accord/primitives/FullRangeRoute.java | 10 +-
.../src/main/java/accord/primitives/FullRoute.java | 2 +-
.../src/main/java/accord/primitives/KeyDeps.java | 74 +-
.../src/main/java/accord/primitives/KeyRoute.java | 92 +-
.../src/main/java/accord/primitives/Keys.java | 5 +-
.../main/java/accord/primitives/PartialDeps.java | 7 +-
.../java/accord/primitives/PartialKeyRoute.java | 34 +-
.../java/accord/primitives/PartialRangeRoute.java | 45 +-
.../main/java/accord/primitives/PartialRoute.java | 8 -
.../main/java/accord/primitives/PartialTxn.java | 15 +-
.../main/java/accord/primitives/Participants.java | 44 +
.../main/java/accord/primitives/ProgressToken.java | 8 +-
.../src/main/java/accord/primitives/Range.java | 39 +-
.../src/main/java/accord/primitives/RangeDeps.java | 154 ++-
.../main/java/accord/primitives/RangeRoute.java | 107 +-
.../src/main/java/accord/primitives/Ranges.java | 143 +--
.../main/java/accord/primitives/RoutableKey.java | 49 +-
.../src/main/java/accord/primitives/Routables.java | 123 +--
.../src/main/java/accord/primitives/Route.java | 43 +-
.../main/java/accord/primitives/RoutingKeys.java | 24 +-
.../src/main/java/accord/primitives/Seekables.java | 4 +-
.../src/main/java/accord/primitives/SyncPoint.java | 34 +-
.../src/main/java/accord/primitives/Timestamp.java | 24 +-
.../src/main/java/accord/primitives/Txn.java | 10 +
.../src/main/java/accord/primitives/TxnId.java | 20 +
.../main/java/accord/primitives/Unseekables.java | 16 +-
.../src/main/java/accord/topology/Topology.java | 72 +-
.../main/java/accord/topology/TopologyManager.java | 399 ++++---
.../BiIndexedTriConsumer.java} | 12 +-
.../accord/utils/DeterministicIdentitySet.java | 133 +--
...isticIdentitySet.java => DeterministicSet.java} | 48 +-
.../ImmutableBitSet.java} | 49 +-
.../src/main/java/accord/utils/IndexedFold.java | 1 +
.../java/accord/utils/IndexedQuadConsumer.java} | 17 +-
.../IndexedRangeQuadFunction.java} | 12 +-
.../src/main/java/accord/utils/Invariants.java | 20 +
.../SimpleReply.java => utils/QuadFunction.java} | 12 +-
.../src/main/java/accord/utils/RandomSource.java | 87 ++
.../java/accord/utils/ReducingIntervalMap.java | 274 ++---
.../main/java/accord/utils/ReducingRangeMap.java | 179 ++--
.../main/java/accord/utils/RelationMultiMap.java | 68 ++
.../java/accord/utils/SearchableRangeList.java | 17 +-
.../accord/utils/SearchableRangeListBuilder.java | 5 +-
.../src/main/java/accord/utils/SimpleBitSet.java | 404 ++++++++
.../src/main/java/accord/utils/SortedArrays.java | 160 ++-
.../java/accord/utils/SortedList.java} | 12 +-
.../src/main/java/accord/utils/Timestamped.java | 28 +-
accord-core/src/main/java/accord/utils/Utils.java | 14 +-
.../accord/utils/async/AsyncChainCombiner.java | 117 +--
.../main/java/accord/utils/async/AsyncChains.java | 188 +++-
.../main/java/accord/utils/async/AsyncResults.java | 29 +-
accord-core/src/test/java/accord/Utils.java | 23 +-
.../src/test/java/accord/burn/BurnTest.java | 23 +-
.../accord/burn/BurnTestConfigurationService.java | 15 +
.../src/test/java/accord/burn/TopologyUpdates.java | 24 +-
.../accord/coordinate/PreAcceptTrackerTest.java | 3 +-
.../test/java/accord/coordinate/RecoverTest.java | 1 -
.../java/accord/coordinate/TopologyChangeTest.java | 6 +-
.../tracking/InvalidationTrackerReconciler.java | 12 +-
.../coordinate/tracking/QuorumTrackerTest.java | 4 +-
.../coordinate/tracking/ReadTrackerTest.java | 5 +-
.../coordinate/tracking/TrackerReconciler.java | 33 +-
.../impl/AbstractConfigurationServiceTest.java | 21 +
.../src/test/java/accord/impl/IntHashKey.java | 14 +-
accord-core/src/test/java/accord/impl/IntKey.java | 14 +-
.../src/test/java/accord/impl/TestAgent.java | 34 +-
.../src/test/java/accord/impl/TopologyUtils.java | 7 +
.../src/test/java/accord/impl/basic/Cluster.java | 53 +-
.../accord/impl/basic/DelayedCommandStores.java | 20 +-
.../src/test/java/accord/impl/basic/NodeSink.java | 2 +-
.../test/java/accord/impl/basic/PendingQueue.java | 2 +
.../accord/impl/basic/PropagatingPendingQueue.java | 12 +
.../java/accord/impl/basic/RandomDelayQueue.java | 27 +-
.../basic/SimulatedDelayedExecutorService.java | 145 ++-
.../accord/impl/basic/TaskExecutorService.java | 2 +-
.../src/test/java/accord/impl/list/ListAgent.java | 5 +-
.../src/test/java/accord/impl/list/ListData.java | 2 +-
.../test/java/accord/impl/list/ListRequest.java | 29 +-
.../src/test/java/accord/impl/list/ListResult.java | 2 +-
.../src/test/java/accord/impl/list/ListStore.java | 25 +-
.../src/test/java/accord/impl/list/ListWrite.java | 2 +-
.../src/test/java/accord/impl/mock/EpochSync.java | 210 ----
.../test/java/accord/impl/mock/MockCluster.java | 69 +-
.../accord/impl/mock/MockConfigurationService.java | 11 +
.../src/test/java/accord/impl/mock/MockStore.java | 3 -
.../test/java/accord/local/CheckedCommands.java | 14 +-
.../java/accord/local/ImmutableCommandTest.java | 75 +-
.../test/java/accord/messages/PreAcceptTest.java | 10 +-
.../test/java/accord/messages/ReadDataTest.java | 16 +-
.../java/accord/messages/TxnRequestScopeTest.java | 11 +-
.../java/accord/primitives/AbstractRangesTest.java | 83 ++
.../test/java/accord/primitives/KeyDepsTest.java | 16 +-
.../java/accord/topology/TopologyManagerTest.java | 70 +-
.../java/accord/topology/TopologyRandomizer.java | 50 +-
.../test/java/accord/topology/TopologyTest.java | 2 +-
.../src/test/java/accord/utils/AccordGens.java | 26 +-
.../java/accord/utils/DeterministicSetTest.java | 87 ++
accord-core/src/test/java/accord/utils/Gen.java | 14 +-
accord-core/src/test/java/accord/utils/Gens.java | 107 ++
.../src/test/java/accord/utils/MessageTask.java | 2 +-
.../src/test/java/accord/utils/Property.java | 14 +-
.../src/test/java/accord/utils/RangesTest.java | 14 +-
.../java/accord/utils/ReducingRangeMapTest.java | 251 ++---
.../test/java/accord/utils/SimpleBitSetTest.java | 190 ++++
.../test/java/accord/utils/SortedArraysTest.java | 36 +-
.../java/accord/utils/async/AsyncChainsTest.java | 158 ++-
.../accord/verify/LinearizabilityVerifier.java | 1 +
.../src/main/java/accord/maelstrom/Cluster.java | 21 +-
.../main/java/accord/maelstrom/MaelstromAgent.java | 2 -
.../main/java/accord/maelstrom/MaelstromKey.java | 12 +
.../main/java/accord/maelstrom/MaelstromRead.java | 4 -
.../java/accord/maelstrom/MaelstromRequest.java | 1 +
.../src/main/java/accord/maelstrom/Main.java | 18 +-
.../java/accord/maelstrom/SimpleConfigService.java | 11 +-
.../src/main/groovy/accord.java-conventions.gradle | 3 +
209 files changed, 9981 insertions(+), 4965 deletions(-)
diff --git a/accord-core/src/main/java/accord/api/ConfigurationService.java b/accord-core/src/main/java/accord/api/ConfigurationService.java
index 177094be..4e6943fc 100644
--- a/accord-core/src/main/java/accord/api/ConfigurationService.java
+++ b/accord-core/src/main/java/accord/api/ConfigurationService.java
@@ -21,6 +21,7 @@ package accord.api;
import javax.annotation.Nullable;
import accord.local.Node;
+import accord.primitives.Ranges;
import accord.topology.Topology;
import accord.utils.async.AsyncResult;
import accord.utils.async.AsyncResults;
@@ -118,6 +119,9 @@ public interface ConfigurationService
/**
* Called when accord data associated with a superseded epoch has been sync'd from previous replicas.
* This should be invoked on each replica once EpochReady.coordination has returned on a replica.
+ *
+ * Once a quorum of these notifications have been received, no new TxnId may be executed in this epoch
+ * (though this is not a transitive property; earlier epochs may yet agree to execute TxnId if they have not been sync'd)
*/
void onRemoteSyncComplete(Node.Id node, long epoch);
@@ -126,6 +130,19 @@ public interface ConfigurationService
* the given epoch
*/
void truncateTopologyUntil(long epoch);
+
+ /**
+ * Called when no new TxnId may be agreed with an epoch less than or equal to the provided one.
+ * This means future epochs are now aware of all TxnId with this epoch or earlier that may be executed
+ * on this range.
+ */
+ void onEpochClosed(Ranges ranges, long epoch);
+
+ /**
+ * Called when all TxnId with an epoch equal to or before this that interact with this range have been executed,
+ * in whatever epoch they execute in. Once the whole range is covered this epoch is redundant, and may be cleaned up.
+ */
+ void onEpochRedundant(Ranges ranges, long epoch);
}
void registerListener(Listener listener);
@@ -151,19 +168,14 @@ public interface ConfigurationService
*/
void fetchTopologyForEpoch(long epoch);
- /**
- * Alert the configuration service of epochs it may not be aware of. This is called called for every TxnRequest
- * received by Accord, so implementations should be lightweight, and avoid blocking or heavy computation.
- */
- default void reportEpoch(long epoch)
- {
- fetchTopologyForEpoch(epoch);
- }
-
/**
* Called after this node learns of an epoch as part of the {@code Listener#onTopologyUpdate} call.
* On invocation the system is not necessarily ready to process the epoch, and the BootstrapReady parameter
* provides indications of when the bootstrap has completed various phases of setup.
*/
void acknowledgeEpoch(EpochReady ready);
+
+ void reportEpochClosed(Ranges ranges, long epoch);
+
+ void reportEpochRedundant(Ranges ranges, long epoch);
}
diff --git a/accord-core/src/main/java/accord/api/DataStore.java b/accord-core/src/main/java/accord/api/DataStore.java
index eaf84aca..d71c742b 100644
--- a/accord-core/src/main/java/accord/api/DataStore.java
+++ b/accord-core/src/main/java/accord/api/DataStore.java
@@ -44,13 +44,8 @@ public interface DataStore
* To be invoked by implementation once we hear back from the replica that it
* has taken a snapshot to process.
*
- * Required parameter unbootstrapped provides those TxnId that were dependencies
- * of the SyncPoint but that were assigned a later execution time, and therefore
- * may be our responsibility, and that we must therefore both apply locally and
- * also retain dependencies to.
- *
- * Optional parameter maxApplied can provide the maximum timestamp of any applied
- * transaction in the data being bootstrapped from this source. If not applied
+ * Optional parameter maxApplied can supply the maximum timestamp of any applied
+ * transaction in the data being bootstrapped from this source. If not provided
* the store will coordinate a cheap global transaction to calculate a lower bound.
*
* Note: if the membership has entirely changed, or all other replicas are still themselves
diff --git a/accord-core/src/main/java/accord/api/ProgressLog.java b/accord-core/src/main/java/accord/api/ProgressLog.java
index 2ec45b78..4ede480f 100644
--- a/accord-core/src/main/java/accord/api/ProgressLog.java
+++ b/accord-core/src/main/java/accord/api/ProgressLog.java
@@ -18,16 +18,15 @@
package accord.api;
-import java.util.Set;
-
import javax.annotation.Nullable;
import accord.coordinate.InformHomeOfTxn;
import accord.local.Command;
import accord.local.CommandStore;
-import accord.local.Node.Id;
-import accord.local.Status.Known;
-import accord.primitives.Unseekables;
+import accord.local.SafeCommand;
+import accord.local.SaveStatus.LocalExecution;
+import accord.primitives.Participants;
+import accord.primitives.Route;
import accord.primitives.TxnId;
/**
@@ -96,7 +95,7 @@ public interface ProgressLog
* A home shard should monitor this transaction for global progress.
* A non-home shard should not receive this message.
*/
- void unwitnessed(TxnId txnId, RoutingKey homeKey, ProgressShard shard);
+ void unwitnessed(TxnId txnId, ProgressShard shard);
/**
* Has been pre-accepted.
@@ -129,7 +128,7 @@ public interface ProgressLog
* A home shard should monitor this transaction for global progress.
* A non-home shard can safely ignore this transaction, as it has been witnessed by a majority of the home shard.
*/
- void readyToExecute(Command command, ProgressShard shard);
+ void readyToExecute(Command command);
/**
* The transaction's outcome has been durably recorded (but not necessarily applied) locally.
@@ -142,18 +141,6 @@ public interface ProgressLog
*/
void executed(Command command, ProgressShard shard);
- /**
- * The transaction has been durably invalidated
- */
- void invalidated(Command command, ProgressShard shard);
-
- /**
- * The transaction's outcome has been durably recorded (but not necessarily applied) locally at all shards.
- *
- * This is only invoked on the home shard, once all local shards have successfully applied.
- */
- void durableLocal(TxnId txnId);
-
/**
* The transaction's outcome has been durably recorded (but not necessarily applied) at a quorum of all shards,
* including at least those node's ids that are provided.
@@ -164,15 +151,7 @@ public interface ProgressLog
* Otherwise, this transaction no longer needs to be monitored, but implementations may wish to ensure that
* the result is propagated to every live replica.
*/
- void durable(Command command, @Nullable Set<Id> persistedOn);
-
- /**
- * The transaction's outcome has been durably recorded (but not necessarily applied) at a quorum of all shards.
- *
- * If this replica has not witnessed the outcome of the transaction, it should poll a majority of each shard
- * for its outcome, using the provided route (if any).
- */
- void durable(TxnId txnId, @Nullable Unseekables<?, ?> unseekables, ProgressShard shard);
+ void durable(Command command);
/**
* The parameter is a command that some other command's execution is most proximally blocked by.
@@ -190,9 +169,17 @@ public interface ProgressLog
* <p>
* In all other scenarios, the implementation is free to choose its course of action.
*
- * @param blockedBy is the transaction id that is blocking progress
- * @param blockedUntil either Committed or Executed; the state we are waiting for
- * @param blockedOn the keys we should report any progress updates to
+ * Either blockedOnRoute or blockedOnParticipants should be non-null.
+ *
+ * @param blockedBy is the transaction id that is blocking progress
+ * @param blockedUntil either Committed or Executed; the state we are waiting for
+ * @param blockedOnRoute the route (if any) we are blocked on execution for
+ * @param blockedOnParticipants the participating keys on which we are blocked for execution
+ */
+ void waiting(SafeCommand blockedBy, LocalExecution blockedUntil, @Nullable Route<?> blockedOnRoute, @Nullable Participants<?> blockedOnParticipants);
+
+ /**
+ * We have finished processing this transaction; ensure its state is cleared
*/
- void waiting(TxnId blockedBy, Known blockedUntil, Unseekables<?, ?> blockedOn);
+ void clear(TxnId txnId);
}
diff --git a/accord-core/src/main/java/accord/api/Read.java b/accord-core/src/main/java/accord/api/Read.java
index c2952263..b9eaf2a1 100644
--- a/accord-core/src/main/java/accord/api/Read.java
+++ b/accord-core/src/main/java/accord/api/Read.java
@@ -20,9 +20,6 @@ package accord.api;
import accord.local.SafeCommandStore;
import accord.primitives.*;
-import accord.primitives.Ranges;
-import accord.primitives.Timestamp;
-import accord.primitives.Txn;
import accord.utils.async.AsyncChain;
diff --git a/accord-core/src/main/java/accord/api/Scheduler.java b/accord-core/src/main/java/accord/api/Scheduler.java
index a1bf2409..34ab7f0c 100644
--- a/accord-core/src/main/java/accord/api/Scheduler.java
+++ b/accord-core/src/main/java/accord/api/Scheduler.java
@@ -25,6 +25,28 @@ import java.util.concurrent.TimeUnit;
*/
public interface Scheduler
{
+ Scheduled CANCELLED = () -> {};
+ Scheduler NEVER_RUN_SCHEDULED = new Scheduler()
+ {
+ @Override
+ public Scheduled recurring(Runnable run, long delay, TimeUnit units)
+ {
+ return CANCELLED;
+ }
+
+ @Override
+ public Scheduled once(Runnable run, long delay, TimeUnit units)
+ {
+ return CANCELLED;
+ }
+
+ @Override
+ public void now(Runnable run)
+ {
+ run.run();
+ }
+ };
+
interface Scheduled
{
void cancel();
diff --git a/accord-core/src/main/java/accord/coordinate/CheckOn.java b/accord-core/src/main/java/accord/coordinate/CheckOn.java
deleted file mode 100644
index cea8b129..00000000
--- a/accord-core/src/main/java/accord/coordinate/CheckOn.java
+++ /dev/null
@@ -1,269 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package accord.coordinate;
-
-import java.util.Collection;
-import java.util.function.BiConsumer;
-
-import accord.local.*;
-import accord.local.Commands.ApplyOutcome;
-import accord.local.Commands.CommitOutcome;
-import accord.primitives.*;
-import accord.utils.Invariants;
-import accord.utils.MapReduceConsume;
-
-import accord.api.RoutingKey;
-import accord.local.Node.Id;
-import accord.messages.CheckStatus.CheckStatusOk;
-import accord.messages.CheckStatus.CheckStatusOkFull;
-import accord.messages.CheckStatus.IncludeInfo;
-
-import java.util.Collections;
-
-import static accord.local.PreLoadContext.contextFor;
-import static accord.local.SaveStatus.NotWitnessed;
-import static accord.local.Status.*;
-import static accord.primitives.Routables.Slice.Minimal;
-
-/**
- * Check on the status of a transaction. Returns early once enough information has been achieved to meet the requested
- * status for the requested {@code route}.
- *
- * If a command is durable (i.e. executed on a majority on all shards) this is sufficient to replicate the command locally.
- */
-public class CheckOn extends CheckShards
-{
- final BiConsumer<? super CheckStatusOkFull, Throwable> callback;
- /**
- * The epoch until which we want to persist any response for locally
- */
- final Known sufficient;
- final long untilLocalEpoch;
- final Route<?> route;
-
- CheckOn(Node node, Known sufficient, TxnId txnId, Route<?> route, long srcEpoch, long untilLocalEpoch, BiConsumer<? super CheckStatusOkFull, Throwable> callback)
- {
- this(node, sufficient, txnId, route, route.with(route.homeKey()), srcEpoch, untilLocalEpoch, callback);
- }
-
- CheckOn(Node node, Known sufficient, TxnId txnId, Route<?> route, Unseekables<?, ?> routeWithHomeKey, long srcEpoch, long untilLocalEpoch, BiConsumer<? super CheckStatusOkFull, Throwable> callback)
- {
- // TODO (desired, efficiency): restore behaviour of only collecting info if e.g. Committed or Executed
- super(node, txnId, routeWithHomeKey, srcEpoch, IncludeInfo.All);
- Invariants.checkArgument(routeWithHomeKey.contains(route.homeKey()));
- this.sufficient = sufficient;
- this.route = route;
- this.callback = callback;
- this.untilLocalEpoch = untilLocalEpoch;
- }
-
- // TODO (required, consider): many callers only need to consult precisely executeAt.epoch remotely
- public static CheckOn checkOn(Known sufficientStatus, Node node, TxnId txnId, Route<?> route, long srcEpoch, long untilLocalEpoch, BiConsumer<? super CheckStatusOkFull, Throwable> callback)
- {
- CheckOn checkOn = new CheckOn(node, sufficientStatus, txnId, route, srcEpoch, untilLocalEpoch, callback);
- checkOn.start();
- return checkOn;
- }
-
- protected Route<?> route()
- {
- return route;
- }
-
- @Override
- protected boolean isSufficient(Id from, CheckStatusOk ok)
- {
- Ranges rangesForNode = topologies().computeRangesForNode(from);
- PartialRoute<?> scope = this.route.slice(rangesForNode);
- return isSufficient(scope, ok);
- }
-
- @Override
- protected boolean isSufficient(CheckStatusOk ok)
- {
- return isSufficient(route, ok);
- }
-
- protected boolean isSufficient(Route<?> scope, CheckStatusOk ok)
- {
- return sufficient.isSatisfiedBy(((CheckStatusOkFull)ok).sufficientFor(scope));
- }
-
- @Override
- protected void onDone(Success success, Throwable failure)
- {
- Invariants.checkState((success == null) != (failure == null));
- if (failure != null)
- {
- callback.accept(null, failure);
- }
- else
- {
- if (success == Success.Success)
- Invariants.checkState(isSufficient(merged));
-
- if (merged.saveStatus == NotWitnessed)
- callback.accept(CheckStatusOkFull.NOT_WITNESSED, null);
- else
- new OnDone().start();
- }
- }
-
- class OnDone implements MapReduceConsume<SafeCommandStore, Void>
- {
- final Route<?> maxRoute;
- final RoutingKey progressKey;
- final CheckStatusOkFull full;
- final Known sufficientFor;
- final PartialTxn partialTxn;
- final PartialDeps partialDeps;
-
- public OnDone()
- {
- Ranges sliceRanges = node.topology().localRangesForEpochs(txnId.epoch(), untilLocalEpoch);
- Ranges covering = route().sliceCovering(sliceRanges, Minimal);
- Unseekables<?, ?> intersectingKeys = route().slice(covering, Minimal);
-
- full = (CheckStatusOkFull) merged;
- sufficientFor = full.sufficientFor(intersectingKeys);
- maxRoute = Route.merge((Route)route(), full.route);
- progressKey = node.trySelectProgressKey(txnId, maxRoute);
-
- PartialTxn partialTxn = null;
- if (sufficientFor.definition.isKnown())
- partialTxn = full.partialTxn.slice(sliceRanges, true).reconstitutePartial(covering);
- this.partialTxn = partialTxn;
-
- PartialDeps partialDeps = null;
- if (sufficientFor.deps.hasDecidedDeps())
- partialDeps = full.committedDeps.slice(sliceRanges).reconstitutePartial(covering);
- this.partialDeps = partialDeps;
- }
-
- void start()
- {
- Seekables<?, ?> keys = Keys.EMPTY;
- if (sufficientFor.definition.isKnown())
- keys = partialTxn.keys();
- else if (sufficientFor.deps.hasProposedOrDecidedDeps())
- keys = partialDeps.keyDeps.keys();
-
- Collection<TxnId> txnIds = Collections.emptyList();
- if (sufficientFor.deps.hasDecidedDeps())
- txnIds = partialDeps.txnIds();
-
- PreLoadContext loadContext = contextFor(txnId, txnIds, keys);
- node.mapReduceConsumeLocal(loadContext, route, txnId.epoch(), untilLocalEpoch, this);
- }
-
- @Override
- public Void apply(SafeCommandStore safeStore)
- {
- switch (sufficientFor.propagate())
- {
- default: throw new IllegalStateException();
- case Accepted:
- case AcceptedInvalidate:
- // we never "propagate" accepted statuses as these are essentially votes,
- // and contribute nothing to our local state machine
- throw new IllegalStateException("Invalid states to propagate");
-
- case Invalidated:
- Commands.commitInvalidate(safeStore, txnId);
- break;
-
- case Applied:
- case PreApplied:
- if (untilLocalEpoch >= full.executeAt.epoch())
- {
- confirm(Commands.commit(safeStore, txnId, maxRoute, progressKey, partialTxn, full.executeAt, partialDeps));
- confirm(Commands.apply(safeStore, txnId, untilLocalEpoch, maxRoute, full.executeAt, partialDeps, full.writes, full.result));
- break;
- }
-
- case Committed:
- case ReadyToExecute:
- confirm(Commands.commit(safeStore, txnId, maxRoute, progressKey, partialTxn, full.executeAt, partialDeps));
- break;
-
- case PreCommitted:
- Commands.precommit(safeStore, txnId, full.executeAt);
- if (!sufficientFor.definition.isKnown())
- break;
-
- case PreAccepted:
- // only preaccept if we coordinate the transaction
- if (untilLocalEpoch <= txnId.epoch() || safeStore.ranges().coordinates(txnId).intersects(maxRoute))
- Commands.preaccept(safeStore, txnId, txnId.epoch(), partialTxn, maxRoute, progressKey);
- break;
-
- case NotWitnessed:
- break;
- }
-
- RoutingKey homeKey = merged.homeKey;
- if (!merged.durability.isDurable() || homeKey == null)
- return null;
-
- if (!safeStore.ranges().coordinates(txnId).contains(homeKey))
- return null;
-
- Timestamp executeAt = merged.saveStatus.known.executeAt.hasDecidedExecuteAt() ? merged.executeAt : null;
- Commands.setDurability(safeStore, txnId, merged.durability, homeKey, executeAt);
- safeStore.progressLog().durable(safeStore.command(txnId).current(), null);
- return null;
- }
-
- @Override
- public Void reduce(Void o1, Void o2)
- {
- return null;
- }
-
- @Override
- public void accept(Void result, Throwable failure)
- {
- callback.accept(failure != null ? null : full, failure);
- }
- }
-
- private static void confirm(CommitOutcome outcome)
- {
- switch (outcome)
- {
- default: throw new IllegalStateException();
- case Redundant:
- case Success:
- return;
- case Insufficient: throw new IllegalStateException("Should have enough information");
- }
- }
-
- private static void confirm(ApplyOutcome outcome)
- {
- switch (outcome)
- {
- default: throw new IllegalStateException();
- case Redundant:
- case Success:
- return;
- case Insufficient: throw new IllegalStateException("Should have enough information");
- }
- }
-}
diff --git a/accord-core/src/main/java/accord/coordinate/CheckShards.java b/accord-core/src/main/java/accord/coordinate/CheckShards.java
index 75866a39..d9bb0f05 100644
--- a/accord-core/src/main/java/accord/coordinate/CheckShards.java
+++ b/accord-core/src/main/java/accord/coordinate/CheckShards.java
@@ -31,37 +31,45 @@ import accord.topology.Topologies;
* A result of null indicates the transaction is globally persistent
* A result of CheckStatusOk indicates the maximum status found for the transaction, which may be used to assess progress
*/
-public abstract class CheckShards extends ReadCoordinator<CheckStatusReply>
+public abstract class CheckShards<U extends Unseekables<?>> extends ReadCoordinator<CheckStatusReply>
{
- final Unseekables<?, ?> contact;
+ final U route;
/**
- * The epoch until which we want to fetch data from remotely
- * TODO (required, consider): configure the epoch we want to start with
+ * The epoch we want to fetch data from remotely
+ * Either txnId.epoch() or executeAt.epoch()
*/
- final long untilRemoteEpoch;
+ final long sourceEpoch;
final IncludeInfo includeInfo;
protected CheckStatusOk merged;
+ protected boolean truncated;
- protected CheckShards(Node node, TxnId txnId, Unseekables<?, ?> contact, long srcEpoch, IncludeInfo includeInfo)
+ // srcEpoch is either txnId.epoch() or executeAt.epoch()
+ protected CheckShards(Node node, TxnId txnId, U route, IncludeInfo includeInfo)
{
- super(node, topologyFor(node, txnId, contact, srcEpoch), txnId);
- this.untilRemoteEpoch = srcEpoch;
- this.contact = contact;
+ this(node, txnId, route, txnId.epoch(), includeInfo);
+ }
+
+ protected CheckShards(Node node, TxnId txnId, U route, long srcEpoch, IncludeInfo includeInfo)
+ {
+ super(node, topologyFor(node, txnId, route, srcEpoch), txnId);
+ this.sourceEpoch = srcEpoch;
+ this.route = route;
this.includeInfo = includeInfo;
}
- private static Topologies topologyFor(Node node, TxnId txnId, Unseekables<?, ?> contact, long epoch)
+ private static Topologies topologyFor(Node node, TxnId txnId, Unseekables<?> contact, long epoch)
{
+ // TODO (expected): only fetch data from source epoch
return node.topology().preciseEpochs(contact, txnId.epoch(), epoch);
}
@Override
protected void contact(Id id)
{
- Unseekables<?, ?> unseekables = contact.slice(topologies().computeRangesForNode(id));
- node.send(id, new CheckStatus(txnId, unseekables, txnId.epoch(), untilRemoteEpoch, includeInfo), this);
+ Unseekables<?> unseekables = route.slice(topologies().computeRangesForNode(id));
+ node.send(id, new CheckStatus(txnId, unseekables, sourceEpoch, includeInfo), this);
}
protected boolean isSufficient(Id from, CheckStatusOk ok) { return isSufficient(ok); }
@@ -86,13 +94,24 @@ public abstract class CheckShards extends ReadCoordinator<CheckStatusReply>
CheckStatusOk ok = (CheckStatusOk) reply;
if (merged == null) merged = ok;
else merged = merged.merge(ok);
-
return checkSufficient(from, ok);
}
else
{
- onFailure(from, new IllegalStateException("Submitted command to a replica that did not own the range"));
- return Action.Abort;
+ switch ((CheckStatus.CheckStatusNack)reply)
+ {
+ default: throw new AssertionError(String.format("Unexpected status: %s", reply));
+ case NotOwned:
+ finishOnFailure(new IllegalStateException("Submitted command to a replica that did not own the range"), true);
+ return Action.Aborted;
+ }
}
}
+
+ @Override
+ protected void finishOnExhaustion()
+ {
+ if (merged != null && merged.isTruncated()) finishOnFailure(new Truncated(txnId, null), false);
+ else super.finishOnExhaustion();
+ }
}
diff --git a/accord-core/src/main/java/accord/coordinate/CoordinateGloballyDurable.java b/accord-core/src/main/java/accord/coordinate/CoordinateGloballyDurable.java
new file mode 100644
index 00000000..5186d5d1
--- /dev/null
+++ b/accord-core/src/main/java/accord/coordinate/CoordinateGloballyDurable.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package accord.coordinate;
+
+import accord.coordinate.tracking.QuorumTracker;
+import accord.coordinate.tracking.RequestStatus;
+import accord.local.Node;
+import accord.local.DurableBefore;
+import accord.messages.Callback;
+import accord.messages.QueryDurableBefore;
+import accord.messages.QueryDurableBefore.DurableBeforeReply;
+import accord.messages.SetGloballyDurable;
+import accord.primitives.TxnId;
+import accord.topology.Topologies;
+import accord.utils.async.AsyncResult;
+import accord.utils.async.AsyncResults.SettableResult;
+
+// TODO (expected): this does not need to query every shard; can disseminate globally any sub-range of the ring
+// (indeed, we could slice both the query and dissemination only so that they always overlap)
+public class CoordinateGloballyDurable extends SettableResult<Void> implements Callback<DurableBeforeReply>
+{
+ final Node node;
+ // TODO (expected): this can be a ReadTracker, we only need one response from each shard
+ final QuorumTracker tracker;
+ private DurableBefore durableBefore = DurableBefore.EMPTY;
+
+ private CoordinateGloballyDurable(Node node, long epoch)
+ {
+ Topologies topologies = node.topology().preciseEpochs(epoch);
+ this.node = node;
+ this.tracker = new QuorumTracker(topologies);
+ }
+
+ public static AsyncResult<Void> coordinate(Node node, long epoch)
+ {
+ CoordinateGloballyDurable coordinate = new CoordinateGloballyDurable(node, epoch);
+ coordinate.start();
+ return coordinate;
+ }
+
+ private void start()
+ {
+ node.send(tracker.nodes(), to -> new QueryDurableBefore(tracker.topologies().currentEpoch()), this);
+ }
+
+ @Override
+ public void onSuccess(Node.Id from, DurableBeforeReply reply)
+ {
+ durableBefore = DurableBefore.merge(durableBefore, reply.durableBeforeMap);
+ if (tracker.recordSuccess(from) == RequestStatus.Success)
+ {
+ if (durableBefore != null && !durableBefore.equals(TxnId.NONE))
+ node.send(tracker.nodes(), new SetGloballyDurable(durableBefore));
+ trySuccess(null);
+ }
+ }
+
+ @Override
+ public void onFailure(Node.Id from, Throwable failure)
+ {
+ if (tracker.recordFailure(from) == RequestStatus.Failed)
+ tryFailure(new Exhausted(null, null));
+ }
+
+ @Override
+ public void onCallbackFailure(Node.Id from, Throwable failure)
+ {
+ tryFailure(failure);
+ }
+}
diff --git a/accord-core/src/main/java/accord/coordinate/CoordinateNoOp.java b/accord-core/src/main/java/accord/coordinate/CoordinateNoOp.java
index 90a1ce98..84c3e465 100644
--- a/accord-core/src/main/java/accord/coordinate/CoordinateNoOp.java
+++ b/accord-core/src/main/java/accord/coordinate/CoordinateNoOp.java
@@ -19,17 +19,14 @@
package accord.coordinate;
import java.util.List;
-import java.util.Set;
import accord.api.Result;
import accord.local.Node;
import accord.messages.Apply;
-import accord.messages.Commit;
import accord.messages.PreAccept.PreAcceptOk;
import accord.primitives.Ballot;
import accord.primitives.Deps;
import accord.primitives.FullRoute;
-import accord.primitives.Ranges;
import accord.primitives.Seekables;
import accord.primitives.Timestamp;
import accord.primitives.Txn;
@@ -40,7 +37,6 @@ import accord.utils.Invariants;
import accord.utils.async.AsyncResult;
import static accord.coordinate.Propose.Invalidate.proposeAndCommitInvalidate;
-import static accord.messages.Commit.Kind.Maximal;
import static accord.primitives.Txn.Kind.NoOp;
/**
@@ -71,6 +67,7 @@ public class CoordinateNoOp extends CoordinatePreAccept<Timestamp>
return coordinate;
}
+ @Override
void onPreAccepted(Topologies topologies, Timestamp executeAt, List<PreAcceptOk> successes)
{
if (executeAt.isRejected())
@@ -84,14 +81,9 @@ public class CoordinateNoOp extends CoordinatePreAccept<Timestamp>
@Override
void onAccepted()
{
- Topologies applyTo = node.topology().forEpoch(route, executeAt.epoch());
- Topologies persistTo = txnId.epoch() == executeAt.epoch() ? applyTo : node.topology().preciseEpochs(route, txnId.epoch(), executeAt.epoch());
Writes writes = txn.execute(txnId, txnId, null);
Result result = txn.result(txnId, executeAt, null);
- // TODO (now): permit Apply to Commit in these cases
- Set<Node.Id> nodes = persistTo.nodes();
- node.send(nodes, id -> new Commit(Maximal, id, persistTo.forEpoch(txnId.epoch()), persistTo, txnId, txn, route, Ranges.EMPTY, executeAt, deps, false));
- node.send(nodes, id -> new Apply(id, persistTo, applyTo, executeAt.epoch(), txnId, route, txn, executeAt, deps, writes, result));
+ Apply.sendMaximal(node, txnId, route, txn, executeAt, deps, writes, result);
accept(executeAt, null);
}
}.start();
diff --git a/accord-core/src/main/java/accord/coordinate/CoordinatePreAccept.java b/accord-core/src/main/java/accord/coordinate/CoordinatePreAccept.java
index 7bff43ef..7eb71ced 100644
--- a/accord-core/src/main/java/accord/coordinate/CoordinatePreAccept.java
+++ b/accord-core/src/main/java/accord/coordinate/CoordinatePreAccept.java
@@ -19,12 +19,15 @@
package accord.coordinate;
import java.util.ArrayList;
+import java.util.LinkedHashMap;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.function.BiConsumer;
import accord.coordinate.tracking.FastPathTracker;
import accord.coordinate.tracking.QuorumTracker;
+import accord.local.CommandStore;
import accord.local.Node;
import accord.local.Node.Id;
import accord.messages.Callback;
@@ -86,6 +89,7 @@ abstract class CoordinatePreAccept<T> extends SettableResult<T> implements Callb
CoordinatePreAccept.this.onCallbackFailure(from, failure);
}
+ @Override
public void onSuccess(Id from, PreAcceptReply reply)
{
synchronized (CoordinatePreAccept.this)
@@ -119,14 +123,20 @@ abstract class CoordinatePreAccept<T> extends SettableResult<T> implements Callb
private final List<PreAcceptOk> successes;
private boolean initialPreAcceptIsDone;
private ExtraPreAccept extraPreAccept;
+ private Map<Id, Object> debug = Invariants.debug() ? new LinkedHashMap<>() : null;
CoordinatePreAccept(Node node, TxnId txnId, Txn txn, FullRoute<?> route)
+ {
+ this(node, txnId, txn, route, node.topology().withUnsyncedEpochs(route, txnId, txnId));
+ }
+
+ CoordinatePreAccept(Node node, TxnId txnId, Txn txn, FullRoute<?> route, Topologies topologies)
{
this.node = node;
this.txnId = txnId;
this.txn = txn;
this.route = route;
- topologies = node.topology().withUnsyncedEpochs(route, txnId, txnId);
+ this.topologies = topologies;
this.tracker = new FastPathTracker(topologies);
this.successes = new ArrayList<>(topologies.estimateUniqueNodes());
}
@@ -140,13 +150,15 @@ abstract class CoordinatePreAccept<T> extends SettableResult<T> implements Callb
{
// TODO (desired, efficiency): consider sending only to electorate of most recent topology (as only these PreAccept votes matter)
// note that we must send to all replicas of old topology, as electorate may not be reachable
- node.send(nodes, to -> new PreAccept(to, topologies, txnId, txn, route),
- node.commandStores().select(route.homeKey()), callback);
+ CommandStore commandStore = CommandStore.maybeCurrent();
+ if (commandStore == null) commandStore = node.commandStores().select(route.homeKey());
+ node.send(nodes, to -> new PreAccept(to, topologies, txnId, txn, route), commandStore, callback);
}
@Override
public synchronized void onFailure(Id from, Throwable failure)
{
+ if (debug != null) debug.putIfAbsent(from, failure);
if (initialPreAcceptIsDone)
return;
@@ -173,8 +185,10 @@ abstract class CoordinatePreAccept<T> extends SettableResult<T> implements Callb
tryFailure(failure);
}
+ @Override
public synchronized void onSuccess(Id from, PreAcceptReply reply)
{
+ if (debug != null) debug.putIfAbsent(from, reply);
if (initialPreAcceptIsDone)
return;
@@ -194,6 +208,7 @@ abstract class CoordinatePreAccept<T> extends SettableResult<T> implements Callb
}
}
+ @Override
public void setFailure(Throwable failure)
{
Invariants.checkState(!initialPreAcceptIsDone || (extraPreAccept != null && !extraPreAccept.extraPreAcceptIsDone));
@@ -227,22 +242,19 @@ abstract class CoordinatePreAccept<T> extends SettableResult<T> implements Callb
// TODO (desired, efficiency): check if we have already have a valid quorum for the future epoch
// (noting that nodes may have adopted new ranges, in which case they should be discounted, and quorums may have changed shape)
node.withEpoch(executeAt.epoch(), () -> {
- synchronized (CoordinatePreAccept.this)
- {
- topologies = node.topology().withUnsyncedEpochs(route, txnId.epoch(), executeAt.epoch());
- boolean equivalent = topologies.oldestEpoch() <= prevTopologies.currentEpoch();
- for (long epoch = topologies.currentEpoch() ; equivalent && epoch > prevTopologies.currentEpoch() ; --epoch)
- equivalent = topologies.forEpoch(epoch).shards().equals(prevTopologies.current().shards());
+ topologies = node.topology().withUnsyncedEpochs(route, txnId.epoch(), executeAt.epoch());
+ boolean equivalent = topologies.oldestEpoch() <= prevTopologies.currentEpoch();
+ for (long epoch = topologies.currentEpoch() ; equivalent && epoch > prevTopologies.currentEpoch() ; --epoch)
+ equivalent = topologies.forEpoch(epoch).shards().equals(prevTopologies.current().shards());
- if (equivalent)
- {
- onPreAccepted(topologies, executeAt, successes);
- }
- else
- {
- extraPreAccept = new ExtraPreAccept(prevTopologies.currentEpoch() + 1, executeAt.epoch());
- extraPreAccept.start();
- }
+ if (equivalent)
+ {
+ onPreAccepted(topologies, executeAt, successes);
+ }
+ else
+ {
+ extraPreAccept = new ExtraPreAccept(prevTopologies.currentEpoch() + 1, executeAt.epoch());
+ extraPreAccept.start();
}
});
}
diff --git a/accord-core/src/main/java/accord/coordinate/CoordinateShardDurable.java b/accord-core/src/main/java/accord/coordinate/CoordinateShardDurable.java
new file mode 100644
index 00000000..daaf81fe
--- /dev/null
+++ b/accord-core/src/main/java/accord/coordinate/CoordinateShardDurable.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package accord.coordinate;
+
+import accord.coordinate.tracking.AppliedTracker;
+import accord.coordinate.tracking.RequestStatus;
+import accord.local.Node;
+import accord.messages.Callback;
+import accord.messages.ReadData;
+import accord.messages.SetShardDurable;
+import accord.messages.WaitUntilApplied;
+import accord.primitives.SyncPoint;
+import accord.topology.Topologies;
+import accord.utils.async.AsyncResult;
+import accord.utils.async.AsyncResults.SettableResult;
+
+/**
+ *
+ */
+public class CoordinateShardDurable extends SettableResult<Void> implements Callback<ReadData.ReadReply>
+{
+ final Node node;
+ final AppliedTracker tracker;
+ final SyncPoint exclusiveSyncPoint;
+
+ private CoordinateShardDurable(Node node, SyncPoint exclusiveSyncPoint)
+ {
+ // TODO (required): this isn't correct, we need to potentially perform a second round if a dependency executes in a future epoch and we have lost ownership of that epoch
+ Topologies topologies = node.topology().forEpoch(exclusiveSyncPoint.ranges, exclusiveSyncPoint.sourceEpoch());
+ this.node = node;
+ this.tracker = new AppliedTracker(topologies);
+ this.exclusiveSyncPoint = exclusiveSyncPoint;
+ }
+
+ public static AsyncResult<Void> coordinate(Node node, SyncPoint exclusiveSyncPoint)
+ {
+ CoordinateShardDurable coordinate = new CoordinateShardDurable(node, exclusiveSyncPoint);
+ coordinate.start();
+ return coordinate;
+ }
+
+ private void start()
+ {
+ node.send(tracker.nodes(), to -> new WaitUntilApplied(to, tracker.topologies(), exclusiveSyncPoint.syncId, exclusiveSyncPoint.ranges, exclusiveSyncPoint.syncId), this);
+ }
+
+ @Override
+ public void onSuccess(Node.Id from, ReadData.ReadReply reply)
+ {
+ if (!reply.isOk())
+ {
+ switch ((ReadData.ReadNack)reply)
+ {
+ default: throw new AssertionError("Unhandled: " + reply);
+
+ case NotCommitted:
+ CoordinateSyncPoint.sendApply(node, from, exclusiveSyncPoint);
+ return;
+
+ case Redundant:
+ tryFailure(new RuntimeException("Unexpected reply"));
+ return;
+
+ case Error:
+ // TODO (required): error propagation
+ tryFailure(new RuntimeException("Unknown error"));
+ return;
+
+ case Invalid:
+ tryFailure(new Invalidated(exclusiveSyncPoint.syncId, exclusiveSyncPoint.homeKey));
+ return;
+ }
+ }
+ else
+ {
+ // TODO (required): we also need to handle ranges not being safe to read
+ if (tracker.recordSuccess(from) == RequestStatus.Success)
+ {
+ node.configService().reportEpochRedundant(exclusiveSyncPoint.ranges, exclusiveSyncPoint.syncId.epoch());
+ node.send(tracker.nodes(), new SetShardDurable(exclusiveSyncPoint));
+ trySuccess(null);
+ }
+ }
+ }
+
+ @Override
+ public void onFailure(Node.Id from, Throwable failure)
+ {
+ if (tracker.recordFailure(from) == RequestStatus.Failed)
+ tryFailure(new Exhausted(exclusiveSyncPoint.syncId, exclusiveSyncPoint.homeKey));
+ }
+
+ @Override
+ public void onCallbackFailure(Node.Id from, Throwable failure)
+ {
+ tryFailure(failure);
+ }
+}
diff --git a/accord-core/src/main/java/accord/coordinate/CoordinateSyncPoint.java b/accord-core/src/main/java/accord/coordinate/CoordinateSyncPoint.java
index 8396b03b..6713694a 100644
--- a/accord-core/src/main/java/accord/coordinate/CoordinateSyncPoint.java
+++ b/accord-core/src/main/java/accord/coordinate/CoordinateSyncPoint.java
@@ -22,12 +22,12 @@ import java.util.List;
import accord.local.Node;
import accord.messages.Apply;
-import accord.messages.Commit;
import accord.messages.PreAccept.PreAcceptOk;
import accord.primitives.Ballot;
import accord.primitives.Deps;
+import accord.primitives.FullRangeRoute;
import accord.primitives.FullRoute;
-import accord.primitives.Seekables;
+import accord.primitives.Ranges;
import accord.primitives.SyncPoint;
import accord.primitives.Timestamp;
import accord.primitives.Txn;
@@ -50,39 +50,44 @@ import static accord.utils.Functions.foldl;
*/
public class CoordinateSyncPoint extends CoordinatePreAccept<SyncPoint>
{
- private CoordinateSyncPoint(Node node, TxnId txnId, Txn txn, FullRoute<?> route)
+ final Ranges ranges;
+ private CoordinateSyncPoint(Node node, TxnId txnId, Txn txn, FullRoute<?> route, Ranges ranges)
{
- super(node, txnId, txn, route);
+ super(node, txnId, txn, route, node.topology().withOpenEpochs(route, txnId, txnId));
+ this.ranges = ranges;
}
- public static AsyncResult<SyncPoint> exclusive(Node node, Seekables<?, ?> keysOrRanges)
+ public static AsyncResult<SyncPoint> exclusive(Node node, Ranges keysOrRanges)
{
return coordinate(node, ExclusiveSyncPoint, keysOrRanges);
}
- public static AsyncResult<SyncPoint> inclusive(Node node, Seekables<?, ?> keysOrRanges)
+ public static AsyncResult<SyncPoint> inclusive(Node node, Ranges keysOrRanges)
{
return coordinate(node, Kind.SyncPoint, keysOrRanges);
}
- private static AsyncResult<SyncPoint> coordinate(Node node, Kind kind, Seekables<?, ?> keysOrRanges)
+ private static AsyncResult<SyncPoint> coordinate(Node node, Kind kind, Ranges ranges)
{
- TxnId txnId = node.nextTxnId(kind, keysOrRanges.domain());
- FullRoute<?> route = node.computeRoute(txnId, keysOrRanges);
- CoordinateSyncPoint coordinate = new CoordinateSyncPoint(node, txnId, node.agent().emptyTxn(kind, keysOrRanges), route);
- coordinate.start();
- return coordinate;
+ TxnId txnId = node.nextTxnId(kind, ranges.domain());
+ return node.withEpoch(txnId.epoch(), () -> {
+ FullRangeRoute route = (FullRangeRoute) node.computeRoute(txnId, ranges);
+ CoordinateSyncPoint coordinate = new CoordinateSyncPoint(node, txnId, node.agent().emptyTxn(kind, ranges), route, ranges);
+ coordinate.start();
+ return coordinate;
+ }).beginAsResult();
}
- public static AsyncResult<SyncPoint> coordinate(Node node, TxnId txnId, Seekables<?, ?> keysOrRanges)
+ public static AsyncResult<SyncPoint> coordinate(Node node, TxnId txnId, Ranges ranges)
{
Invariants.checkState(txnId.rw() == Kind.SyncPoint || txnId.rw() == ExclusiveSyncPoint);
- FullRoute<?> route = node.computeRoute(txnId, keysOrRanges);
- CoordinateSyncPoint coordinate = new CoordinateSyncPoint(node, txnId, node.agent().emptyTxn(txnId.rw(), keysOrRanges), route);
+ FullRangeRoute route = (FullRangeRoute) node.computeRoute(txnId, ranges);
+ CoordinateSyncPoint coordinate = new CoordinateSyncPoint(node, txnId, node.agent().emptyTxn(txnId.rw(), ranges), route, ranges);
coordinate.start();
return coordinate;
}
+ @Override
void onNewEpoch(Topologies topologies, Timestamp executeAt, List<PreAcceptOk> successes)
{
// SyncPoint transactions always propose their own txnId as their executeAt, as they are not really executed.
@@ -93,6 +98,7 @@ public class CoordinateSyncPoint extends CoordinatePreAccept<SyncPoint>
onPreAccepted(topologies, executeAt, successes);
}
+ @Override
void onPreAccepted(Topologies topologies, Timestamp executeAt, List<PreAcceptOk> successes)
{
Deps deps = Deps.merge(successes, ok -> ok.deps);
@@ -111,14 +117,20 @@ public class CoordinateSyncPoint extends CoordinatePreAccept<SyncPoint>
@Override
void onAccepted()
{
- Topologies commit = node.topology().preciseEpochs(route, txnId.epoch(), executeAt.epoch());
- Topologies latest = commit.size() == 1 ? commit : node.topology().forEpoch(route, executeAt.epoch());
- if (latest != commit)
- node.send(commit.nodes(), id -> new Commit(Commit.Kind.Maximal, id, commit.forEpoch(txnId.epoch()), commit, txnId, txn, route, null, executeAt, deps, false));
- node.send(latest.nodes(), id -> new Apply(id, latest, latest, executeAt.epoch(), txnId, route, txn, executeAt, deps, txn.execute(txnId, executeAt, null), txn.result(txnId, executeAt, null)));
- accept(new SyncPoint(txnId, deps, route), null);
+ Apply.sendMaximal(node, txnId, route, txn, executeAt, deps, txn.execute(txnId, executeAt, null), txn.result(txnId, executeAt, null));
+ node.configService().reportEpochClosed(ranges, txnId.epoch());
+ accept(new SyncPoint(txnId, deps, ranges, (FullRangeRoute) route), null);
}
}.start();
}
}
+
+ public static void sendApply(Node node, Node.Id to, SyncPoint syncPoint)
+ {
+ TxnId txnId = syncPoint.syncId;
+ Timestamp executeAt = txnId;
+ Txn txn = node.agent().emptyTxn(txnId.rw(), syncPoint.ranges);
+ Deps deps = syncPoint.waitFor;
+ Apply.sendMaximal(node, to, txnId, syncPoint.route(), txn, executeAt, deps, txn.execute(txnId, executeAt, null), txn.result(txnId, executeAt, null));
+ }
}
diff --git a/accord-core/src/main/java/accord/coordinate/CoordinateTransaction.java b/accord-core/src/main/java/accord/coordinate/CoordinateTransaction.java
index 9c477ba5..ccdd5871 100644
--- a/accord-core/src/main/java/accord/coordinate/CoordinateTransaction.java
+++ b/accord-core/src/main/java/accord/coordinate/CoordinateTransaction.java
@@ -56,6 +56,7 @@ public class CoordinateTransaction extends CoordinatePreAccept<Result>
return coordinate;
}
+ @Override
void onPreAccepted(Topologies topologies, Timestamp executeAt, List<PreAcceptOk> successes)
{
if (tracker.hasFastPathAccepted())
diff --git a/accord-core/src/main/java/accord/coordinate/CoordinationFailed.java b/accord-core/src/main/java/accord/coordinate/CoordinationFailed.java
index 12e3a4a2..4a2b74f5 100644
--- a/accord-core/src/main/java/accord/coordinate/CoordinationFailed.java
+++ b/accord-core/src/main/java/accord/coordinate/CoordinationFailed.java
@@ -26,7 +26,7 @@ import accord.primitives.TxnId;
/**
* Thrown when a transaction exceeds its specified timeout for obtaining a result for a client
*/
-public class CoordinationFailed extends Throwable
+public class CoordinationFailed extends RuntimeException
{
private @Nullable TxnId txnId;
private @Nullable RoutingKey homeKey;
@@ -35,6 +35,7 @@ public class CoordinationFailed extends Throwable
this.txnId = txnId;
this.homeKey = homeKey;
}
+
public CoordinationFailed(TxnId txnId, @Nullable RoutingKey homeKey, String message)
{
super(message);
diff --git a/accord-core/src/main/java/accord/coordinate/Execute.java b/accord-core/src/main/java/accord/coordinate/Execute.java
index f02f13aa..da39a056 100644
--- a/accord-core/src/main/java/accord/coordinate/Execute.java
+++ b/accord-core/src/main/java/accord/coordinate/Execute.java
@@ -41,23 +41,23 @@ import static accord.messages.Commit.Kind.Maximal;
class Execute extends ReadCoordinator<ReadReply>
{
final Txn txn;
- final Seekables<?, ?> readScope;
+ final Participants<?> readScope;
final FullRoute<?> route;
final Timestamp executeAt;
final Deps deps;
- final Topologies applyTo;
+ final Topologies executes;
final BiConsumer<? super Result, Throwable> callback;
private Data data;
- private Execute(Node node, TxnId txnId, Txn txn, FullRoute<?> route, Seekables<?, ?> readScope, Timestamp executeAt, Deps deps, BiConsumer<? super Result, Throwable> callback)
+ private Execute(Node node, TxnId txnId, Txn txn, FullRoute<?> route, Participants<?> readScope, Timestamp executeAt, Deps deps, BiConsumer<? super Result, Throwable> callback)
{
- super(node, node.topology().forEpoch(readScope.toUnseekables(), executeAt.epoch()), txnId);
+ super(node, node.topology().forEpoch(readScope, executeAt.epoch()), txnId);
this.txn = txn;
this.route = route;
this.readScope = readScope;
this.executeAt = executeAt;
this.deps = deps;
- this.applyTo = node.topology().forEpoch(route, executeAt.epoch());
+ this.executes = node.topology().forEpoch(route, executeAt.epoch());
this.callback = callback;
}
@@ -65,15 +65,13 @@ class Execute extends ReadCoordinator<ReadReply>
{
if (txn.read().keys().isEmpty())
{
- Topologies applyTo = node.topology().forEpoch(route, executeAt.epoch());
- Topologies persistTo = txnId.epoch() == executeAt.epoch() ? applyTo : node.topology().preciseEpochs(route, txnId.epoch(), executeAt.epoch());
Result result = txn.result(txnId, executeAt, null);
- Persist.persist(node, persistTo, applyTo, txnId, route, txn, executeAt, deps, txn.execute(txnId, executeAt, null), result);
+ Persist.persist(node, txnId, route, txn, executeAt, deps, txn.execute(txnId, executeAt, null), result);
callback.accept(result, null);
}
else
{
- Execute execute = new Execute(node, txnId, txn, route, txn.keys(), executeAt, deps, callback);
+ Execute execute = new Execute(node, txnId, txn, route, txn.keys().toParticipants(), executeAt, deps, callback);
execute.start();
}
}
@@ -81,7 +79,7 @@ class Execute extends ReadCoordinator<ReadReply>
@Override
protected void start(Set<Id> readSet)
{
- Commit.commitMinimalAndRead(node, applyTo, txnId, txn, route, readScope, executeAt, deps, readSet, this);
+ Commit.commitMinimalAndRead(node, executes, txnId, txn, route, readScope, executeAt, deps, readSet, this);
}
@Override
@@ -118,7 +116,7 @@ class Execute extends ReadCoordinator<ReadReply>
return Action.Reject;
case Redundant:
callback.accept(null, new Preempted(txnId, route.homeKey()));
- return Action.Abort;
+ return Action.Aborted;
case NotCommitted:
// the replica may be missing the original commit, or the additional commit, so send everything
Topologies topology = node.topology().preciseEpochs(route, txnId.epoch(), executeAt.epoch());
@@ -128,7 +126,7 @@ class Execute extends ReadCoordinator<ReadReply>
return Action.TryAlternative;
case Invalid:
callback.accept(null, new IllegalStateException("Submitted a read command to a replica that did not own the range"));
- return Action.Abort;
+ return Action.Aborted;
}
}
@@ -141,8 +139,7 @@ class Execute extends ReadCoordinator<ReadReply>
Result result = txn.result(txnId, executeAt, data);
callback.accept(result, null);
// avoid re-calculating topologies if it is unchanged
- Topologies persistTo = txnId.epoch() == executeAt.epoch() ? applyTo : node.topology().preciseEpochs(route, txnId.epoch(), executeAt.epoch());
- Persist.persist(node, persistTo, applyTo, txnId, route, txn, executeAt, deps, txn.execute(txnId, executeAt, data), result);
+ Persist.persist(node, executes, txnId, route, txn, executeAt, deps, txn.execute(txnId, executeAt, data), result);
}
else
{
diff --git a/accord-core/src/main/java/accord/coordinate/FetchCoordinator.java b/accord-core/src/main/java/accord/coordinate/FetchCoordinator.java
index 60d0bba1..707a329c 100644
--- a/accord-core/src/main/java/accord/coordinate/FetchCoordinator.java
+++ b/accord-core/src/main/java/accord/coordinate/FetchCoordinator.java
@@ -83,21 +83,21 @@ public abstract class FetchCoordinator
contacted = contacted.with(ranges);
inflight = inflight.with(ranges);
waitingOn = waitingOn.with(ranges);
- uncontacted = uncontacted.difference(ranges);
+ uncontacted = uncontacted.subtract(ranges);
}
void slow(Ranges ranges)
{
checkArgument(waitingOn.containsAll(ranges));
- waitingOn = waitingOn.difference(ranges);
+ waitingOn = waitingOn.subtract(ranges);
}
void successful(Ranges ranges)
{
checkArgument(inflight.containsAll(ranges));
successful = successful.with(ranges);
- inflight = inflight.difference(ranges);
- waitingOn = waitingOn.difference(ranges);
+ inflight = inflight.subtract(ranges);
+ waitingOn = waitingOn.subtract(ranges);
}
Ranges waitingOn(Ranges ranges)
@@ -109,8 +109,8 @@ public abstract class FetchCoordinator
{
checkArgument(inflight.containsAll(ranges));
unsuccessful = unsuccessful.with(ranges);
- inflight = inflight.difference(ranges);
- waitingOn = waitingOn.difference(ranges);
+ inflight = inflight.subtract(ranges);
+ waitingOn = waitingOn.subtract(ranges);
}
void fail()
@@ -228,7 +228,7 @@ public abstract class FetchCoordinator
++inflight;
state.contact(contact);
contact(state.id, contact);
- needed = needed.difference(contact);
+ needed = needed.subtract(contact);
if (needed.isEmpty())
break;
}
diff --git a/accord-core/src/main/java/accord/coordinate/FetchData.java b/accord-core/src/main/java/accord/coordinate/FetchData.java
index 48e29c73..774df8ab 100644
--- a/accord-core/src/main/java/accord/coordinate/FetchData.java
+++ b/accord-core/src/main/java/accord/coordinate/FetchData.java
@@ -21,97 +21,453 @@ package accord.coordinate;
import java.util.function.BiConsumer;
import accord.api.RoutingKey;
+import accord.local.Command;
+import accord.local.Commands;
import accord.local.Node;
+import accord.local.PreLoadContext;
+import accord.local.SafeCommand;
+import accord.local.SafeCommandStore;
+import accord.local.Status;
import accord.local.Status.Known;
+import accord.local.Status.Phase;
+import accord.messages.CheckStatus;
+import accord.messages.CheckStatus.CheckStatusOkFull;
+import accord.messages.CheckStatus.WithQuorum;
import accord.primitives.*;
import accord.utils.Invariants;
+import accord.utils.MapReduceConsume;
import javax.annotation.Nullable;
-import static accord.local.Status.LogicalEpoch.Coordination;
-import static accord.local.Status.Known.*;
-import static accord.local.Status.Outcome.OutcomeKnown;
-import static accord.local.Status.Outcome.OutcomeUnknown;
+import static accord.coordinate.Infer.InvalidateAndCallback.locallyInvalidateAndCallback;
+import static accord.local.PreLoadContext.contextFor;
+import static accord.local.Status.NotDefined;
+import static accord.local.Status.Phase.Cleanup;
+import static accord.local.Status.PreApplied;
+import static accord.messages.CheckStatus.WithQuorum.NoQuorum;
+import static accord.primitives.Routables.Slice.Minimal;
+import static accord.primitives.Route.castToRoute;
+import static accord.primitives.Route.isRoute;
/**
* Find data and persist locally
*
- * TODO (desired, efficiency): accept lower bound epoch to avoid fetching data we should already have
+ * TODO (expected): avoid multiple command stores performing duplicate queries to other shards
*/
-public class FetchData
+public class FetchData extends CheckShards<Route<?>>
{
- public static Object fetch(Known fetch, Node node, TxnId txnId, Unseekables<?, ?> someUnseekables, long untilLocalEpoch, BiConsumer<Known, Throwable> callback)
+ public static Object fetch(Known fetch, Node node, TxnId txnId, Unseekables<?> someUnseekables, BiConsumer<? super Known, Throwable> callback)
{
- return fetch(fetch, node, txnId, someUnseekables, null, untilLocalEpoch, callback);
+ return fetch(fetch, node, txnId, someUnseekables, null, callback);
}
- public static Object fetch(Known fetch, Node node, TxnId txnId, Unseekables<?, ?> someUnseekables, @Nullable Timestamp executeAt, long untilLocalEpoch, BiConsumer<Known, Throwable> callback)
+ public static Object fetch(Known fetch, Node node, TxnId txnId, Unseekables<?> someUnseekables, @Nullable Timestamp executeAt, BiConsumer<? super Known, Throwable> callback)
{
- if (someUnseekables.kind().isRoute()) return fetch(fetch, node, txnId, Route.tryCastToRoute(someUnseekables), executeAt, untilLocalEpoch, callback);
- else return fetchWithSomeRoutables(fetch, node, txnId, someUnseekables, untilLocalEpoch, callback);
+ if (someUnseekables.kind().isRoute()) return fetch(fetch, node, txnId, castToRoute(someUnseekables), executeAt, callback);
+ else return fetchViaSomeRoute(fetch, node, txnId, someUnseekables, executeAt, callback);
}
- public static Object fetch(Known fetch, Node node, TxnId txnId, Route<?> route, @Nullable Timestamp executeAt, long untilLocalEpoch, BiConsumer<Known, Throwable> callback)
+ public static Object fetch(Known fetch, Node node, TxnId txnId, Route<?> route, @Nullable Timestamp executeAt, BiConsumer<? super Known, Throwable> callback)
{
- Invariants.checkArgument(node.topology().hasEpoch(untilLocalEpoch));
- Ranges ranges = node.topology().localRangesForEpochs(txnId.epoch(), untilLocalEpoch);
+ long srcEpoch = fetch.fetchEpoch(txnId, executeAt);
+ if (!node.topology().hasEpoch(srcEpoch))
+ return node.topology().awaitEpoch(srcEpoch).map(ignore -> fetch(fetch, node, txnId, route, executeAt, callback)).beginAsResult();
+
+ Invariants.checkArgument(node.topology().hasEpoch(srcEpoch), "Unknown epoch %d, latest known is %d", srcEpoch, node.epoch());
+ Ranges ranges = node.topology().localRangesForEpochs(txnId.epoch(), srcEpoch);
if (!route.covers(ranges))
{
- return fetchWithHomeKey(fetch, node, txnId, route.homeKey(), untilLocalEpoch, callback);
+ return fetchWithIncompleteRoute(fetch, node, txnId, route, executeAt, callback);
}
else
{
- return fetchInternal(ranges, fetch, node, txnId, route.sliceStrict(ranges), executeAt, untilLocalEpoch, callback);
+ return fetchInternal(ranges, fetch, node, txnId, route.sliceStrict(ranges), executeAt, callback);
}
}
- private static Object fetchWithSomeRoutables(Known fetch, Node node, TxnId txnId, Unseekables<?, ?> someUnseekables, long untilLocalEpoch, BiConsumer<Known, Throwable> callback)
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ private static Object fetchViaSomeRoute(Known fetch, Node node, TxnId txnId, Unseekables<?> someUnseekables, @Nullable Timestamp executeAt, BiConsumer<? super Known, Throwable> callback)
{
- Invariants.checkArgument(node.topology().hasEpoch(untilLocalEpoch));
- return FindHomeKey.findHomeKey(node, txnId, someUnseekables, (foundHomeKey, fail) -> {
+ return FindSomeRoute.findSomeRoute(node, txnId, someUnseekables, (foundRoute, fail) -> {
if (fail != null) callback.accept(null, fail);
- else if (foundHomeKey == null) callback.accept(Nothing, null);
- else fetchWithHomeKey(fetch, node, txnId, foundHomeKey, untilLocalEpoch, callback);
+ else if (foundRoute.route == null)
+ {
+ reportRouteNotFound(node, txnId, someUnseekables, executeAt, foundRoute.known, callback);
+ }
+ else if (isRoute(someUnseekables) && someUnseekables.containsAll(foundRoute.route))
+ {
+ // this is essentially a reentrancy check; we can only reach this point if we have already tried once to fetchSomeRoute
+ // (as a user-provided Route is used to fetchRoute, not fetchSomeRoute)
+ reportRouteNotFound(node, txnId, someUnseekables, executeAt, foundRoute.known, callback);
+ }
+ else
+ {
+ Route<?> route = foundRoute.route;
+ if (isRoute(someUnseekables))
+ route = Route.merge(route, (Route)someUnseekables);
+ fetch(fetch, node, txnId, route, executeAt, callback);
+ }
});
}
- private static Object fetchWithHomeKey(Known fetch, Node node, TxnId txnId, RoutingKey homeKey, long untilLocalEpoch, BiConsumer<Known, Throwable> callback)
+ private static void reportRouteNotFound(Node node, TxnId txnId, Unseekables<?> someUnseekables, @Nullable Timestamp executeAt, Known found, BiConsumer<? super Known, Throwable> callback)
+ {
+ Invariants.checkState(executeAt == null);
+ switch (found.outcome)
+ {
+ default: throw new AssertionError("Unknown outcome: " + found.outcome);
+ case Invalidated:
+ locallyInvalidateAndCallback(node, txnId, someUnseekables, found, callback);
+ break;
+
+ case Unknown:
+ if (found.canProposeInvalidation())
+ {
+ Invalidate.invalidate(node, txnId, someUnseekables, (outcome, throwable) -> {
+ Known known = throwable != null ? null : outcome.asProgressToken().status == Status.Invalidated ? Known.Invalidated : Known.Nothing;
+ callback.accept(known, throwable);
+ });
+ break;
+ }
+ case Erased:
+ case WasApply:
+ case Apply:
+ callback.accept(found, null);
+ }
+ }
+
+ private static Object fetchWithIncompleteRoute(Known fetch, Node node, TxnId txnId, Route<?> someRoute, @Nullable Timestamp executeAt, BiConsumer<? super Known, Throwable> callback)
{
- Invariants.checkArgument(node.topology().hasEpoch(untilLocalEpoch));
- return FindRoute.findRoute(node, txnId, homeKey, (foundRoute, fail) -> {
+ long srcEpoch = fetch.fetchEpoch(txnId, executeAt);
+ Invariants.checkArgument(node.topology().hasEpoch(srcEpoch), "Unknown epoch %d, latest known is %d", srcEpoch, node.epoch());
+ return FindRoute.findRoute(node, txnId, someRoute.withHomeKey(), (foundRoute, fail) -> {
if (fail != null) callback.accept(null, fail);
- else if (foundRoute == null) callback.accept(Nothing, null);
- else fetch(fetch, node, txnId, foundRoute.route, foundRoute.executeAt, untilLocalEpoch, callback);
+ else if (foundRoute == null) fetchViaSomeRoute(fetch, node, txnId, someRoute, executeAt, callback);
+ else fetch(fetch, node, txnId, foundRoute.route, foundRoute.executeAt, callback);
});
}
- public static Object fetch(Known fetch, Node node, TxnId txnId, FullRoute<?> route, @Nullable Timestamp executeAt, long untilLocalEpoch, BiConsumer<Known, Throwable> callback)
+ public static Object fetch(Known fetch, Node node, TxnId txnId, FullRoute<?> route, @Nullable Timestamp executeAt, BiConsumer<? super Known, Throwable> callback)
{
- Ranges ranges = node.topology().localRangesForEpochs(txnId.epoch(), untilLocalEpoch);
- return fetchInternal(ranges, fetch, node, txnId, route.sliceStrict(ranges), executeAt, untilLocalEpoch, callback);
+ return node.awaitEpoch(executeAt).map(ignore -> {
+ Ranges ranges = node.topology().localRangesForEpochs(txnId.epoch(), fetch.fetchEpoch(txnId, executeAt));
+ return fetchInternal(ranges, fetch, node, txnId, route.sliceStrict(ranges), executeAt, callback);
+ }).beginAsResult();
}
- private static Object fetchInternal(Ranges ranges, Known target, Node node, TxnId txnId, PartialRoute<?> route, @Nullable Timestamp executeAt, long untilLocalEpoch, BiConsumer<Known, Throwable> callback)
+ private static Object fetchInternal(Ranges ranges, Known target, Node node, TxnId txnId, PartialRoute<?> route, @Nullable Timestamp executeAt, BiConsumer<? super Known, Throwable> callback)
{
- long srcEpoch = executeAt == null || target.epoch() == Coordination ? txnId.epoch() : executeAt.epoch();
- if (!node.topology().hasEpoch(srcEpoch))
- return node.topology().awaitEpoch(srcEpoch).map(ignore -> fetchInternal(ranges, target, node, txnId, route, executeAt, untilLocalEpoch, callback)).beginAsResult();
-
+ long srcEpoch = target.fetchEpoch(txnId, executeAt);
+ Invariants.checkArgument(node.topology().hasEpoch(srcEpoch), "Unknown epoch %d, latest known is %d", srcEpoch, node.epoch());
PartialRoute<?> fetch = route.sliceStrict(ranges);
- return CheckOn.checkOn(target, node, txnId, fetch, srcEpoch, untilLocalEpoch, (ok, fail) -> {
+ return fetchData(target, node, txnId, fetch, srcEpoch, (sufficientFor, fail) -> {
if (fail != null) callback.accept(null, fail);
- else if (ok == null) callback.accept(Nothing, null);
- else
+ else callback.accept(sufficientFor, null);
+ });
+ }
+
+ final BiConsumer<Known, Throwable> callback;
+ /**
+ * The epoch until which we want to persist any response for locally
+ */
+ final Known target;
+
+ private FetchData(Node node, Known target, TxnId txnId, Route<?> route, long sourceEpoch, BiConsumer<Known, Throwable> callback)
+ {
+ this(node, target, txnId, route, route.withHomeKey(), sourceEpoch, callback);
+ }
+
+ private FetchData(Node node, Known target, TxnId txnId, Route<?> route, Route<?> routeWithHomeKey, long sourceEpoch, BiConsumer<Known, Throwable> callback)
+ {
+ // TODO (desired, efficiency): restore behaviour of only collecting info if e.g. Committed or Executed
+ super(node, txnId, routeWithHomeKey, sourceEpoch, CheckStatus.IncludeInfo.All);
+ Invariants.checkArgument(routeWithHomeKey.contains(route.homeKey()), "route %s does not contain %s", routeWithHomeKey, route.homeKey());
+ this.target = target;
+ this.callback = callback;
+ }
+
+ private static FetchData fetchData(Known sufficientStatus, Node node, TxnId txnId, Route<?> route, long epoch, BiConsumer<Known, Throwable> callback)
+ {
+ FetchData fetch = new FetchData(node, sufficientStatus, txnId, route, epoch, callback);
+ fetch.start();
+ return fetch;
+ }
+
+ protected Route<?> route()
+ {
+ return route;
+ }
+
+ @Override
+ protected boolean isSufficient(Node.Id from, CheckStatus.CheckStatusOk ok)
+ {
+ Ranges rangesForNode = topologies().computeRangesForNode(from);
+ PartialRoute<?> scope = this.route.slice(rangesForNode);
+ return isSufficient(scope, ok);
+ }
+
+ @Override
+ protected boolean isSufficient(CheckStatus.CheckStatusOk ok)
+ {
+ return isSufficient(route, ok);
+ }
+
+ protected boolean isSufficient(Route<?> scope, CheckStatus.CheckStatusOk ok)
+ {
+ return target.isSatisfiedBy(((CheckStatusOkFull)ok).sufficientFor(scope.participants(), NoQuorum));
+ }
+
+ @Override
+ protected void onDone(ReadCoordinator.Success success, Throwable failure)
+ {
+ Invariants.checkState((success == null) != (failure == null));
+ if (failure != null)
+ {
+ callback.accept(null, failure);
+ }
+ else
+ {
+ if (success == ReadCoordinator.Success.Success)
+ Invariants.checkState(isSufficient(merged), "Status %s is not sufficient", merged);
+
+ OnDone.propagate(node, txnId, sourceEpoch, success.withQuorum, route(), target, (CheckStatusOkFull) merged, callback);
+ }
+ }
+
+ static class OnDone implements MapReduceConsume<SafeCommandStore, Void>, EpochSupplier
+ {
+ final Node node;
+ final TxnId txnId;
+ final Route<?> route;
+ final RoutingKey progressKey;
+ final CheckStatusOkFull full;
+ // this is a WHOLE NODE measure, so if commit epoch has more ranges we do not count as committed if we can only commit in coordination epoch
+ final Known achieved;
+ final PartialTxn partialTxn;
+ final PartialDeps partialDeps;
+ final long toEpoch;
+ final BiConsumer<Known, Throwable> callback;
+
+ OnDone(Node node, TxnId txnId, Route<?> route, RoutingKey progressKey, CheckStatusOkFull full, Known achieved, PartialTxn partialTxn, PartialDeps partialDeps, long toEpoch, BiConsumer<Known, Throwable> callback)
+ {
+ this.node = node;
+ this.txnId = txnId;
+ this.route = route;
+ this.progressKey = progressKey;
+ this.full = full;
+ this.achieved = achieved;
+ this.partialTxn = partialTxn;
+ this.partialDeps = partialDeps;
+ this.toEpoch = toEpoch;
+ this.callback = callback;
+ }
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ public static void propagate(Node node, TxnId txnId, long sourceEpoch, WithQuorum withQuorum, Route route, @Nullable Known target, CheckStatusOkFull full, BiConsumer<Known, Throwable> callback)
+ {
+ if (full.saveStatus.status == NotDefined && full.invalidIfNotAtLeast == NotDefined)
{
- // even if we have enough information to Apply for the requested epochs, if we didn't request enough
- // information to fulfil that phase locally we should downgrade the response we give to the callback
- Known sufficientFor = ok.sufficientFor(fetch);
- // if we discover the executeAt as part of this action, use that to decide if we requested enough info
- Timestamp exec = executeAt != null ? executeAt : ok.saveStatus.known.executeAt.hasDecidedExecuteAt() ? ok.executeAt : null;
- if (sufficientFor.outcome == OutcomeKnown && (exec == null || untilLocalEpoch < exec.epoch()))
- sufficientFor = sufficientFor.with(OutcomeUnknown);
- callback.accept(sufficientFor, null);
+ callback.accept(Known.Nothing, null);
+ return;
}
- });
+
+ Invariants.checkState(sourceEpoch == txnId.epoch() || (full.executeAt != null && sourceEpoch == full.executeAt.epoch()));
+ Route<?> maxRoute = Route.merge(route, full.route);
+
+ // TODO (required): permit individual shards that are behind to catch up by themselves
+ long toEpoch = sourceEpoch;
+ Ranges sliceRanges = node.topology().localRangesForEpochs(txnId.epoch(), toEpoch);
+ if (!maxRoute.covers(sliceRanges))
+ {
+ callback.accept(Known.Nothing, null);
+ return;
+ }
+
+ RoutingKey progressKey = node.trySelectProgressKey(txnId, maxRoute);
+
+ Ranges covering = maxRoute.sliceCovering(sliceRanges, Minimal);
+ Participants<?> participatingKeys = maxRoute.participants().slice(covering, Minimal);
+ Known achieved = full.sufficientFor(participatingKeys, withQuorum);
+ if (achieved.executeAt.hasDecidedExecuteAt() && full.executeAt.epoch() > toEpoch)
+ {
+ Ranges acceptRanges;
+ if (!node.topology().hasEpoch(full.executeAt.epoch()) ||
+ (!maxRoute.covers(acceptRanges = node.topology().localRangesForEpochs(txnId.epoch(), full.executeAt.epoch()))))
+ {
+ // we don't know what the execution epoch requires, so we cannot be sure we can replicate it locally
+ // we *could* wait until we have the local epoch before running this
+ Status.Outcome outcome = achieved.outcome.propagatesBetweenShards() ? achieved.outcome : Status.Outcome.Unknown;
+ achieved = new Known(achieved.definition, achieved.executeAt, Status.KnownDeps.DepsUnknown, outcome);
+ }
+ else
+ {
+ // TODO (expected): this should only be the two precise epochs, not the full range of epochs
+ sliceRanges = acceptRanges;
+ covering = maxRoute.sliceCovering(sliceRanges, Minimal);
+ participatingKeys = maxRoute.participants().slice(covering, Minimal);
+ Known knownForExecution = full.sufficientFor(participatingKeys, withQuorum);
+ if ((target != null && target.isSatisfiedBy(knownForExecution)) || knownForExecution.isSatisfiedBy(achieved))
+ {
+ achieved = knownForExecution;
+ toEpoch = full.executeAt.epoch();
+ }
+ else
+ {
+ Invariants.checkState(sourceEpoch == txnId.epoch(), "%d != %d", sourceEpoch, txnId.epoch());
+ achieved = new Known(achieved.definition, achieved.executeAt, knownForExecution.deps, knownForExecution.outcome);
+ }
+ }
+ }
+
+ PartialTxn partialTxn = null;
+ if (achieved.definition.isKnown())
+ partialTxn = full.partialTxn.slice(sliceRanges, true).reconstitutePartial(covering);
+
+ PartialDeps partialDeps = null;
+ if (achieved.deps.hasDecidedDeps())
+ partialDeps = full.committedDeps.slice(sliceRanges).reconstitutePartial(covering);
+
+ new OnDone(node, txnId, maxRoute, progressKey, full, achieved, partialTxn, partialDeps, toEpoch, callback).start();
+ }
+
+ void start()
+ {
+ Seekables<?, ?> keys = Keys.EMPTY;
+ if (achieved.definition.isKnown())
+ keys = partialTxn.keys();
+ else if (achieved.deps.hasProposedOrDecidedDeps())
+ keys = partialDeps.keyDeps.keys();
+
+ PreLoadContext loadContext = contextFor(txnId, keys);
+ node.mapReduceConsumeLocal(loadContext, route, txnId.epoch(), toEpoch, this);
+ }
+
+ @Override
+ public Void apply(SafeCommandStore safeStore)
+ {
+ SafeCommand safeCommand = safeStore.get(txnId, this, route);
+ Command command = safeCommand.current();
+ if (command.saveStatus().phase.compareTo(Phase.Persist) >= 0)
+ return null;
+
+ Status propagate = achieved.propagate();
+ if (command.hasBeen(propagate))
+ {
+ if (full.maxSaveStatus.phase == Cleanup && full.durability.isDurableOrInvalidated() && Infer.safeToCleanup(safeStore, command, route, full.executeAt))
+ Commands.setTruncatedApply(safeStore, safeCommand);
+ return null;
+ }
+
+ switch (propagate)
+ {
+ default: throw new IllegalStateException("Unexpected status: " + propagate);
+ case Accepted:
+ case AcceptedInvalidate:
+ // we never "propagate" accepted statuses as these are essentially votes,
+ // and contribute nothing to our local state machine
+ throw new IllegalStateException("Invalid states to propagate: " + achieved.propagate());
+
+ case Truncated:
+ // if our peers have truncated this command, then either:
+ // 1) we have already applied it locally; 2) the command doesn't apply locally; 3) we are stale; or 4) the command is invalidated
+ if (command.hasBeen(PreApplied) || command.saveStatus().isUninitialised())
+ break;
+
+ if (Infer.safeToCleanup(safeStore, command, route, full.executeAt))
+ {
+ Commands.setErased(safeStore, safeCommand);
+ break;
+ }
+
+ // TODO (required): check if we are stale
+ // otherwise we are either stale, or the command didn't reach consensus
+
+ case Invalidated:
+ Commands.commitInvalidate(safeStore, safeCommand, route);
+ break;
+
+ case Applied:
+ case PreApplied:
+ Invariants.checkState(full.executeAt != null);
+ if (toEpoch >= full.executeAt.epoch())
+ {
+ confirm(Commands.apply(safeStore, safeCommand, txnId, route, progressKey, full.executeAt, partialDeps, partialTxn, full.writes, full.result));
+ break;
+ }
+
+ case Committed:
+ case ReadyToExecute:
+ confirm(Commands.commit(safeStore, safeCommand, txnId, route, progressKey, partialTxn, full.executeAt, partialDeps));
+ break;
+
+ case PreCommitted:
+ Commands.precommit(safeStore, safeCommand, txnId, full.executeAt, route);
+ if (!achieved.definition.isKnown())
+ break;
+
+ case PreAccepted:
+ // only preaccept if we coordinate the transaction
+ if (safeStore.ranges().coordinates(txnId).intersects(route) && Route.isFullRoute(route))
+ Commands.preaccept(safeStore, safeCommand, txnId, txnId.epoch(), partialTxn, Route.castToFullRoute(route), progressKey);
+ break;
+
+ case NotDefined:
+ break;
+ }
+
+ RoutingKey homeKey = full.homeKey;
+ if (!full.durability.isDurable() || homeKey == null)
+ return null;
+
+ if (!safeStore.ranges().coordinates(txnId).contains(homeKey))
+ return null;
+
+ Timestamp executeAt = full.saveStatus.known.executeAt.hasDecidedExecuteAt() ? full.executeAt : null;
+ Commands.setDurability(safeStore, safeCommand, full.durability, route, executeAt);
+ return null;
+ }
+
+ @Override
+ public Void reduce(Void o1, Void o2)
+ {
+ return null;
+ }
+
+ @Override
+ public void accept(Void result, Throwable failure)
+ {
+ callback.accept(failure == null ? achieved : null, failure);
+ }
+
+ @Override
+ public long epoch()
+ {
+ return toEpoch;
+ }
+ }
+
+ private static void confirm(Commands.CommitOutcome outcome)
+ {
+ switch (outcome)
+ {
+ default: throw new IllegalStateException("Unknown outcome: " + outcome);
+ case Redundant:
+ case Success:
+ return;
+ case Insufficient: throw new IllegalStateException("Should have enough information");
+ }
}
+
+ private static void confirm(Commands.ApplyOutcome outcome)
+ {
+ switch (outcome)
+ {
+ default: throw new IllegalStateException("Unknown outcome: " + outcome);
+ case Redundant:
+ case Success:
+ return;
+ case Insufficient: throw new IllegalStateException("Should have enough information");
+ }
+ }
+
}
diff --git a/accord-core/src/main/java/accord/coordinate/FindRoute.java b/accord-core/src/main/java/accord/coordinate/FindRoute.java
index 986406d7..f85ba888 100644
--- a/accord-core/src/main/java/accord/coordinate/FindRoute.java
+++ b/accord-core/src/main/java/accord/coordinate/FindRoute.java
@@ -20,7 +20,6 @@ package accord.coordinate;
import java.util.function.BiConsumer;
-import accord.api.RoutingKey;
import accord.local.Node;
import accord.local.Status;
import accord.messages.CheckStatus.CheckStatusOk;
@@ -32,7 +31,7 @@ import static accord.primitives.Route.isFullRoute;
/**
* Find the Route of a known (txnId, homeKey) pair
*/
-public class FindRoute extends CheckShards
+public class FindRoute extends CheckShards<Route<?>>
{
public static class Result
{
@@ -53,15 +52,15 @@ public class FindRoute extends CheckShards
}
final BiConsumer<Result, Throwable> callback;
- FindRoute(Node node, TxnId txnId, RoutingKey homeKey, BiConsumer<Result, Throwable> callback)
+ FindRoute(Node node, TxnId txnId, Route<?> someRoute, BiConsumer<Result, Throwable> callback)
{
- super(node, txnId, RoutingKeys.of(homeKey), txnId.epoch(), IncludeInfo.Route);
+ super(node, txnId, someRoute, IncludeInfo.Route);
this.callback = callback;
}
- public static FindRoute findRoute(Node node, TxnId txnId, RoutingKey homeKey, BiConsumer<Result, Throwable> callback)
+ public static FindRoute findRoute(Node node, TxnId txnId, Route<?> someRoute, BiConsumer<Result, Throwable> callback)
{
- FindRoute findRoute = new FindRoute(node, txnId, homeKey, callback);
+ FindRoute findRoute = new FindRoute(node, txnId, someRoute, callback);
findRoute.start();
return findRoute;
}
diff --git a/accord-core/src/main/java/accord/coordinate/FindHomeKey.java b/accord-core/src/main/java/accord/coordinate/FindSomeRoute.java
similarity index 52%
rename from accord-core/src/main/java/accord/coordinate/FindHomeKey.java
rename to accord-core/src/main/java/accord/coordinate/FindSomeRoute.java
index 02312d58..2babeac7 100644
--- a/accord-core/src/main/java/accord/coordinate/FindHomeKey.java
+++ b/accord-core/src/main/java/accord/coordinate/FindSomeRoute.java
@@ -20,30 +20,48 @@ package accord.coordinate;
import java.util.function.BiConsumer;
-import accord.api.RoutingKey;
import accord.local.Node;
+import accord.local.Status.Known;
import accord.messages.CheckStatus.CheckStatusOk;
import accord.messages.CheckStatus.IncludeInfo;
+import accord.messages.CheckStatus.WithQuorum;
+import accord.primitives.Route;
import accord.primitives.Unseekables;
import accord.primitives.TxnId;
+import static accord.local.Status.Known.Nothing;
+
/**
* Find the homeKey of a txnId with some known keys
*/
-public class FindHomeKey extends CheckShards
+public class FindSomeRoute extends CheckShards<Unseekables<?>>
{
- final BiConsumer<RoutingKey, Throwable> callback;
- FindHomeKey(Node node, TxnId txnId, Unseekables<?, ?> unseekables, BiConsumer<RoutingKey, Throwable> callback)
+ static class Result
+ {
+ public final Route<?> route;
+ public final Known known;
+ public final WithQuorum withQuorum;
+
+ Result(Route<?> route, Known known, WithQuorum withQuorum)
+ {
+ this.route = route;
+ this.known = known;
+ this.withQuorum = withQuorum;
+ }
+ }
+
+ final BiConsumer<Result, Throwable> callback;
+ FindSomeRoute(Node node, TxnId txnId, Unseekables<?> unseekables, BiConsumer<Result, Throwable> callback)
{
- super(node, txnId, unseekables, txnId.epoch(), IncludeInfo.No);
+ super(node, txnId, unseekables, IncludeInfo.Route);
this.callback = callback;
}
- public static FindHomeKey findHomeKey(Node node, TxnId txnId, Unseekables<?, ?> unseekables, BiConsumer<RoutingKey, Throwable> callback)
+ public static FindSomeRoute findSomeRoute(Node node, TxnId txnId, Unseekables<?> unseekables, BiConsumer<Result, Throwable> callback)
{
- FindHomeKey findHomeKey = new FindHomeKey(node, txnId, unseekables, callback);
- findHomeKey.start();
- return findHomeKey;
+ FindSomeRoute findSomeRoute = new FindSomeRoute(node, txnId, unseekables, callback);
+ findSomeRoute.start();
+ return findSomeRoute;
}
@Override
@@ -56,6 +74,7 @@ public class FindHomeKey extends CheckShards
protected void onDone(Success success, Throwable failure)
{
if (failure != null) callback.accept(null, failure);
- else callback.accept(merged == null ? null : merged.homeKey, null);
+ else if (merged == null) callback.accept(new Result(null, Nothing, success.withQuorum), null);
+ else callback.accept(new Result(merged.route, merged.inferredOrKnown(success.withQuorum), success.withQuorum), null);
}
}
diff --git a/accord-core/src/main/java/accord/coordinate/Infer.java b/accord-core/src/main/java/accord/coordinate/Infer.java
new file mode 100644
index 00000000..dc42d834
--- /dev/null
+++ b/accord-core/src/main/java/accord/coordinate/Infer.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package accord.coordinate;
+
+import java.util.function.BiConsumer;
+
+import javax.annotation.Nullable;
+
+import accord.local.Command;
+import accord.local.Commands;
+import accord.local.Node;
+import accord.local.PreLoadContext;
+import accord.local.SafeCommand;
+import accord.local.SafeCommandStore;
+import accord.local.SaveStatus;
+import accord.local.Status;
+import accord.messages.CheckStatus;
+import accord.primitives.Participants;
+import accord.primitives.Ranges;
+import accord.primitives.Route;
+import accord.primitives.Timestamp;
+import accord.primitives.TxnId;
+import accord.primitives.Unseekables;
+import accord.utils.Invariants;
+import accord.utils.MapReduceConsume;
+
+import static accord.local.PreLoadContext.contextFor;
+import static accord.local.Status.Durability.Majority;
+import static accord.local.Status.PreAccepted;
+import static accord.local.Status.PreApplied;
+import static accord.local.Status.PreCommitted;
+import static accord.messages.CheckStatus.WithQuorum.HasQuorum;
+import static accord.primitives.Route.castToRoute;
+import static accord.primitives.Route.isRoute;
+
+public class Infer
+{
+ private static abstract class CleanupAndCallback<T> implements MapReduceConsume<SafeCommandStore, Void>
+ {
+ final Node node;
+ final TxnId txnId;
+ final Unseekables<?> someUnseekables;
+ final T param;
+ final BiConsumer<T, Throwable> callback;
+
+ private CleanupAndCallback(Node node, TxnId txnId, Unseekables<?> someUnseekables, T param, BiConsumer<T, Throwable> callback)
+ {
+ this.node = node;
+ this.txnId = txnId;
+ this.someUnseekables = someUnseekables;
+ this.param = param;
+ this.callback = callback;
+ }
+
+ void start()
+ {
+ PreLoadContext loadContext = contextFor(txnId);
+ Unseekables<?> propagateTo = isRoute(someUnseekables) ? castToRoute(someUnseekables).withHomeKey() : someUnseekables;
+ node.mapReduceConsumeLocal(loadContext, propagateTo, txnId.epoch(), txnId.epoch(), this);
+ }
+
+ @Override
+ public Void apply(SafeCommandStore safeStore)
+ {
+ // we're applying an invalidation, so the record will not be cleaned up until the whole range is truncated
+ return apply(safeStore, safeStore.get(txnId, txnId, someUnseekables));
+ }
+
+ abstract Void apply(SafeCommandStore safeStore, SafeCommand safeCommand);
+
+ @Override
+ public Void reduce(Void o1, Void o2)
+ {
+ return null;
+ }
+
+ @Override
+ public void accept(Void result, Throwable failure)
+ {
+ callback.accept(param, failure);
+ }
+ }
+
+ static class InvalidateAndCallback<T> extends CleanupAndCallback<T>
+ {
+ private InvalidateAndCallback(Node node, TxnId txnId, Unseekables<?> someUnseekables, T param, BiConsumer<T, Throwable> callback)
+ {
+ super(node, txnId, someUnseekables, param, callback);
+ }
+
+ public static <T> void locallyInvalidateAndCallback(Node node, TxnId txnId, Unseekables<?> someUnseekables, T param, BiConsumer<T, Throwable> callback)
+ {
+ new InvalidateAndCallback<T>(node, txnId, someUnseekables, param, callback).start();
+ }
+
+ @Override
+ Void apply(SafeCommandStore safeStore, SafeCommand safeCommand)
+ {
+ // we're applying an invalidation, so the record will not be cleaned up until the whole range is truncated
+ Command command = safeCommand.current();
+ // TODO (required): consider the !command.hasBeen(PreCommitted) condition
+ Invariants.checkState(!command.hasBeen(PreCommitted) || command.hasBeen(Status.Truncated));
+ Commands.commitInvalidate(safeStore, safeCommand, someUnseekables);
+ return null;
+ }
+ }
+
+ static class EraseNonParticipatingAndCallback<T> extends CleanupAndCallback<T>
+ {
+ private EraseNonParticipatingAndCallback(Node node, TxnId txnId, Unseekables<?> someUnseekables, T param, BiConsumer<T, Throwable> callback)
+ {
+ super(node, txnId, someUnseekables, param, callback);
+ }
+
+ public static <T> void eraseNonParticipatingAndCallback(Node node, TxnId txnId, Unseekables<?> someUnseekables, T param, BiConsumer<T, Throwable> callback)
+ {
+ if (!Route.isRoute(someUnseekables)) callback.accept(param, null);
+ else new EraseNonParticipatingAndCallback<>(node, txnId, someUnseekables, param, callback).start();
+ }
+
+ @Override
+ Void apply(SafeCommandStore safeStore, SafeCommand safeCommand)
+ {
+ // we're applying an invalidation, so the record will not be cleaned up until the whole range is truncated
+ Command command = safeCommand.current();
+ if (!command.hasBeen(PreApplied) && safeToCleanup(safeStore, command, Route.castToRoute(someUnseekables), null))
+ Commands.setErased(safeStore, safeCommand);
+ return null;
+ }
+ }
+
+ public static Status invalidIfNotAtLeast(SafeCommandStore safeStore, TxnId txnId, Unseekables<?> query)
+ {
+ if (safeStore.commandStore().globalDurability(txnId).compareTo(Majority) >= 0)
+ {
+ Unseekables<?> preacceptsWith = isRoute(query) ? castToRoute(query).withHomeKey() : query;
+ return safeStore.commandStore().isRejectedIfNotPreAccepted(txnId, preacceptsWith) ? PreAccepted : PreCommitted;
+ }
+
+ // TODO (expected, consider): should we force this to be a Route or a Participants?
+ if (isRoute(query))
+ {
+ Participants<?> participants = castToRoute(query).participants();
+ // TODO (desired): limit to local participants to avoid O(n2) work across cluster
+ if (safeStore.commandStore().durableBefore().isSomeShardDurable(txnId, participants, Majority))
+ return PreCommitted;
+ }
+
+ if (safeStore.commandStore().durableBefore().isUniversal(txnId, safeStore.ranges().allAt(txnId.epoch())))
+ return PreCommitted;
+
+ return Status.NotDefined;
+ }
+
+ public static boolean inferInvalidated(CheckStatus.WithQuorum withQuorum, Status invalidIfNotAtLeast, SaveStatus saveStatus, SaveStatus maxSaveStatus)
+ {
+ if (saveStatus == SaveStatus.Invalidated)
+ return true;
+
+ if (withQuorum != HasQuorum)
+ return false;
+
+ // should not be possible to reach a quorum without finding the definition unless cleanup is in progress
+ Invariants.checkState(saveStatus != SaveStatus.Accepted || maxSaveStatus.phase == Status.Phase.Cleanup);
+
+ if (saveStatus.status.compareTo(invalidIfNotAtLeast) < 0)
+ return true;
+
+ return invalidIfNotAtLeast.compareTo(PreAccepted) >= 0 && saveStatus == SaveStatus.AcceptedInvalidate;
+ }
+
+ public static boolean safeToCleanup(SafeCommandStore safeStore, Command command, Route<?> fetchedWith, @Nullable Timestamp executeAt)
+ {
+ Invariants.checkArgument(fetchedWith != null || command.route() != null);
+ TxnId txnId = command.txnId();
+ if (command.is(Status.NotDefined))
+ return !command.saveStatus().isUninitialised();
+
+ if (command.route() != null || fetchedWith.covers(safeStore.ranges().allAt(txnId.epoch())))
+ {
+ Route<?> route = command.route();
+ if (route == null) route = fetchedWith;
+
+ executeAt = command.executeAtIfKnown(Timestamp.nonNullOrMax(executeAt, txnId));
+ Ranges coordinateRanges = safeStore.ranges().coordinates(txnId);
+ Ranges acceptRanges = executeAt.epoch() == txnId.epoch() ? coordinateRanges : safeStore.ranges().allBetween(txnId, executeAt);
+ return !route.participatesIn(coordinateRanges) && !route.participatesIn(acceptRanges);
+ }
+ return false;
+ }
+}
diff --git a/accord-core/src/main/java/accord/coordinate/InformHomeOfTxn.java b/accord-core/src/main/java/accord/coordinate/InformHomeOfTxn.java
index dc6499cf..4f392009 100644
--- a/accord-core/src/main/java/accord/coordinate/InformHomeOfTxn.java
+++ b/accord-core/src/main/java/accord/coordinate/InformHomeOfTxn.java
@@ -18,13 +18,13 @@
package accord.coordinate;
-import accord.api.RoutingKey;
import accord.coordinate.tracking.QuorumTracker.QuorumShardTracker;
import accord.local.Node;
import accord.local.Node.Id;
import accord.messages.Callback;
import accord.messages.InformOfTxnId;
import accord.messages.SimpleReply;
+import accord.primitives.Route;
import accord.topology.Shard;
import accord.primitives.TxnId;
import accord.utils.async.AsyncChain;
@@ -36,23 +36,23 @@ import static accord.coordinate.tracking.AbstractTracker.ShardOutcomes.Success;
public class InformHomeOfTxn extends AsyncResults.SettableResult<Void> implements Callback<SimpleReply>
{
final TxnId txnId;
- final RoutingKey homeKey;
+ final Route<?> someRoute;
final QuorumShardTracker tracker;
Throwable failure;
- InformHomeOfTxn(TxnId txnId, RoutingKey homeKey, Shard homeShard)
+ InformHomeOfTxn(TxnId txnId, Route<?> someRoute, Shard homeShard)
{
this.txnId = txnId;
- this.homeKey = homeKey;
+ this.someRoute = someRoute;
this.tracker = new QuorumShardTracker(homeShard);
}
- public static AsyncChain<Void> inform(Node node, TxnId txnId, RoutingKey homeKey)
+ public static AsyncChain<Void> inform(Node node, TxnId txnId, Route<?> someRoute)
{
return node.withEpoch(txnId.epoch(), () -> {
- Shard homeShard = node.topology().forEpoch(homeKey, txnId.epoch());
- InformHomeOfTxn inform = new InformHomeOfTxn(txnId, homeKey, homeShard);
- node.send(homeShard.nodes, new InformOfTxnId(txnId, homeKey), inform);
+ Shard homeShard = node.topology().forEpoch(someRoute.homeKey(), txnId.epoch());
+ InformHomeOfTxn inform = new InformHomeOfTxn(txnId, someRoute, homeShard);
+ node.send(homeShard.nodes, new InformOfTxnId(txnId, someRoute), inform);
return inform;
});
}
diff --git a/accord-core/src/main/java/accord/coordinate/Invalidate.java b/accord-core/src/main/java/accord/coordinate/Invalidate.java
index c1bdf74f..d6b4e162 100644
--- a/accord-core/src/main/java/accord/coordinate/Invalidate.java
+++ b/accord-core/src/main/java/accord/coordinate/Invalidate.java
@@ -28,14 +28,12 @@ import accord.coordinate.tracking.InvalidationTracker;
import accord.coordinate.tracking.InvalidationTracker.InvalidationShardTracker;
import accord.coordinate.tracking.RequestStatus;
import accord.local.Node.Id;
-import accord.local.Status;
import accord.messages.Commit;
import accord.local.*;
import accord.primitives.*;
import accord.topology.Topologies;
import accord.api.RoutingKey;
-import accord.local.Node;
import accord.messages.BeginInvalidation;
import accord.messages.BeginInvalidation.InvalidateReply;
import accord.messages.Callback;
@@ -46,14 +44,16 @@ import javax.annotation.Nullable;
import static accord.coordinate.Propose.Invalidate.proposeInvalidate;
import static accord.local.PreLoadContext.contextFor;
import static accord.local.Status.Accepted;
+import static accord.local.Status.PreAccepted;
import static accord.primitives.ProgressToken.INVALIDATED;
+import static accord.primitives.ProgressToken.TRUNCATED;
public class Invalidate implements Callback<InvalidateReply>
{
private final Node node;
private final Ballot ballot;
private final TxnId txnId;
- private final Unseekables<?, ?> invalidateWith;
+ private final Unseekables<?> invalidateWith;
private final BiConsumer<Outcome, Throwable> callback;
private boolean isDone;
@@ -64,7 +64,7 @@ public class Invalidate implements Callback<InvalidateReply>
private Throwable failure;
private final Map<Id, InvalidateReply> debug = Invariants.debug() ? new HashMap<>() : null;
- private Invalidate(Node node, Ballot ballot, TxnId txnId, Unseekables<?, ?> invalidateWith, boolean transitivelyInvokedByPriorInvalidation, BiConsumer<Outcome, Throwable> callback)
+ private Invalidate(Node node, Ballot ballot, TxnId txnId, Unseekables<?> invalidateWith, boolean transitivelyInvokedByPriorInvalidation, BiConsumer<Outcome, Throwable> callback)
{
this.callback = callback;
this.node = node;
@@ -76,12 +76,12 @@ public class Invalidate implements Callback<InvalidateReply>
this.tracker = new InvalidationTracker(topologies);
}
- public static Invalidate invalidate(Node node, TxnId txnId, Unseekables<?, ?> invalidateWith, BiConsumer<Outcome, Throwable> callback)
+ public static Invalidate invalidate(Node node, TxnId txnId, Unseekables<?> invalidateWith, BiConsumer<Outcome, Throwable> callback)
{
return invalidate(node, txnId, invalidateWith, false, callback);
}
- public static Invalidate invalidate(Node node, TxnId txnId, Unseekables<?, ?> invalidateWith, boolean transitivelyInvokedByPriorInvalidation, BiConsumer<Outcome, Throwable> callback)
+ public static Invalidate invalidate(Node node, TxnId txnId, Unseekables<?> invalidateWith, boolean transitivelyInvokedByPriorInvalidation, BiConsumer<Outcome, Throwable> callback)
{
Ballot ballot = new Ballot(node.uniqueNow());
Invalidate invalidate = new Invalidate(node, ballot, txnId, invalidateWith, transitivelyInvokedByPriorInvalidation, callback);
@@ -102,7 +102,7 @@ public class Invalidate implements Callback<InvalidateReply>
if (debug != null) debug.put(from, reply);
replies.add(reply);
- handle(tracker.recordSuccess(from, reply.isPromised(), reply.acceptedFastPath));
+ handle(tracker.recordSuccess(from, reply.isPromised(), reply.hasDecision(), reply.acceptedFastPath));
}
@Override
@@ -149,77 +149,57 @@ public class Invalidate implements Callback<InvalidateReply>
// first look to see if it has already been
{
- FullRoute<?> route = InvalidateReply.findRoute(replies);
- RoutingKey homeKey = route != null ? route.homeKey() : InvalidateReply.findHomeKey(replies);
+ FullRoute<?> fullRoute = InvalidateReply.findRoute(replies);
+ Route<?> someRoute = InvalidateReply.mergeRoutes(replies);
InvalidateReply maxReply = InvalidateReply.max(replies);
switch (maxReply.status)
{
default: throw new IllegalStateException();
+ case Truncated:
+ isDone = true;
+ callback.accept(TRUNCATED, null);
+ return;
+
case AcceptedInvalidate:
// latest accept also invalidating, so we're on the same page and should finish our invalidation
- case NotWitnessed:
+ case NotDefined:
break;
case PreAccepted:
if (tracker.isSafeToInvalidate() || transitivelyInvokedByPriorInvalidation)
break;
- case Accepted:
- case PreCommitted:
- case Committed:
- case ReadyToExecute:
- case PreApplied:
case Applied:
+ case PreApplied:
+ case ReadyToExecute:
+ case Committed:
+ case PreCommitted:
+ Invariants.checkState(maxReply.status == PreAccepted || !invalidateWith.contains(someRoute.homeKey()) || fullRoute != null);
+
+ case Accepted:
// TODO (desired, efficiency): if we see Committed or above, go straight to Execute if we have assembled enough information
- if (route != null)
- {
- // The data we see might have made it only to a minority in the event of PreAccept ONLY.
- // We want to protect against infinite loops, so we inform the recovery of the state we have
- // witnessed during our initial invalidation.
-
- // However, if the state is not guaranteed to be recoverable (i.e. PreAccept/NotWitnessed),
- // we do not relay this information unless we can guarantee that any shard recovery may contact
- // has been prevented from reaching a _later_ fast-path decision by our promises.
- // Which means checking we contacted every shard, since we only reach this point if we have promises
- // from every shard we contacted.
-
- // Note that there's lots of scope for variations in behaviour here, but lots of care is needed.
-
- Status witnessedByInvalidation = maxReply.status;
- if (!witnessedByInvalidation.hasBeen(Accepted))
- {
- Invariants.checkState(tracker.all(InvalidationShardTracker::isPromised));
- if (!invalidateWith.containsAll(route))
- witnessedByInvalidation = null;
- }
- RecoverWithRoute.recover(node, ballot, txnId, route, witnessedByInvalidation, callback);
- }
- else if (homeKey != null)
- {
- Invariants.checkState(maxReply.status.hasBeen(Accepted) || tracker.all(InvalidationShardTracker::isPromised));
- // if we included the home shard, and we have either a recoverable status OR have not rejected the fast path,
- // we must have at least one response that should contain the Route
- if (invalidateWith.contains(homeKey) && tracker.isPromisedForKey(homeKey, txnId.epoch()))
- throw new IllegalStateException("Received replies from a node that must have known the route, but that did not include it");
-
- // if < Accepted, we should have short-circuited to invalidation above. This guarantees no Invaldate/Recover loop, as any later status will forbid invoking Invalidate
- Invariants.checkState(!(transitivelyInvokedByPriorInvalidation && !maxReply.status.hasBeen(Accepted)));
-
- Status witnessedByInvalidation = maxReply.status;
- if (!witnessedByInvalidation.hasBeen(Accepted))
- {
- Invariants.checkState(tracker.all(InvalidationShardTracker::isPromised));
- if (!invalidateWith.contains(homeKey))
- witnessedByInvalidation = null;
- }
-
- RecoverWithHomeKey.recover(node, txnId, homeKey, witnessedByInvalidation, callback);
- }
- else
+ Invariants.checkState(fullRoute != null, "Received a reply from a node that must have known some route, but that did not include it"); // we now require the FullRoute on all replicas to preaccept, commit or apply
+ // The data we see might have made it only to a minority in the event of PreAccept ONLY.
+ // We want to protect against infinite loops, so we inform the recovery of the state we have
+ // witnessed during our initial invalidation.
+
+ // However, if the state is not guaranteed to be recoverable (i.e. PreAccept/NotWitnessed),
+ // we do not relay this information unless we can guarantee that any shard recovery may contact
+ // has been prevented from reaching a _later_ fast-path decision by our promises.
+ // Which means checking we contacted every shard, since we only reach that point if we have promises
+ // from every shard we contacted.
+
+ // Note that there's lots of scope for variations in behaviour here, but lots of care is needed.
+
+ Status witnessedByInvalidation = maxReply.status;
+ if (!witnessedByInvalidation.hasBeen(Accepted))
{
- throw new IllegalStateException("Received a reply from a node that must have known the homeKey, but that did not include it");
+ Invariants.checkState(tracker.all(InvalidationShardTracker::isPromised));
+ if (!invalidateWith.containsAll(fullRoute))
+ witnessedByInvalidation = null;
}
+ RecoverWithRoute.recover(node, ballot, txnId, fullRoute, witnessedByInvalidation, callback);
return;
case Invalidated:
@@ -234,8 +214,8 @@ public class Invalidate implements Callback<InvalidateReply>
// Probably simplest to do so, but perhaps better for user if we don't.
Ranges ranges = Ranges.of(tracker.promisedShard().range);
// we look up by TxnId at the target node, so it's fine to pick a RoutingKey even if it's a range transaction
- RoutingKey invalidateWithKey = invalidateWith.slice(ranges).get(0).someIntersectingRoutingKey(ranges);
- proposeInvalidate(node, ballot, txnId, invalidateWithKey, (success, fail) -> {
+ RoutingKey someKey = invalidateWith.slice(ranges).get(0).someIntersectingRoutingKey(ranges);
+ proposeInvalidate(node, ballot, txnId, someKey, (success, fail) -> {
/*
We're now inside our *exactly once* callback we registered with proposeInvalidate, and we need to
make sure we honour our own exactly once semantics with {@code callback}.
@@ -260,15 +240,27 @@ public class Invalidate implements Callback<InvalidateReply>
});
}
+ @SuppressWarnings({"unchecked", "rawtypes"})
private void commitInvalidate()
{
@Nullable Route<?> route = InvalidateReply.mergeRoutes(replies);
+ if (route == null && Route.isRoute(invalidateWith)) route = Route.castToRoute(invalidateWith);
+ if (route != null) route = route.withHomeKey();
+
// TODO (desired, efficiency): commitInvalidate (and others) should skip the network for local applications,
// so we do not need to explicitly do so here before notifying the waiter
- Commit.Invalidate.commitInvalidate(node, txnId, route != null ? Unseekables.merge(route, (Unseekables)invalidateWith) : invalidateWith, txnId);
+ Unseekables<?> commitTo = Unseekables.merge(route, (Unseekables) invalidateWith);
+ Commit.Invalidate.commitInvalidate(node, txnId, commitTo, txnId);
+ commitInvalidateLocal(commitTo);
+ }
+
+ private void commitInvalidateLocal(Unseekables<?> commitTo)
+ {
+ // TODO (desired): merge with FetchData.InvalidateOnDone
+ // TODO (desired): when sending to network, register a callback for when local application of commitInvalidate message ahs been performed, so no need to special-case
// TODO (required, consider): pick a reasonable upper bound, so we don't invalidate into an epoch/commandStore that no longer cares about this command
- node.forEachLocalSince(contextFor(txnId), invalidateWith, txnId, safeStore -> {
- Commands.commitInvalidate(safeStore, txnId);
+ node.forEachLocal(contextFor(txnId), commitTo, txnId.epoch(), txnId.epoch(), safeStore -> {
+ Commands.commitInvalidate(safeStore, safeStore.get(txnId, txnId, commitTo), commitTo);
}).begin((s, f) -> {
callback.accept(INVALIDATED, null);
if (f != null) // TODO (required): consider exception handling more carefully: should we catch these prior to passing to callbacks?
diff --git a/accord-core/src/main/java/accord/coordinate/MaybeRecover.java b/accord-core/src/main/java/accord/coordinate/MaybeRecover.java
index 457593db..ab4ea919 100644
--- a/accord-core/src/main/java/accord/coordinate/MaybeRecover.java
+++ b/accord-core/src/main/java/accord/coordinate/MaybeRecover.java
@@ -20,51 +20,46 @@ package accord.coordinate;
import java.util.function.BiConsumer;
-import javax.annotation.Nullable;
-
import accord.local.Status.Known;
+import accord.messages.CheckStatus.WithQuorum;
import accord.primitives.*;
import accord.utils.Invariants;
-import accord.api.RoutingKey;
import accord.local.Node;
import accord.messages.CheckStatus.CheckStatusOk;
import accord.messages.CheckStatus.IncludeInfo;
+import static accord.coordinate.Infer.EraseNonParticipatingAndCallback.eraseNonParticipatingAndCallback;
+import static accord.coordinate.Infer.InvalidateAndCallback.locallyInvalidateAndCallback;
import static accord.utils.Functions.reduceNonNull;
/**
* A result of null indicates the transaction is globally persistent
* A result of CheckStatusOk indicates the maximum status found for the transaction, which may be used to assess progress
*/
-public class MaybeRecover extends CheckShards
+public class MaybeRecover extends CheckShards<Route<?>>
{
- @Nullable final Route<?> route;
- final RoutingKey homeKey;
final ProgressToken prevProgress;
final BiConsumer<Outcome, Throwable> callback;
- MaybeRecover(Node node, TxnId txnId, RoutingKey homeKey, @Nullable Route<?> route, ProgressToken prevProgress, BiConsumer<Outcome, Throwable> callback)
+ MaybeRecover(Node node, TxnId txnId, Route<?> someRoute, ProgressToken prevProgress, BiConsumer<Outcome, Throwable> callback)
{
// we only want to enquire with the home shard, but we prefer maximal route information for running Invalidation against, if necessary
- super(node, txnId, RoutingKeys.of(homeKey), txnId.epoch(), IncludeInfo.Route);
- this.homeKey = homeKey;
- this.route = route;
+ super(node, txnId, someRoute.withHomeKey(), IncludeInfo.Route);
this.prevProgress = prevProgress;
this.callback = callback;
}
- public static void maybeRecover(Node node, TxnId txnId, RoutingKey homeKey, @Nullable Route<?> route,
- ProgressToken prevProgress, BiConsumer<Outcome, Throwable> callback)
+ public static void maybeRecover(Node node, TxnId txnId, Route<?> someRoute, ProgressToken prevProgress, BiConsumer<Outcome, Throwable> callback)
{
- MaybeRecover maybeRecover = new MaybeRecover(node, txnId, homeKey, route, prevProgress, callback);
+ MaybeRecover maybeRecover = new MaybeRecover(node, txnId, someRoute, prevProgress, callback);
maybeRecover.start();
}
@Override
protected boolean isSufficient(CheckStatusOk ok)
{
- return hasMadeProgress(ok);
+ return hasMadeProgress(ok) || ok.durability.isDurableOrInvalidated();
}
public boolean hasMadeProgress(CheckStatusOk ok)
@@ -73,6 +68,7 @@ public class MaybeRecover extends CheckShards
|| ok.toProgressToken().compareTo(prevProgress) > 0);
}
+ @SuppressWarnings({"unchecked", "rawtypes"})
@Override
protected void onDone(Success success, Throwable fail)
{
@@ -84,32 +80,44 @@ public class MaybeRecover extends CheckShards
{
Invariants.checkState(merged != null);
Known known = merged.saveStatus.known;
+ Route<?> someRoute = reduceNonNull(Route::union, (Route)this.route, merged.route);
switch (known.outcome)
{
default: throw new AssertionError();
- case OutcomeUnknown:
- if (!known.isDefinitionKnown() && !Route.isFullRoute(merged.route))
+ case Unknown:
+ if (known.canProposeInvalidation() && !Route.isFullRoute(merged.route))
{
- // order important, as route could be a Route which does not implement RoutingKeys.union
- Unseekables<?, ?> someKeys = reduceNonNull(Unseekables::merge, (Unseekables)this.contact, merged.route, route);
// for correctness reasons, we have not necessarily preempted the initial pre-accept round and
// may have raced with it, so we must attempt to recover anything we see pre-accepted.
- Invalidate.invalidate(node, txnId, someKeys.with(homeKey), callback);
+ Invalidate.invalidate(node, txnId, someRoute, callback);
break;
}
- case OutcomeKnown:
- case OutcomeApplied:
+
+ case Apply:
// we have included the home key, and one that witnessed the definition has responded, so it should also know the full route
- Invariants.checkState(Route.isFullRoute(merged.route));
- if (hasMadeProgress(merged)) callback.accept(merged.toProgressToken(), null);
- else node.recover(txnId, Route.castToFullRoute(merged.route)).addCallback(callback);
+ if (hasMadeProgress(merged))
+ {
+ callback.accept(merged.toProgressToken(), null);
+ }
+ else
+ {
+ Invariants.checkState(Route.isFullRoute(merged.route), "Require a full route but given %s", merged.route);
+ node.recover(txnId, Route.castToFullRoute(merged.route)).addCallback(callback);
+ }
+ break;
+
+ case WasApply:
+ callback.accept(merged.toProgressToken(), null);
break;
- case InvalidationApplied:
- // TODO (easy, efficiency): we should simply invoke commitInvalidate
- Unseekables<?, ?> someKeys = reduceNonNull(Unseekables::merge, (Unseekables)contact, merged.route, route);
- Invalidate.invalidate(node, txnId, someKeys.with(homeKey), callback);
+ case Invalidated:
+ locallyInvalidateAndCallback(node, txnId, someRoute, merged.toProgressToken(), callback);
+
+ case Erased:
+ WithQuorum withQuorum = success.withQuorum;
+ if (!merged.inferInvalidated(withQuorum)) eraseNonParticipatingAndCallback(node, txnId, someRoute, merged.toProgressToken(), callback);
+ else locallyInvalidateAndCallback(node, txnId, someRoute, merged.toProgressToken(), callback);
}
}
}
diff --git a/accord-core/src/main/java/accord/coordinate/Persist.java b/accord-core/src/main/java/accord/coordinate/Persist.java
index 28cdec31..0607722f 100644
--- a/accord-core/src/main/java/accord/coordinate/Persist.java
+++ b/accord-core/src/main/java/accord/coordinate/Persist.java
@@ -29,16 +29,14 @@ import accord.messages.Apply;
import accord.messages.Apply.ApplyReply;
import accord.messages.Callback;
import accord.messages.Commit;
-import accord.messages.InformHomeDurable;
-import accord.primitives.Deps;
-import accord.primitives.Txn;
+import accord.messages.InformDurable;
import accord.primitives.*;
-import accord.topology.Shard;
import accord.topology.Topologies;
import static accord.coordinate.tracking.RequestStatus.Success;
-import static accord.local.Status.Durability.Durable;
-import static accord.local.Status.Durability.Universal;
+import static accord.local.Status.Durability.Majority;
+import static accord.messages.Apply.executes;
+import static accord.messages.Apply.participates;
import static accord.messages.Commit.Kind.Maximal;
public class Persist implements Callback<ApplyReply>
@@ -53,22 +51,33 @@ public class Persist implements Callback<ApplyReply>
final Set<Id> persistedOn;
boolean isDone;
- // persistTo should be a superset of applyTo, and includes those replicas/ranges that no longer replicate at the execution epoch
- // but did replicate for coordination and would like to be informed of the transaction's status (i.e. apply a no-op apply)
- public static void persist(Node node, Topologies persistTo, Topologies appliesTo, TxnId txnId, FullRoute<?> route, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result)
+ public static void persist(Node node, TxnId txnId, FullRoute<?> route, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result)
{
- Persist persist = new Persist(node, appliesTo, txnId, route, txn, executeAt, deps);
- node.send(persistTo.nodes(), to -> new Apply(to, persistTo, appliesTo, executeAt.epoch(), txnId, route, txn, executeAt, deps, writes, result), persist);
+ Topologies executes = executes(node, route, executeAt);
+ persist(node, executes, txnId, route, txn, executeAt, deps, writes, result);
}
- public static void persistAndCommitMaximal(Node node, TxnId txnId, FullRoute<?> route, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result)
+ public static void persist(Node node, Topologies executes, TxnId txnId, FullRoute<?> route, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result)
{
- Topologies coordinate = node.topology().forEpoch(route, txnId.epoch());
- Topologies applyTo = node.topology().forEpoch(route, executeAt.epoch());
- Topologies persistTo = txnId.epoch() == executeAt.epoch() ? applyTo : node.topology().preciseEpochs(route, txnId.epoch(), executeAt.epoch());
- Persist persist = new Persist(node, persistTo, txnId, route, txn, executeAt, deps);
- node.send(persistTo.nodes(), to -> new Commit(Maximal, to, coordinate.current(), persistTo, txnId, txn, route, null, executeAt, deps, false), persist);
- node.send(applyTo.nodes(), to -> new Apply(to, persistTo, applyTo, executeAt.epoch(), txnId, route, txn, executeAt, deps, writes, result), persist);
+ Topologies participates = participates(node, route, txnId, executeAt, executes);
+ Persist persist = new Persist(node, executes, txnId, route, txn, executeAt, deps);
+ node.send(participates.nodes(), to -> Apply.applyMinimal(to, participates, executes, txnId, route, txn, executeAt, deps, writes, result), persist);
+ }
+
+ public static void persistMaximal(Node node, TxnId txnId, FullRoute<?> route, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result)
+ {
+ Topologies executes = executes(node, route, executeAt);
+ Topologies participates = participates(node, route, txnId, executeAt, executes);
+ Persist persist = new Persist(node, participates, txnId, route, txn, executeAt, deps);
+ node.send(participates.nodes(), to -> Apply.applyMaximal(to, participates, executes, txnId, route, txn, executeAt, deps, writes, result), persist);
+ }
+
+ public static void persistPartialMaximal(Node node, TxnId txnId, Unseekables<?> sendTo, FullRoute<?> route, PartialTxn txn, Timestamp executeAt, Deps deps, Writes writes, Result result)
+ {
+ Topologies executes = executes(node, sendTo, executeAt);
+ Topologies participates = participates(node, sendTo, txnId, executeAt, executes);
+ Persist persist = new Persist(node, participates, txnId, route, txn, executeAt, deps);
+ node.send(participates.nodes(), to -> Apply.applyMaximal(to, participates, executes, txnId, route, txn, executeAt, deps, writes, result), persist);
}
private Persist(Node node, Topologies topologies, TxnId txnId, FullRoute<?> route, Txn txn, Timestamp executeAt, Deps deps)
@@ -96,15 +105,9 @@ public class Persist implements Callback<ApplyReply>
{
if (!isDone)
{
- // TODO (low priority, consider, efficiency): send to non-home replicas also, so they may clear their log more easily?
- Shard homeShard = node.topology().forEpochIfKnown(route.homeKey(), txnId.epoch());
- node.send(homeShard, new InformHomeDurable(txnId, route.homeKey(), executeAt, Durable, new HashSet<>(persistedOn)));
isDone = true;
- }
- else if (!tracker.hasInFlight() && !tracker.hasFailures())
- {
- Shard homeShard = node.topology().forEpochIfKnown(route.homeKey(), txnId.epoch());
- node.send(homeShard, new InformHomeDurable(txnId, route.homeKey(), executeAt, Universal, new HashSet<>(persistedOn)));
+ Topologies topologies = tracker.topologies();
+ node.send(topologies.nodes(), to -> new InformDurable(to, topologies, route, txnId, executeAt, Majority));
}
}
break;
diff --git a/accord-core/src/main/java/accord/coordinate/Propose.java b/accord-core/src/main/java/accord/coordinate/Propose.java
index c4e56236..b72408ee 100644
--- a/accord-core/src/main/java/accord/coordinate/Propose.java
+++ b/accord-core/src/main/java/accord/coordinate/Propose.java
@@ -95,6 +95,10 @@ abstract class Propose<R> implements Callback<AcceptReply>
switch (reply.outcome())
{
default: throw new IllegalStateException();
+ case Truncated:
+ isDone = true;
+ callback.accept(null, new Truncated(txnId, route.homeKey()));
+ break;
case Redundant:
case RejectedBallot:
isDone = true;
@@ -135,33 +139,34 @@ abstract class Propose<R> implements Callback<AcceptReply>
final Node node;
final Ballot ballot;
final TxnId txnId;
- final RoutingKey invalidateWithKey;
+ final RoutingKey someParticipant;
final BiConsumer<Void, Throwable> callback;
+ final Map<Id, AcceptReply> debug = debug() ? new HashMap<>() : null;
private final QuorumShardTracker acceptTracker;
private boolean isDone;
- Invalidate(Node node, Shard shard, Ballot ballot, TxnId txnId, RoutingKey invalidateWithKey, BiConsumer<Void, Throwable> callback)
+ Invalidate(Node node, Shard shard, Ballot ballot, TxnId txnId, RoutingKey someParticipant, BiConsumer<Void, Throwable> callback)
{
this.node = node;
this.acceptTracker = new QuorumShardTracker(shard);
this.ballot = ballot;
this.txnId = txnId;
- this.invalidateWithKey = invalidateWithKey;
+ this.someParticipant = someParticipant;
this.callback = callback;
}
- public static Invalidate proposeInvalidate(Node node, Ballot ballot, TxnId txnId, RoutingKey invalidateWithKey, BiConsumer<Void, Throwable> callback)
+ public static Invalidate proposeInvalidate(Node node, Ballot ballot, TxnId txnId, RoutingKey invalidateWithParticipant, BiConsumer<Void, Throwable> callback)
{
- Shard shard = node.topology().forEpochIfKnown(invalidateWithKey, txnId.epoch());
- Invalidate invalidate = new Invalidate(node, shard, ballot, txnId, invalidateWithKey, callback);
- node.send(shard.nodes, to -> new Accept.Invalidate(ballot, txnId, invalidateWithKey), invalidate);
+ Shard shard = node.topology().forEpochIfKnown(invalidateWithParticipant, txnId.epoch());
+ Invalidate invalidate = new Invalidate(node, shard, ballot, txnId, invalidateWithParticipant, callback);
+ node.send(shard.nodes, to -> new Accept.Invalidate(ballot, txnId, invalidateWithParticipant), invalidate);
return invalidate;
}
- public static Invalidate proposeAndCommitInvalidate(Node node, Ballot ballot, TxnId txnId, RoutingKey invalidateWithKey, Route<?> commitInvalidationTo, Timestamp invalidateUntil, BiConsumer<?, Throwable> callback)
+ public static Invalidate proposeAndCommitInvalidate(Node node, Ballot ballot, TxnId txnId, RoutingKey invalidateWithParticipant, Route<?> commitInvalidationTo, Timestamp invalidateUntil, BiConsumer<?, Throwable> callback)
{
- return proposeInvalidate(node, ballot, txnId, invalidateWithKey, (success, fail) -> {
+ return proposeInvalidate(node, ballot, txnId, invalidateWithParticipant, (success, fail) -> {
if (fail != null)
{
callback.accept(null, fail);
@@ -170,7 +175,7 @@ abstract class Propose<R> implements Callback<AcceptReply>
{
node.withEpoch(invalidateUntil.epoch(), () -> {
commitInvalidate(node, txnId, commitInvalidationTo, invalidateUntil);
- callback.accept(null, new Invalidated(txnId, invalidateWithKey));
+ callback.accept(null, new Invalidated(txnId, invalidateWithParticipant));
});
}
});
@@ -182,6 +187,8 @@ abstract class Propose<R> implements Callback<AcceptReply>
if (isDone)
return;
+ if (debug != null) debug.put(from, reply);
+
if (!reply.isOk())
{
isDone = true;
diff --git a/accord-core/src/main/java/accord/coordinate/ProposeAndExecute.java b/accord-core/src/main/java/accord/coordinate/ProposeAndExecute.java
index 8bf4f1d6..b9687ff6 100644
--- a/accord-core/src/main/java/accord/coordinate/ProposeAndExecute.java
+++ b/accord-core/src/main/java/accord/coordinate/ProposeAndExecute.java
@@ -51,6 +51,7 @@ class ProposeAndExecute extends Propose<Result>
propose.start();
}
+ @Override
void onAccepted()
{
Deps deps = Deps.merge(acceptOks, ok -> ok.deps);
diff --git a/accord-core/src/main/java/accord/coordinate/ReadCoordinator.java b/accord-core/src/main/java/accord/coordinate/ReadCoordinator.java
index da744f7d..4cb55e83 100644
--- a/accord-core/src/main/java/accord/coordinate/ReadCoordinator.java
+++ b/accord-core/src/main/java/accord/coordinate/ReadCoordinator.java
@@ -25,6 +25,7 @@ import com.google.common.collect.Lists;
import accord.coordinate.tracking.ReadTracker;
import accord.local.Node;
import accord.local.Node.Id;
+import accord.messages.CheckStatus.WithQuorum;
import accord.primitives.Ranges;
import accord.primitives.TxnId;
import accord.topology.Topologies;
@@ -32,6 +33,8 @@ import accord.utils.Invariants;
import java.util.*;
+import static accord.messages.CheckStatus.WithQuorum.HasQuorum;
+import static accord.messages.CheckStatus.WithQuorum.NoQuorum;
import static accord.utils.Invariants.debug;
import static com.google.common.collect.Sets.newHashSetWithExpectedSize;
@@ -40,9 +43,9 @@ public abstract class ReadCoordinator<Reply extends accord.messages.Reply> exten
public enum Action
{
/**
- * Immediately fail the coordination
+ * The coordination has been failed prior to returning this response, and no further action should be taken.
*/
- Abort,
+ Aborted,
/**
* This response is unsuitable for the purposes of this coordination, whether individually or as a quorum.
@@ -77,7 +80,17 @@ public abstract class ReadCoordinator<Reply extends accord.messages.Reply> exten
ApprovePartial
}
- protected enum Success { Quorum, Success }
+ protected enum Success
+ {
+ Quorum(HasQuorum), Success(NoQuorum);
+
+ public final WithQuorum withQuorum;
+
+ Success(WithQuorum withQuorum)
+ {
+ this.withQuorum = withQuorum;
+ }
+ }
protected final Node node;
protected final TxnId txnId;
@@ -109,7 +122,7 @@ public abstract class ReadCoordinator<Reply extends accord.messages.Reply> exten
switch (process(from, reply))
{
default: throw new IllegalStateException();
- case Abort:
+ case Aborted:
isDone = true;
break;
@@ -176,13 +189,37 @@ public abstract class ReadCoordinator<Reply extends accord.messages.Reply> exten
finishOnFailure();
}
+ protected void finishOnFailure(Throwable failure, boolean overrideExistingFailure)
+ {
+ Invariants.checkState(!isDone);
+ if (overrideExistingFailure)
+ {
+ if (this.failure != null)
+ failure.addSuppressed(this.failure);
+ this.failure = failure;
+ }
+ else
+ {
+ if (this.failure != null) this.failure.addSuppressed(failure);
+ else this.failure = failure;
+ }
+ finishOnFailure();
+ }
+
protected void finishOnFailure()
{
Invariants.checkState(!isDone);
+ Invariants.checkState(failure != null);
isDone = true;
+ onDone(null, failure);
+ }
+
+ protected void finishOnExhaustion()
+ {
+ Invariants.checkState(!isDone);
if (failure == null)
failure = new Exhausted(txnId, null);
- onDone(null, failure);
+ finishOnFailure();
}
private void handle(RequestStatus result)
@@ -198,7 +235,7 @@ public abstract class ReadCoordinator<Reply extends accord.messages.Reply> exten
onDone(waitingOnData == 0 ? Success.Success : Success.Quorum, null);
break;
case Failed:
- finishOnFailure();
+ finishOnExhaustion();
}
}
diff --git a/accord-core/src/main/java/accord/coordinate/Recover.java b/accord-core/src/main/java/accord/coordinate/Recover.java
index a0c68a0b..9dd8f08e 100644
--- a/accord-core/src/main/java/accord/coordinate/Recover.java
+++ b/accord-core/src/main/java/accord/coordinate/Recover.java
@@ -27,25 +27,34 @@ import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.BiConsumer;
-import accord.coordinate.tracking.*;
-import accord.primitives.*;
-import accord.messages.Commit;
-import accord.utils.Invariants;
-import accord.utils.async.AsyncResult;
-import accord.utils.async.AsyncResults;
-
import accord.api.Result;
-import accord.topology.Topologies;
-import accord.messages.Callback;
+import accord.coordinate.tracking.QuorumTracker;
+import accord.coordinate.tracking.RecoveryTracker;
import accord.local.Node;
import accord.local.Node.Id;
import accord.messages.BeginRecovery;
import accord.messages.BeginRecovery.RecoverOk;
import accord.messages.BeginRecovery.RecoverReply;
+import accord.messages.Callback;
+import accord.messages.Commit;
import accord.messages.WaitOnCommit;
import accord.messages.WaitOnCommit.WaitOnCommitOk;
+import accord.primitives.Ballot;
+import accord.primitives.Deps;
+import accord.primitives.FullRoute;
+import accord.primitives.Participants;
+import accord.primitives.ProgressToken;
+import accord.primitives.Ranges;
+import accord.primitives.Timestamp;
+import accord.primitives.Txn;
+import accord.primitives.TxnId;
+import accord.topology.Topologies;
import accord.topology.Topology;
+import accord.utils.Invariants;
+import accord.utils.async.AsyncResult;
+import accord.utils.async.AsyncResults;
+import static accord.coordinate.Infer.InvalidateAndCallback.locallyInvalidateAndCallback;
import static accord.coordinate.Propose.Invalidate.proposeInvalidate;
import static accord.coordinate.ProposeAndExecute.proposeAndExecute;
import static accord.coordinate.tracking.RequestStatus.Failed;
@@ -64,11 +73,11 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
// are given earlier timestamps we can retry without restarting.
final QuorumTracker tracker;
- AwaitCommit(Node node, TxnId txnId, Unseekables<?, ?> unseekables)
+ AwaitCommit(Node node, TxnId txnId, Participants<?> participants)
{
- Topology topology = node.topology().globalForEpoch(txnId.epoch()).forSelection(unseekables, Topology.OnUnknown.REJECT);
+ Topology topology = node.topology().globalForEpoch(txnId.epoch()).forSelection(participants);
this.tracker = new QuorumTracker(new Topologies.Single(node.topology().sorter(), topology));
- node.send(topology.nodes(), to -> new WaitOnCommit(to, topology, txnId, unseekables), this);
+ node.send(topology.nodes(), to -> new WaitOnCommit(to, topology, txnId, participants), this);
}
@Override
@@ -103,7 +112,7 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
for (int i = 0 ; i < waitOn.txnIdCount() ; ++i)
{
TxnId txnId = waitOn.txnId(i);
- new AwaitCommit(node, txnId, waitOn.someUnseekables(txnId)).addCallback((success, failure) -> {
+ new AwaitCommit(node, txnId, waitOn.participants(txnId)).addCallback((success, failure) -> {
if (result.isDone())
return;
if (success != null && remaining.decrementAndGet() == 0)
@@ -210,7 +219,11 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
Timestamp executeAt = acceptOrCommit.executeAt;
switch (acceptOrCommit.status)
{
- default: throw new IllegalStateException();
+ default: throw new IllegalStateException("Unknown status: " + acceptOrCommit.status);
+ case Truncated:
+ callback.accept(ProgressToken.TRUNCATED, null);
+ return;
+
case Invalidated:
{
commitInvalidate();
@@ -223,10 +236,10 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
Deps committedDeps = tryMergeCommittedDeps();
node.withEpoch(executeAt.epoch(), () -> {
// TODO (required, consider): when writes/result are partially replicated, need to confirm we have quorum of these
- if (committedDeps != null) Persist.persistAndCommitMaximal(node, txnId, route, txn, executeAt, committedDeps, acceptOrCommit.writes, acceptOrCommit.result);
+ if (committedDeps != null) Persist.persistMaximal(node, txnId, route, txn, executeAt, committedDeps, acceptOrCommit.writes, acceptOrCommit.result);
else CollectDeps.withDeps(node, txnId, route, txn.keys(), executeAt, (deps, fail) -> {
if (fail != null) accept(null, fail);
- else Persist.persistAndCommitMaximal(node, txnId, route, txn, executeAt, deps, acceptOrCommit.writes, acceptOrCommit.result);
+ else Persist.persistMaximal(node, txnId, route, txn, executeAt, deps, acceptOrCommit.writes, acceptOrCommit.result);
});
});
accept(acceptOrCommit.result, null);
@@ -284,7 +297,7 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
return;
}
- case NotWitnessed:
+ case NotDefined:
case PreAccepted:
throw new IllegalStateException("Should only be possible to have Accepted or later commands");
}
@@ -315,12 +328,15 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
});
return;
}
+ // TODO (required): if there are epochs before txnId.epoch that haven't propagated their dependencies we potentially have a problem
+ // as we may propose deps that don't witness a transaction they should witness.
+ // in this case we should run GetDeps before proposing
propose(txnId, deps);
}
private void invalidate()
{
- proposeInvalidate(node, ballot, txnId, route.homeKey(), (success, fail) -> {
+ proposeInvalidate(node, ballot, txnId, route.someParticipatingKey(), (success, fail) -> {
if (fail != null) accept(null, fail);
else commitInvalidate();
});
@@ -331,7 +347,7 @@ public class Recover implements Callback<RecoverReply>, BiConsumer<Result, Throw
Timestamp invalidateUntil = recoverOks.stream().map(ok -> ok.executeAt).reduce(txnId, Timestamp::max);
node.withEpoch(invalidateUntil.epoch(), () -> Commit.Invalidate.commitInvalidate(node, txnId, route, invalidateUntil));
isDone = true;
- callback.accept(ProgressToken.INVALIDATED, null);
+ locallyInvalidateAndCallback(node, txnId, route, ProgressToken.INVALIDATED, callback);
}
private void propose(Timestamp executeAt, Deps deps)
diff --git a/accord-core/src/main/java/accord/coordinate/RecoverWithRoute.java b/accord-core/src/main/java/accord/coordinate/RecoverWithRoute.java
index a71f12f6..6f4559e5 100644
--- a/accord-core/src/main/java/accord/coordinate/RecoverWithRoute.java
+++ b/accord-core/src/main/java/accord/coordinate/RecoverWithRoute.java
@@ -20,42 +20,51 @@ package accord.coordinate;
import java.util.function.BiConsumer;
-import accord.local.Status;
-import accord.local.Status.Known;
-import accord.primitives.*;
-import accord.utils.Invariants;
-
+import accord.coordinate.FetchData.OnDone;
import accord.local.Node;
import accord.local.Node.Id;
+import accord.local.Status;
+import accord.local.Status.Known;
import accord.messages.CheckStatus;
import accord.messages.CheckStatus.CheckStatusOk;
import accord.messages.CheckStatus.CheckStatusOkFull;
import accord.messages.CheckStatus.IncludeInfo;
+import accord.primitives.Ballot;
+import accord.primitives.Deps;
+import accord.primitives.FullRoute;
+import accord.primitives.PartialRoute;
+import accord.primitives.Ranges;
+import accord.primitives.Route;
+import accord.primitives.Txn;
+import accord.primitives.TxnId;
+import accord.primitives.Unseekables;
import accord.topology.Topologies;
-
+import accord.utils.Invariants;
import javax.annotation.Nullable;
-import static accord.messages.Commit.Invalidate.commitInvalidate;
+import static accord.local.Status.KnownExecuteAt.ExecuteAtKnown;
+import static accord.local.Status.Outcome.Apply;
+import static accord.messages.CheckStatus.WithQuorum.HasQuorum;
+import static accord.messages.CheckStatus.WithQuorum.NoQuorum;
import static accord.primitives.ProgressToken.APPLIED;
import static accord.primitives.ProgressToken.INVALIDATED;
+import static accord.primitives.ProgressToken.TRUNCATED;
import static accord.primitives.Route.castToFullRoute;
-public class RecoverWithRoute extends CheckShards
+public class RecoverWithRoute extends CheckShards<FullRoute<?>>
{
final @Nullable Ballot promisedBallot; // if non-null, has already been promised by some shard
- final FullRoute<?> route;
final BiConsumer<Outcome, Throwable> callback;
final Status witnessedByInvalidation;
private RecoverWithRoute(Node node, Topologies topologies, @Nullable Ballot promisedBallot, TxnId txnId, FullRoute<?> route, Status witnessedByInvalidation, BiConsumer<Outcome, Throwable> callback)
{
- super(node, txnId, route, txnId.epoch(), IncludeInfo.All);
+ super(node, txnId, route, IncludeInfo.All);
// if witnessedByInvalidation == AcceptedInvalidate then we cannot assume its definition was known, and our comparison with the status is invalid
Invariants.checkState(witnessedByInvalidation != Status.AcceptedInvalidate);
// if witnessedByInvalidation == Invalidated we should anyway not be recovering
Invariants.checkState(witnessedByInvalidation != Status.Invalidated);
this.promisedBallot = promisedBallot;
- this.route = route;
this.callback = callback;
this.witnessedByInvalidation = witnessedByInvalidation;
assert topologies.oldestEpoch() == topologies.currentEpoch() && topologies.currentEpoch() == txnId.epoch();
@@ -85,13 +94,13 @@ public class RecoverWithRoute extends CheckShards
private FullRoute<?> route()
{
- return castToFullRoute(contact);
+ return castToFullRoute(this.route);
}
@Override
public void contact(Id to)
{
- node.send(to, new CheckStatus(to, topologies(), txnId, route, IncludeInfo.All), this);
+ node.send(to, new CheckStatus(to, topologies(), txnId, route, sourceEpoch, IncludeInfo.All), this);
}
@Override
@@ -111,7 +120,7 @@ public class RecoverWithRoute extends CheckShards
protected boolean isSufficient(Route<?> route, CheckStatusOk ok)
{
CheckStatusOkFull full = (CheckStatusOkFull)ok;
- Known sufficientTo = full.sufficientFor(route);
+ Known sufficientTo = full.sufficientFor(route.participants(), NoQuorum);
if (!sufficientTo.isDefinitionKnown())
return false;
@@ -131,15 +140,16 @@ public class RecoverWithRoute extends CheckShards
return;
}
- CheckStatusOkFull merged = (CheckStatusOkFull) this.merged;
- Known known = merged.sufficientFor(route);
+ CheckStatusOkFull full = (CheckStatusOkFull) this.merged;
+ Known known = full.sufficientFor(route.participants(), success == Success.Quorum ? HasQuorum : NoQuorum);
+
switch (known.outcome)
{
default: throw new AssertionError();
- case OutcomeUnknown:
+ case Unknown:
if (known.definition.isKnown())
{
- Txn txn = merged.partialTxn.reconstitute(route);
+ Txn txn = full.partialTxn.reconstitute(route);
Recover.recover(node, txnId, txn, route, callback);
}
else
@@ -150,17 +160,34 @@ public class RecoverWithRoute extends CheckShards
}
break;
- case OutcomeApplied:
- case OutcomeKnown:
- Invariants.checkState(known.definition.isKnown());
- Invariants.checkState(known.executeAt.hasDecidedExecuteAt());
- // TODO (required): we might not be able to reconstitute Txn if we have GC'd on some shards
- Txn txn = merged.partialTxn.reconstitute(route);
+ case WasApply:
+ case Apply:
+ if (!known.isDefinitionKnown() || known.executeAt != ExecuteAtKnown || known.outcome != Apply)
+ {
+ if (!full.truncated.isEmpty() && known.executeAt == ExecuteAtKnown)
+ {
+ // we might have only part of the full transaction, and a shard may have truncated;
+ // in this case we want to skip straight to apply, but only for the shards that haven't truncated
+ Unseekables<?> sendTo = route.subtract(full.truncated);
+ if (!sendTo.isEmpty())
+ {
+ Invariants.checkState(full.committedDeps.covering.containsAll(sendTo));
+ Invariants.checkState(full.partialTxn.covering().containsAll(sendTo));
+ Persist.persistPartialMaximal(node, txnId, sendTo, route, full.partialTxn, full.executeAt, full.committedDeps, full.writes, full.result);
+ }
+ }
+
+ OnDone.propagate(node, txnId, sourceEpoch, success.withQuorum, route, null, full, (s, f) -> callback.accept(f == null ? full.toProgressToken() : null, f));
+ break;
+ }
+
+ // TODO (required): might not be able to fully recover transaction - may only have enough for local shard
+ Txn txn = full.partialTxn.reconstitute(route);
if (known.deps.hasDecidedDeps())
{
- Deps deps = merged.committedDeps.reconstitute(route());
- node.withEpoch(merged.executeAt.epoch(), () -> {
- Persist.persistAndCommitMaximal(node, txnId, route(), txn, merged.executeAt, deps, merged.writes, merged.result);
+ Deps deps = full.committedDeps.reconstitute(route());
+ node.withEpoch(full.executeAt.epoch(), () -> {
+ Persist.persistMaximal(node, txnId, route(), txn, full.executeAt, deps, full.writes, full.result);
});
callback.accept(APPLIED, null);
}
@@ -170,13 +197,15 @@ public class RecoverWithRoute extends CheckShards
}
break;
- case InvalidationApplied:
+ case Invalidated:
if (witnessedByInvalidation != null && witnessedByInvalidation.hasBeen(Status.PreCommitted))
throw new IllegalStateException("We previously invalidated, finding a status that should be recoverable");
- long untilEpoch = node.topology().epoch();
- commitInvalidate(node, txnId, route, untilEpoch);
- callback.accept(INVALIDATED, null);
+ OnDone.propagate(node, txnId, sourceEpoch, success.withQuorum, route, null, full, (s, f) -> callback.accept(f == null ? INVALIDATED : null, f));
+ break;
+
+ case Erased:
+ OnDone.propagate(node, txnId, sourceEpoch, success.withQuorum, route, null, full, (s, f) -> callback.accept(f == null ? TRUNCATED : null, f));
break;
}
}
diff --git a/accord-core/src/main/java/accord/coordinate/RecoverWithHomeKey.java b/accord-core/src/main/java/accord/coordinate/RecoverWithSomeRoute.java
similarity index 71%
rename from accord-core/src/main/java/accord/coordinate/RecoverWithHomeKey.java
rename to accord-core/src/main/java/accord/coordinate/RecoverWithSomeRoute.java
index 33d01dad..d893ae80 100644
--- a/accord-core/src/main/java/accord/coordinate/RecoverWithHomeKey.java
+++ b/accord-core/src/main/java/accord/coordinate/RecoverWithSomeRoute.java
@@ -20,12 +20,12 @@ package accord.coordinate;
import java.util.function.BiConsumer;
-import accord.api.RoutingKey;
import accord.local.Node;
import accord.local.Status;
import accord.messages.CheckStatus.CheckStatusOk;
import accord.messages.CheckStatus.IncludeInfo;
-import accord.primitives.RoutingKeys;
+import accord.primitives.ProgressToken;
+import accord.primitives.Route;
import accord.primitives.TxnId;
import accord.utils.Invariants;
@@ -36,27 +36,25 @@ import static accord.primitives.Route.isFullRoute;
* A result of null indicates the transaction is globally persistent
* A result of CheckStatusOk indicates the maximum status found for the transaction, which may be used to assess progress
*/
-public class RecoverWithHomeKey extends CheckShards implements BiConsumer<Object, Throwable>
+public class RecoverWithSomeRoute extends CheckShards<Route<?>> implements BiConsumer<Object, Throwable>
{
- final RoutingKey homeKey;
final BiConsumer<Outcome, Throwable> callback;
final Status witnessedByInvalidation;
- RecoverWithHomeKey(Node node, TxnId txnId, RoutingKey homeKey, Status witnessedByInvalidation, BiConsumer<Outcome, Throwable> callback)
+ RecoverWithSomeRoute(Node node, TxnId txnId, Route<?> someRoute, Status witnessedByInvalidation, BiConsumer<Outcome, Throwable> callback)
{
- super(node, txnId, RoutingKeys.of(homeKey), txnId.epoch(), IncludeInfo.Route);
+ super(node, txnId, someRoute.withHomeKey(), IncludeInfo.Route);
this.witnessedByInvalidation = witnessedByInvalidation;
// if witnessedByInvalidation == AcceptedInvalidate then we cannot assume its definition was known, and our comparison with the status is invalid
Invariants.checkState(witnessedByInvalidation != Status.AcceptedInvalidate);
// if witnessedByInvalidation == Invalidated we should anyway not be recovering
Invariants.checkState(witnessedByInvalidation != Status.Invalidated);
- this.homeKey = homeKey;
this.callback = callback;
}
- public static RecoverWithHomeKey recover(Node node, TxnId txnId, RoutingKey homeKey, Status witnessedByInvalidation, BiConsumer<Outcome, Throwable> callback)
+ public static RecoverWithSomeRoute recover(Node node, TxnId txnId, Route<?> route, Status witnessedByInvalidation, BiConsumer<Outcome, Throwable> callback)
{
- RecoverWithHomeKey maybeRecover = new RecoverWithHomeKey(node, txnId, homeKey, witnessedByInvalidation, callback);
+ RecoverWithSomeRoute maybeRecover = new RecoverWithSomeRoute(node, txnId, route, witnessedByInvalidation, callback);
maybeRecover.start();
return maybeRecover;
}
@@ -91,9 +89,16 @@ public class RecoverWithHomeKey extends CheckShards implements BiConsumer<Object
callback.accept(null, new IllegalStateException());
return;
case Quorum:
- if (witnessedByInvalidation != null && witnessedByInvalidation.compareTo(Status.PreAccepted) > 0)
- throw new IllegalStateException("We previously invalidated, finding a status that should be recoverable");
- Invalidate.invalidate(node, txnId, contact.with(homeKey), true, callback);
+ if (merged != null && merged.isTruncated())
+ {
+ callback.accept(ProgressToken.TRUNCATED, null);
+ }
+ else
+ {
+ if (witnessedByInvalidation != null && witnessedByInvalidation.compareTo(Status.PreAccepted) > 0)
+ throw new IllegalStateException("We previously invalidated, finding a status that should be recoverable");
+ Invalidate.invalidate(node, txnId, route, true, callback);
+ }
}
}
else
diff --git a/accord-core/src/main/java/accord/primitives/AbstractUnseekableKeys.java b/accord-core/src/main/java/accord/coordinate/Truncated.java
similarity index 64%
copy from accord-core/src/main/java/accord/primitives/AbstractUnseekableKeys.java
copy to accord-core/src/main/java/accord/coordinate/Truncated.java
index 214190e8..d03e44e8 100644
--- a/accord-core/src/main/java/accord/primitives/AbstractUnseekableKeys.java
+++ b/accord-core/src/main/java/accord/coordinate/Truncated.java
@@ -16,23 +16,25 @@
* limitations under the License.
*/
-package accord.primitives;
+package accord.coordinate;
-import accord.api.RoutingKey;
+import javax.annotation.Nullable;
-import java.util.Arrays;
+import accord.api.RoutingKey;
+import accord.primitives.TxnId;
-// TODO: do we need this class?
-public abstract class AbstractUnseekableKeys<KS extends Unseekables<RoutingKey, ?>> extends AbstractKeys<RoutingKey, KS> implements Iterable<RoutingKey>, Unseekables<RoutingKey, KS>
+/**
+ * Thrown when a transaction's state has been truncated prior to the completion of some operation upon it
+ */
+public class Truncated extends CoordinationFailed
{
- AbstractUnseekableKeys(RoutingKey[] keys)
+ public Truncated(TxnId txnId, @Nullable RoutingKey homeKey)
{
- super(keys);
+ super(txnId, homeKey);
}
- @Override
- public final int indexOf(RoutingKey key)
+ public Truncated(TxnId txnId, @Nullable RoutingKey homeKey, String message)
{
- return Arrays.binarySearch(keys, key);
+ super(txnId, homeKey, message);
}
}
diff --git a/accord-core/src/main/java/accord/coordinate/tracking/AbstractTracker.java b/accord-core/src/main/java/accord/coordinate/tracking/AbstractTracker.java
index 9a52218b..1b4f9d28 100644
--- a/accord-core/src/main/java/accord/coordinate/tracking/AbstractTracker.java
+++ b/accord-core/src/main/java/accord/coordinate/tracking/AbstractTracker.java
@@ -156,6 +156,7 @@ public abstract class AbstractTracker<ST extends ShardTracker>
{
for (ST tracker : trackers)
{
+ if (tracker == null) continue;
if (test.test(tracker))
return true;
}
@@ -166,6 +167,7 @@ public abstract class AbstractTracker<ST extends ShardTracker>
{
for (ST tracker : trackers)
{
+ if (tracker == null) continue;
if (!test.test(tracker))
return false;
}
diff --git a/accord-core/src/main/java/accord/coordinate/tracking/AppliedTracker.java b/accord-core/src/main/java/accord/coordinate/tracking/AppliedTracker.java
new file mode 100644
index 00000000..159f2b88
--- /dev/null
+++ b/accord-core/src/main/java/accord/coordinate/tracking/AppliedTracker.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package accord.coordinate.tracking;
+
+import accord.local.Node;
+import accord.topology.Shard;
+import accord.topology.Topologies;
+
+import static accord.coordinate.tracking.AbstractTracker.ShardOutcomes.Fail;
+import static accord.coordinate.tracking.AbstractTracker.ShardOutcomes.NoChange;
+import static accord.coordinate.tracking.AbstractTracker.ShardOutcomes.Success;
+
+public class AppliedTracker extends AbstractTracker<AppliedTracker.AppliedShardTracker>
+{
+ public static class AppliedShardTracker extends ShardTracker
+ {
+ protected int waitingOn;
+
+ public AppliedShardTracker(Shard shard)
+ {
+ super(shard);
+ waitingOn = shard.rf();
+ }
+
+ public ShardOutcomes onSuccess(Object ignore)
+ {
+ return --waitingOn == 0 ? Success : NoChange;
+ }
+
+ // return true iff hasFailed()
+ public ShardOutcomes onFailure(Object ignore)
+ {
+ if (waitingOn < 0)
+ return NoChange;
+ waitingOn = -1;
+ return Fail;
+ }
+
+ public boolean hasReachedQuorum()
+ {
+ return waitingOn == 0;
+ }
+
+ boolean hasInFlight()
+ {
+ return waitingOn > 0;
+ }
+
+ boolean hasFailed()
+ {
+ return waitingOn < 0;
+ }
+ }
+
+ public AppliedTracker(Topologies topologies)
+ {
+ super(topologies, AppliedShardTracker[]::new, shard -> new AppliedShardTracker(shard));
+ }
+
+ public RequestStatus recordSuccess(Node.Id node)
+ {
+ return recordResponse(this, node, AppliedShardTracker::onSuccess, null);
+ }
+
+ // return true iff hasFailed()
+ public RequestStatus recordFailure(Node.Id node)
+ {
+ return recordResponse(this, node, AppliedShardTracker::onFailure, null);
+ }
+
+ public boolean hasFailed()
+ {
+ return any(AppliedShardTracker::hasFailed);
+ }
+
+ public boolean hasInFlight()
+ {
+ return any(AppliedShardTracker::hasInFlight);
+ }
+
+ public boolean hasReachedQuorum()
+ {
+ return all(AppliedShardTracker::hasReachedQuorum);
+ }
+}
diff --git a/accord-core/src/main/java/accord/coordinate/tracking/FastPathTracker.java b/accord-core/src/main/java/accord/coordinate/tracking/FastPathTracker.java
index 76db5b4f..7d30955c 100644
--- a/accord-core/src/main/java/accord/coordinate/tracking/FastPathTracker.java
+++ b/accord-core/src/main/java/accord/coordinate/tracking/FastPathTracker.java
@@ -82,6 +82,7 @@ public class FastPathTracker extends AbstractTracker<FastPathTracker.FastPathQuo
}
// return NewQuorumSuccess ONLY once fast path is rejected
+ @Override
public ShardOutcome<? super FastPathTracker> onQuorumSuccess(Node.Id node)
{
++successes;
@@ -98,6 +99,7 @@ public class FastPathTracker extends AbstractTracker<FastPathTracker.FastPathQuo
return NoChange;
}
+ @Override
public ShardOutcome<? super FastPathTracker> onMaybeFastPathSuccess(Node.Id node)
{
++successes;
@@ -110,6 +112,7 @@ public class FastPathTracker extends AbstractTracker<FastPathTracker.FastPathQuo
return quorumIfRejectsFastPath();
}
+ @Override
public ShardOutcome<? super FastPathTracker> onFailure(@Nonnull Node.Id from)
{
if (++failures > shard.maxFailures)
@@ -145,12 +148,14 @@ public class FastPathTracker extends AbstractTracker<FastPathTracker.FastPathQuo
return fastPathAccepts == shard.fastPathQuorumSize;
}
+ @Override
@VisibleForTesting
public boolean hasMetFastPathCriteria()
{
return fastPathAccepts >= shard.fastPathQuorumSize;
}
+ @Override
@VisibleForTesting
public boolean hasRejectedFastPath()
{
diff --git a/accord-core/src/main/java/accord/coordinate/tracking/InvalidationTracker.java b/accord-core/src/main/java/accord/coordinate/tracking/InvalidationTracker.java
index 1ba35823..c9582260 100644
--- a/accord-core/src/main/java/accord/coordinate/tracking/InvalidationTracker.java
+++ b/accord-core/src/main/java/accord/coordinate/tracking/InvalidationTracker.java
@@ -32,8 +32,9 @@ public class InvalidationTracker extends AbstractTracker<InvalidationTracker.Inv
private int fastPathRejects;
private int fastPathInflight;
private int promises;
+ private boolean hasDecision;
private int inflight;
- private boolean isDecided;
+ private boolean isFinal;
private InvalidationShardTracker(Shard shard)
{
@@ -42,7 +43,7 @@ public class InvalidationTracker extends AbstractTracker<InvalidationTracker.Inv
fastPathInflight = shard.fastPathElectorate.size();
}
- public InvalidationShardTracker onSuccess(Node.Id from, boolean isPromised, boolean withFastPath)
+ public InvalidationShardTracker onSuccess(Node.Id from, boolean isPromised, boolean isDecided, boolean withFastPath)
{
if (shard.fastPathElectorate.contains(from))
{
@@ -50,6 +51,7 @@ public class InvalidationTracker extends AbstractTracker<InvalidationTracker.Inv
if (!withFastPath) ++fastPathRejects;
}
if (isPromised) ++promises;
+ if (isDecided) hasDecision = true;
--inflight;
return this;
}
@@ -62,9 +64,9 @@ public class InvalidationTracker extends AbstractTracker<InvalidationTracker.Inv
return this;
}
- public boolean isDecided()
+ public boolean isFinal()
{
- return isFastPathDecided() && isPromiseDecided();
+ return hasDecision || (isFastPathDecided() && isPromiseDecided());
}
private boolean isFastPathDecided()
@@ -97,15 +99,25 @@ public class InvalidationTracker extends AbstractTracker<InvalidationTracker.Inv
return promises >= shard.slowPathQuorumSize;
}
+ public boolean isPromisedOrHasDecision()
+ {
+ return isPromised() || hasDecision();
+ }
+
+ public boolean hasDecision()
+ {
+ return hasDecision;
+ }
+
@Override
public ShardOutcomes apply(InvalidationTracker tracker, int shardIndex)
{
- if (isDecided)
+ if (isFinal)
return NoChange;
if (isFastPathRejected()) tracker.rejectsFastPath = true;
if (isPromised() && tracker.promisedShard < 0) tracker.promisedShard = shardIndex;
- if (isDecided()) isDecided = true;
+ if (isFinal()) isFinal = true;
if (tracker.rejectsFastPath && tracker.promisedShard >= 0)
{
@@ -113,8 +125,8 @@ public class InvalidationTracker extends AbstractTracker<InvalidationTracker.Inv
return Success;
}
- if (isDecided && --tracker.waitingOnShards == 0)
- return tracker.all(InvalidationShardTracker::isPromised) ? Success : Fail;
+ if (isFinal && --tracker.waitingOnShards == 0)
+ return tracker.all(InvalidationShardTracker::isPromisedOrHasDecision) ? Success : Fail;
return NoChange;
}
@@ -149,9 +161,9 @@ public class InvalidationTracker extends AbstractTracker<InvalidationTracker.Inv
return get(shardIndex, withinShardIndex).isPromised();
}
- public RequestStatus recordSuccess(Node.Id from, boolean isPromised, boolean acceptedFastPath)
+ public RequestStatus recordSuccess(Node.Id from, boolean isPromised, boolean hasDecision, boolean acceptedFastPath)
{
- return recordResponse(this, from, (shard, node) -> shard.onSuccess(node, isPromised, acceptedFastPath), from);
+ return recordResponse(this, from, (shard, node) -> shard.onSuccess(node, isPromised, hasDecision, acceptedFastPath), from);
}
// return true iff hasFailed()
diff --git a/accord-core/src/main/java/accord/impl/AbstractConfigurationService.java b/accord-core/src/main/java/accord/impl/AbstractConfigurationService.java
index fc77187a..7132f8d1 100644
--- a/accord-core/src/main/java/accord/impl/AbstractConfigurationService.java
+++ b/accord-core/src/main/java/accord/impl/AbstractConfigurationService.java
@@ -28,6 +28,7 @@ import org.slf4j.LoggerFactory;
import accord.api.ConfigurationService;
import accord.local.Node;
+import accord.primitives.Ranges;
import accord.topology.Topology;
import accord.utils.Invariants;
import accord.utils.async.AsyncChain;
@@ -287,15 +288,27 @@ public abstract class AbstractConfigurationService<EpochState extends AbstractCo
reportTopology(topology, true);
}
- protected void remoteSyncCompletePreListenerNotify(Node.Id node, long epoch) {}
+ protected void receiveRemoteSyncCompletePreListenerNotify(Node.Id node, long epoch) {}
- public synchronized void remoteSyncComplete(Node.Id node, long epoch)
+ public synchronized void receiveRemoteSyncComplete(Node.Id node, long epoch)
{
- remoteSyncCompletePreListenerNotify(node, epoch);
+ receiveRemoteSyncCompletePreListenerNotify(node, epoch);
for (Listener listener : listeners)
listener.onRemoteSyncComplete(node, epoch);
}
+ public synchronized void receiveClosed(Ranges ranges, long epoch)
+ {
+ for (Listener listener : listeners)
+ listener.onEpochClosed(ranges, epoch);
+ }
+
+ public synchronized void receiveRedundant(Ranges ranges, long epoch)
+ {
+ for (Listener listener : listeners)
+ listener.onEpochRedundant(ranges, epoch);
+ }
+
protected void truncateTopologiesPreListenerNotify(long epoch) {}
protected void truncateTopologiesPostListenerNotify(long epoch) {}
diff --git a/accord-core/src/main/java/accord/impl/AbstractFetchCoordinator.java b/accord-core/src/main/java/accord/impl/AbstractFetchCoordinator.java
index 283611e7..eedf19d1 100644
--- a/accord-core/src/main/java/accord/impl/AbstractFetchCoordinator.java
+++ b/accord-core/src/main/java/accord/impl/AbstractFetchCoordinator.java
@@ -29,10 +29,10 @@ import org.slf4j.LoggerFactory;
import accord.api.Data;
import accord.api.DataStore;
+import accord.coordinate.CoordinateSyncPoint;
import accord.coordinate.FetchCoordinator;
import accord.local.CommandStore;
import accord.local.Node;
-import accord.local.Status;
import accord.messages.Callback;
import accord.messages.MessageType;
import accord.messages.ReadData;
@@ -48,6 +48,7 @@ import accord.utils.async.AsyncChains;
import accord.utils.async.AsyncResult;
import accord.utils.async.AsyncResults;
+import static accord.messages.ReadData.ReadNack.NotCommitted;
import static accord.primitives.Routables.Slice.Minimal;
public abstract class AbstractFetchCoordinator extends FetchCoordinator
@@ -119,6 +120,11 @@ public abstract class AbstractFetchCoordinator extends FetchCoordinator
protected abstract void onReadOk(Node.Id from, CommandStore commandStore, Data data, Ranges ranges);
+ protected boolean collectMaxApplied()
+ {
+ return false;
+ }
+
@Override
public void contact(Node.Id to, Ranges ranges)
{
@@ -127,24 +133,30 @@ public abstract class AbstractFetchCoordinator extends FetchCoordinator
Ranges ownedRanges = ownedRangesForNode(to);
Invariants.checkArgument(ownedRanges.containsAll(ranges), "Got a reply from %s for ranges %s, but owned ranges %s does not contain all the ranges", to, ranges, ownedRanges);
PartialDeps partialDeps = syncPoint.waitFor.slice(ownedRanges, ranges);
- node.send(to, new FetchRequest(syncPoint.sourceEpoch(), syncPoint.syncId, ranges, partialDeps, rangeReadTxn(ranges)), new Callback<ReadData.ReadReply>()
+ node.send(to, new FetchRequest(syncPoint.sourceEpoch(), syncPoint.syncId, ranges, partialDeps, rangeReadTxn(ranges), collectMaxApplied()), new Callback<ReadData.ReadReply>()
{
@Override
public void onSuccess(Node.Id from, ReadData.ReadReply reply)
{
if (!reply.isOk())
{
- fail(to, new RuntimeException(reply.toString()));
- inflight.remove(key).cancel();
- switch ((ReadData.ReadNack) reply)
+ if (reply == NotCommitted)
{
- default: throw new AssertionError("Unhandled enum: " + reply);
- case Invalid:
- case Redundant:
- case NotCommitted:
- throw new AssertionError(String.format("Unexpected reply: %s", reply));
- case Error:
- // TODO (required): ensure errors are propagated to coordinators and can be logged
+ CoordinateSyncPoint.sendApply(node, from, syncPoint);
+ }
+ else
+ {
+ fail(to, new RuntimeException(reply.toString()));
+ inflight.remove(key).cancel();
+ switch ((ReadData.ReadNack) reply)
+ {
+ default: throw new AssertionError("Unhandled enum: " + reply);
+ case Invalid:
+ case Redundant:
+ throw new AssertionError(String.format("Unexpected reply: %s", reply));
+ case Error:
+ // TODO (required): ensure errors are propagated to coordinators and can be logged
+ }
}
return;
}
@@ -159,7 +171,7 @@ public abstract class AbstractFetchCoordinator extends FetchCoordinator
inflight.remove(key).cancel();
return;
}
- received = ranges.difference(ok.unavailable);
+ received = ranges.subtract(ok.unavailable);
}
else
{
@@ -220,34 +232,44 @@ public abstract class AbstractFetchCoordinator extends FetchCoordinator
public static class FetchRequest extends WaitAndReadData
{
public final PartialDeps partialDeps;
+ public final boolean collectMaxApplied;
private transient Timestamp maxApplied;
- public FetchRequest(long sourceEpoch, TxnId syncId, Ranges ranges, PartialDeps partialDeps, PartialTxn partialTxn)
+ public FetchRequest(long sourceEpoch, TxnId syncId, Ranges ranges, PartialDeps partialDeps, PartialTxn partialTxn, boolean collectMaxApplied)
{
- super(ranges, sourceEpoch, Status.Applied, partialDeps, Timestamp.MAX, syncId, partialTxn);
+ super(syncId, ranges, syncId, sourceEpoch, partialTxn);
this.partialDeps = partialDeps;
+ this.collectMaxApplied = collectMaxApplied;
}
@Override
protected void readComplete(CommandStore commandStore, Data result, Ranges unavailable)
{
- Ranges slice = commandStore.rangesForEpochHolder().get().allAt(executeReadAt).difference(unavailable);
- commandStore.maxAppliedFor(readScope, slice).begin((newMaxApplied, failure) -> {
- if (failure != null)
- {
- commandStore.agent().onUncaughtException(failure);
- }
- else
- {
- synchronized (this)
+ Ranges slice = commandStore.unsafeRangesForEpoch().allAt(txnId).subtract(unavailable);
+ if (collectMaxApplied)
+ {
+ commandStore.maxAppliedFor((Ranges)readScope, slice).begin((newMaxApplied, failure) -> {
+ if (failure != null)
{
- if (maxApplied == null) maxApplied = newMaxApplied;
- else maxApplied = Timestamp.max(maxApplied, newMaxApplied);
- Ranges reportUnavailable = unavailable.slice((Ranges)this.readScope, Minimal);
- super.readComplete(commandStore, result, reportUnavailable);
+ commandStore.agent().onUncaughtException(failure);
}
- }
- });
+ else
+ {
+ synchronized (this)
+ {
+ if (maxApplied == null) maxApplied = newMaxApplied;
+ else maxApplied = Timestamp.max(maxApplied, newMaxApplied);
+ Ranges reportUnavailable = unavailable.slice((Ranges)this.readScope, Minimal);
+ super.readComplete(commandStore, result, reportUnavailable);
+ }
+ }
+ });
+ }
+ else
+ {
+ Ranges reportUnavailable = unavailable.slice((Ranges)this.readScope, Minimal);
+ super.readComplete(commandStore, result, reportUnavailable);
+ }
}
@Override
@@ -265,8 +287,8 @@ public abstract class AbstractFetchCoordinator extends FetchCoordinator
public static class FetchResponse extends ReadData.ReadOk
{
- public final Timestamp maxApplied;
- public FetchResponse(@Nullable Ranges unavailable, @Nullable Data data, Timestamp maxApplied)
+ public final @Nullable Timestamp maxApplied;
+ public FetchResponse(@Nullable Ranges unavailable, @Nullable Data data, @Nullable Timestamp maxApplied)
{
super(unavailable, data);
this.maxApplied = maxApplied;
diff --git a/accord-core/src/main/java/accord/impl/AbstractSafeCommandStore.java b/accord-core/src/main/java/accord/impl/AbstractSafeCommandStore.java
index 36bc696b..955e686b 100644
--- a/accord-core/src/main/java/accord/impl/AbstractSafeCommandStore.java
+++ b/accord-core/src/main/java/accord/impl/AbstractSafeCommandStore.java
@@ -30,6 +30,7 @@ import accord.impl.CommandTimeseries.CommandLoader;
import accord.local.Command;
import accord.local.CommonAttributes;
import accord.local.PreLoadContext;
+import accord.local.RedundantBefore;
import accord.local.SafeCommand;
import accord.local.SafeCommandStore;
import accord.primitives.Ranges;
@@ -38,7 +39,7 @@ import accord.primitives.Seekable;
import accord.primitives.Seekables;
import accord.primitives.TxnId;
-public abstract class AbstractSafeCommandStore<CommandType extends SafeCommand, CommandsForKeyType extends SafeCommandsForKey> implements SafeCommandStore
+public abstract class AbstractSafeCommandStore<CommandType extends SafeCommand, CommandsForKeyType extends SafeCommandsForKey> extends SafeCommandStore
{
private static class PendingRegistration<T>
{
@@ -85,47 +86,42 @@ public abstract class AbstractSafeCommandStore<CommandType extends SafeCommand,
}
@Override
- public CommandType ifPresent(TxnId txnId)
- {
- CommandType command = getCommandInternal(txnId);
- if (command == null)
- throw new IllegalStateException(String.format("%s was not specified in PreLoadContext", txnId));
- if (command.isEmpty())
- return null;
- return command;
- }
-
- @Override
- public CommandType ifLoaded(TxnId txnId)
+ protected CommandType getInternalIfLoadedAndInitialised(TxnId txnId)
{
CommandType command = getIfLoaded(txnId, this::getCommandInternal, this::addCommandInternal, this::getIfLoaded);
- if (command == null)
+ if (command == null || command.isEmpty())
return null;
- if (command.isEmpty())
- command.notWitnessed();
return command;
}
@Override
- public CommandType command(TxnId txnId)
+ public CommandType getInternal(TxnId txnId)
{
CommandType command = getCommandInternal(txnId);
if (command == null)
throw new IllegalStateException(String.format("%s was not specified in PreLoadContext", txnId));
if (command.isEmpty())
- command.notWitnessed();
+ command.uninitialised();
return command;
}
- protected abstract CommandLoader<?> cfkLoader();
+ protected abstract CommandLoader<?> cfkLoader(RoutableKey key);
- public CommandsForKeyType ifLoaded(RoutableKey key)
+ public CommandsForKeyType ifLoadedAndInitialised(RoutableKey key)
{
CommandsForKeyType cfk = getIfLoaded(key, this::getCommandsForKeyInternal, this::addCommandsForKeyInternal, this::getIfLoaded);
if (cfk == null)
return null;
if (cfk.isEmpty())
- cfk.initialize(cfkLoader());
+ {
+ cfk.initialize(cfkLoader(key));
+ }
+ else
+ {
+ RedundantBefore.Entry entry = commandStore().redundantBefore().get(key.toUnseekable());
+ if (entry != null && cfk.current().hasRedundant(entry.redundantBefore))
+ cfk.set(cfk.current().withoutRedundant(entry.redundantBefore));
+ }
return cfk;
}
@@ -135,7 +131,7 @@ public abstract class AbstractSafeCommandStore<CommandType extends SafeCommand,
if (cfk == null)
throw new IllegalStateException(String.format("%s was not specified in PreLoadContext", key));
if (cfk.isEmpty())
- cfk.initialize(cfkLoader());
+ cfk.initialize(cfkLoader(key));
return cfk;
}
@@ -189,7 +185,7 @@ public abstract class AbstractSafeCommandStore<CommandType extends SafeCommand,
for (PendingRegistration<T> pendingRegistration : pendingRegistrations)
{
TxnId txnId = pendingRegistration.txnId;
- CommandType safeCommand = command(pendingRegistration.txnId);
+ CommandType safeCommand = getInternal(pendingRegistration.txnId);
Command command = safeCommand.current();
CommonAttributes attrs = updates.getOrDefault(txnId, command);
attrs = completer.complete(pendingRegistration.value, pendingRegistration.slice, safeCommand, attrs);
@@ -207,7 +203,7 @@ public abstract class AbstractSafeCommandStore<CommandType extends SafeCommand,
Map<TxnId, CommonAttributes> attributeUpdates = new HashMap<>();
completeRegistrations(attributeUpdates, pendingSeekablesRegistrations, this::completeRegistration);
completeRegistrations(attributeUpdates, pendingSeekableRegistrations, this::completeRegistration);
- attributeUpdates.forEach(((txnId, attributes) -> command(txnId).updateAttributes(attributes)));
+ attributeUpdates.forEach(((txnId, attributes) -> get(txnId).updateAttributes(attributes)));
}
}
diff --git a/accord-core/src/main/java/accord/impl/CommandTimeseries.java b/accord-core/src/main/java/accord/impl/CommandTimeseries.java
index 69d4dbcd..4cd34f51 100644
--- a/accord-core/src/main/java/accord/impl/CommandTimeseries.java
+++ b/accord-core/src/main/java/accord/impl/CommandTimeseries.java
@@ -44,8 +44,7 @@ import static accord.utils.Utils.ensureSortedMutable;
public class CommandTimeseries<D>
{
- public enum TestTimestamp
- {BEFORE, AFTER}
+ public enum TestTimestamp { BEFORE, AFTER }
private final Seekable keyOrRange;
protected final CommandLoader<D> loader;
@@ -104,6 +103,11 @@ public class CommandTimeseries<D>
return commands.isEmpty() ? Timestamp.NONE : commands.keySet().last();
}
+ public Timestamp minTimestamp()
+ {
+ return commands.isEmpty() ? Timestamp.NONE : commands.keySet().first();
+ }
+
/**
* All commands before/after (exclusive of) the given timestamp
* <p>
@@ -139,6 +143,12 @@ public class CommandTimeseries<D>
return initialValue;
}
+ public Timestamp maxExecuteAtBefore(Timestamp before)
+ {
+ return commands.headMap(before).values().stream().map(loader::executeAt)
+ .filter(Objects::nonNull).reduce(Timestamp::max).orElse(before);
+ }
+
Stream<TxnId> between(Timestamp min, Timestamp max, Predicate<Status> statusPredicate)
{
return commands.subMap(min, true, max, true).values().stream()
@@ -221,6 +231,13 @@ public class CommandTimeseries<D>
return this;
}
+ public Update<D> removeBefore(Timestamp timestamp)
+ {
+ commands = ensureSortedMutable(commands);
+ commands.headMap(timestamp, false).clear();
+ return this;
+ }
+
public CommandTimeseries<D> build()
{
return new CommandTimeseries<>(this);
diff --git a/accord-core/src/main/java/accord/impl/CommandsForKey.java b/accord-core/src/main/java/accord/impl/CommandsForKey.java
index 97142d3a..48cbc7c7 100644
--- a/accord-core/src/main/java/accord/impl/CommandsForKey.java
+++ b/accord-core/src/main/java/accord/impl/CommandsForKey.java
@@ -25,7 +25,6 @@ import accord.primitives.*;
import com.google.common.collect.ImmutableSortedMap;
import java.util.*;
-import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import static accord.local.Status.PreAccepted;
@@ -33,6 +32,8 @@ import static accord.local.Status.PreCommitted;
public class CommandsForKey implements CommandTimeseriesHolder
{
+ public static final long NO_LAST_EXECUTED_HLC = Long.MIN_VALUE;
+
public static class SerializerSupport
{
public static Listener listener(Key key)
@@ -41,12 +42,12 @@ public class CommandsForKey implements CommandTimeseriesHolder
}
public static <D> CommandsForKey create(Key key, Timestamp max,
- Timestamp lastExecutedTimestamp, long lastExecutedMicros, Timestamp lastWriteTimestamp,
+ Timestamp lastExecutedTimestamp, long lastExecutedHlc, Timestamp lastWriteTimestamp,
CommandLoader<D> loader,
ImmutableSortedMap<Timestamp, D> byId,
ImmutableSortedMap<Timestamp, D> byExecuteAt)
{
- return new CommandsForKey(key, max, lastExecutedTimestamp, lastExecutedMicros, lastWriteTimestamp, loader, byId, byExecuteAt);
+ return new CommandsForKey(key, max, lastExecutedTimestamp, lastExecutedHlc, lastWriteTimestamp, loader, byId, byExecuteAt);
}
}
@@ -99,18 +100,19 @@ public class CommandsForKey implements CommandTimeseriesHolder
}
}
- // TODO (now): add validation that anything inserted into *committedBy* has everything prior in its dependencies
private final Key key;
private final Timestamp max;
private final Timestamp lastExecutedTimestamp;
- private final long lastExecutedMicros;
+ // TODO (desired): we have leaked C* implementation details here
+ private final long rawLastExecutedHlc;
private final Timestamp lastWriteTimestamp;
private final CommandTimeseries<?> byId;
+ // TODO (expected): we probably do not need to separately maintain byExecuteAt - probably better to just filter byId
private final CommandTimeseries<?> byExecuteAt;
<D> CommandsForKey(Key key, Timestamp max,
Timestamp lastExecutedTimestamp,
- long lastExecutedMicros,
+ long rawLastExecutedHlc,
Timestamp lastWriteTimestamp,
CommandTimeseries<D> byId,
CommandTimeseries<D> byExecuteAt)
@@ -118,7 +120,7 @@ public class CommandsForKey implements CommandTimeseriesHolder
this.key = key;
this.max = max;
this.lastExecutedTimestamp = lastExecutedTimestamp;
- this.lastExecutedMicros = lastExecutedMicros;
+ this.rawLastExecutedHlc = rawLastExecutedHlc;
this.lastWriteTimestamp = lastWriteTimestamp;
this.byId = byId;
this.byExecuteAt = byExecuteAt;
@@ -126,13 +128,13 @@ public class CommandsForKey implements CommandTimeseriesHolder
<D> CommandsForKey(Key key, Timestamp max,
Timestamp lastExecutedTimestamp,
- long lastExecutedMicros,
+ long rawLastExecutedHlc,
Timestamp lastWriteTimestamp,
CommandLoader<D> loader,
ImmutableSortedMap<Timestamp, D> committedById,
ImmutableSortedMap<Timestamp, D> committedByExecuteAt)
{
- this(key, max, lastExecutedTimestamp, lastExecutedMicros, lastWriteTimestamp,
+ this(key, max, lastExecutedTimestamp, rawLastExecutedHlc, lastWriteTimestamp,
new CommandTimeseries<>(key, loader, committedById),
new CommandTimeseries<>(key, loader, committedByExecuteAt));
}
@@ -142,7 +144,7 @@ public class CommandsForKey implements CommandTimeseriesHolder
this.key = key;
this.max = Timestamp.NONE;
this.lastExecutedTimestamp = Timestamp.NONE;
- this.lastExecutedMicros = 0;
+ this.rawLastExecutedHlc = 0;
this.lastWriteTimestamp = Timestamp.NONE;
this.byId = new CommandTimeseries<>(key, loader);
this.byExecuteAt = new CommandTimeseries<>(key, loader);
@@ -160,13 +162,13 @@ public class CommandsForKey implements CommandTimeseriesHolder
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
CommandsForKey that = (CommandsForKey) o;
- return lastExecutedMicros == that.lastExecutedMicros
- && key.equals(that.key)
- && Objects.equals(max, that.max)
- && Objects.equals(lastExecutedTimestamp, that.lastExecutedTimestamp)
- && Objects.equals(lastWriteTimestamp, that.lastWriteTimestamp)
- && byId.equals(that.byId)
- && byExecuteAt.equals(that.byExecuteAt);
+ return rawLastExecutedHlc == that.rawLastExecutedHlc
+ && key.equals(that.key)
+ && Objects.equals(max, that.max)
+ && Objects.equals(lastExecutedTimestamp, that.lastExecutedTimestamp)
+ && Objects.equals(lastWriteTimestamp, that.lastWriteTimestamp)
+ && byId.equals(that.byId)
+ && byExecuteAt.equals(that.byExecuteAt);
}
@Override
@@ -196,9 +198,14 @@ public class CommandsForKey implements CommandTimeseriesHolder
return lastExecutedTimestamp;
}
- public long lastExecutedMicros()
+ public long lastExecutedHlc()
+ {
+ return rawLastExecutedHlc == NO_LAST_EXECUTED_HLC ? lastExecutedTimestamp.hlc() : rawLastExecutedHlc;
+ }
+
+ public long rawLastExecutedHlc()
{
- return lastExecutedMicros;
+ return rawLastExecutedHlc;
}
public Timestamp lastWriteTimestamp()
@@ -218,19 +225,31 @@ public class CommandsForKey implements CommandTimeseriesHolder
return byExecuteAt;
}
- public void forWitnessed(Timestamp minTs, Timestamp maxTs, Consumer<TxnId> consumer)
+ public boolean hasRedundant(TxnId redundantBefore)
{
- byId.between(minTs, maxTs, status -> status.hasBeen(PreAccepted)).forEach(consumer);
- byExecuteAt.between(minTs, maxTs, status -> status.hasBeen(PreCommitted)).forEach(consumer);
+ return byId.minTimestamp().compareTo(redundantBefore) < 0;
}
- private static long getTimestampMicros(Timestamp timestamp)
+ public CommandsForKey withoutRedundant(TxnId redundantBefore)
{
- return timestamp.hlc();
+ Timestamp removeExecuteAt = byId.maxExecuteAtBefore(redundantBefore);
+
+ return new CommandsForKey(key, max.compareTo(redundantBefore) < 0 ? Timestamp.NONE : max,
+ lastExecutedTimestamp.compareTo(redundantBefore) < 0 ? Timestamp.NONE : lastExecutedTimestamp,
+ rawLastExecutedHlc < redundantBefore.hlc() ? NO_LAST_EXECUTED_HLC : rawLastExecutedHlc,
+ lastWriteTimestamp.compareTo(redundantBefore) < 0 ? Timestamp.NONE : lastWriteTimestamp,
+ (CommandTimeseries)byId.beginUpdate().removeBefore(redundantBefore).build(),
+ (CommandTimeseries)byExecuteAt.beginUpdate().removeBefore(removeExecuteAt).build()
+ );
}
+ public void forWitnessed(Timestamp minTs, Timestamp maxTs, Consumer<TxnId> consumer)
+ {
+ byId.between(minTs, maxTs, status -> status.hasBeen(PreAccepted)).forEach(consumer);
+ byExecuteAt.between(minTs, maxTs, status -> status.hasBeen(PreCommitted)).forEach(consumer);
+ }
- private void validateExecuteAtTime(Timestamp executeAt, boolean isForWriteTxn)
+ public void validateExecuteAtTime(Timestamp executeAt, boolean isForWriteTxn)
{
if (executeAt.compareTo(lastWriteTimestamp) < 0)
throw new IllegalArgumentException(String.format("%s is less than the most recent write timestamp %s", executeAt, lastWriteTimestamp));
@@ -245,18 +264,9 @@ public class CommandsForKey implements CommandTimeseriesHolder
throw new IllegalArgumentException(String.format("%s is greater than the most recent executed timestamp, cfk should be updated", executeAt, lastExecutedTimestamp));
}
- public int nowInSecondsFor(Timestamp executeAt, boolean isForWriteTxn)
- {
- validateExecuteAtTime(executeAt, isForWriteTxn);
- // we use the executeAt time instead of the monotonic database timestamp to prevent uneven
- // ttl expiration in extreme cases, ie 1M+ writes/second to a key causing timestamps to overflow
- // into the next second on some keys and not others.
- return Math.toIntExact(TimeUnit.MICROSECONDS.toSeconds(getTimestampMicros(lastExecutedTimestamp)));
- }
-
- public long timestampMicrosFor(Timestamp executeAt, boolean isForWriteTxn)
+ public long hlcFor(Timestamp executeAt, boolean isForWriteTxn)
{
validateExecuteAtTime(executeAt, isForWriteTxn);
- return lastExecutedMicros;
+ return rawLastExecutedHlc == NO_LAST_EXECUTED_HLC ? lastExecutedTimestamp.hlc() : rawLastExecutedHlc;
}
}
diff --git a/accord-core/src/main/java/accord/impl/CoordinateDurabilityScheduling.java b/accord-core/src/main/java/accord/impl/CoordinateDurabilityScheduling.java
new file mode 100644
index 00000000..35b9ea77
--- /dev/null
+++ b/accord-core/src/main/java/accord/impl/CoordinateDurabilityScheduling.java
@@ -0,0 +1,356 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package accord.impl;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.primitives.Ints;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import accord.coordinate.CoordinateGloballyDurable;
+import accord.coordinate.CoordinateShardDurable;
+import accord.coordinate.CoordinateSyncPoint;
+import accord.local.Node;
+import accord.local.ShardDistributor;
+import accord.primitives.Range;
+import accord.primitives.Ranges;
+import accord.primitives.Routable.Domain;
+import accord.primitives.SyncPoint;
+import accord.primitives.TxnId;
+import accord.topology.Shard;
+import accord.topology.Topology;
+import accord.utils.Invariants;
+import accord.utils.async.AsyncChain;
+import accord.utils.async.AsyncResult;
+
+import static accord.primitives.Txn.Kind.ExclusiveSyncPoint;
+import static java.util.concurrent.TimeUnit.MICROSECONDS;
+
+/**
+ * Helper methods and classes to invoke coordination to propagate information about durability.
+ *
+ * Both CoordinateShardDurable and CoordinateGloballyDurable use wall time to loosely coordinate between nodes
+ * so that they take non-overlapping (hopefully) turns doing the coordination.
+ *
+ * Both of these have a concept of rounds where rounds have a known duration in wall time, and the current round is known
+ * based on time since the epoch, and the point in time where a node should do something in a given round is known based
+ * on its index in the sorted list of nodes ids in the current epoch.
+ *
+ * Coordinate globally durable is simpler because they just need to take turns so nodes just calculate when it is their
+ * turn and invoke CoordinateGloballyDurable.
+ *
+ * CoordinateShardDurable needs nodes to not overlap on the ranges they operate on or the exlusive sync points overlap
+ * with each other and block progress. A target duration to process the entire ring is set, and then each node in the
+ * current round has a time in the round that it should start processing, and the time it starts and the subranges it is
+ * responsible for rotates backwards every round so that a down node doesn't prevent a subrange from being processed.
+ *
+ * The work for CoordinateShardDurable is further subdivided where each subrange a node operates on is divided a fixed
+ * number of times and then processed one at a time with a fixed wait between them.
+ *
+ * // TODO review will the scheduler shut down on its own or do the individual tasks need to be canceled manually?
+ * // TODO (expected): cap number of coordinations we can have in flight at once
+ * Didn't go with recurring because it doesn't play well with async execution of these tasks
+ */
+public class CoordinateDurabilityScheduling
+{
+ private static final Logger logger = LoggerFactory.getLogger(CoordinateDurabilityScheduling.class);
+
+ private final Node node;
+
+ /*
+ * In each round at each node wait this amount of time between initiating new CoordinateShardDurable
+ */
+ private int frequencyMicros = Ints.saturatedCast(TimeUnit.MILLISECONDS.toMicros(500L));
+
+ /*
+ * In each round at each node wait this amount of time between allocating a CoordinateShardDurable txnId
+ * and coordinating the shard durability
+ */
+ private int txnIdLagMicros = Ints.saturatedCast(TimeUnit.SECONDS.toMicros(1L));
+
+ /*
+ * In each round at each node wait this amount of time between allocating a CoordinateShardDurable txnId
+ * and coordinating the shard durability
+ */
+ private int durabilityLagMicros = Ints.saturatedCast(TimeUnit.MILLISECONDS.toMicros(500L));
+
+ /*
+ * Target for how often the entire ring should be processed in microseconds. Every node will start at an offset in the current round that is based
+ * on this value / by (total # replicas * its index in the current round). The current round is determined by dividing time since the epoch by this
+ * duration.
+ */
+ private int shardCycleTimeMicros = Ints.saturatedCast(TimeUnit.SECONDS.toMicros(30));
+
+ /*
+ * Every node will independently attempt to invoke CoordinateGloballyDurable
+ * with a target gap between invocations of globalCycleTimeMicros
+ *
+ * This is done by nodes taking turns for each scheduled attempt that is due by calculating what # attempt is
+ * next for the current node ordinal in the cluster and time since the unix epoch and attempting to invoke then. If all goes
+ * well they end up doing it periodically in a timely fashion with the target gap achieved.
+ *
+ * TODO (desired): run this more often, but for less than the whole cluster, patterned in such a way as to ensure rapid cross-pollination
+ */
+ private long globalCycleTimeMicros = TimeUnit.SECONDS.toMicros(30);
+
+ private Topology currentGlobalTopology;
+ private final Map<Shard, Integer> localIndexes = new HashMap<>();
+ private int globalIndex;
+
+ private long nextGlobalSyncTimeMicros;
+ private long prevShardSyncTimeMicros;
+ volatile boolean stop;
+
+ public CoordinateDurabilityScheduling(Node node)
+ {
+ this.node = node;
+ }
+
+ public void setFrequency(int frequency, TimeUnit units)
+ {
+ this.frequencyMicros = Ints.saturatedCast(units.toMicros(frequency));
+ }
+
+ public void setTxnIdLag(int txnIdLag, TimeUnit units)
+ {
+ this.txnIdLagMicros = Ints.saturatedCast(units.toMicros(txnIdLag));
+ }
+
+ public void setDurabilityLag(int durabilityLag, TimeUnit units)
+ {
+ this.durabilityLagMicros = Ints.saturatedCast(units.toMicros(durabilityLag));
+ }
+
+ public void setShardCycleTime(int shardCycleTime, TimeUnit units)
+ {
+ this.shardCycleTimeMicros = Ints.saturatedCast(units.toMicros(shardCycleTime));
+ }
+
+ public void setGlobalCycleTime(long globalCycleTime, TimeUnit units)
+ {
+ this.globalCycleTimeMicros = units.toMicros(globalCycleTime);
+ }
+
+ /**
+ * Schedule regular invocations of CoordinateShardDurable and CoordinateGloballyDurable
+ */
+ public void start()
+ {
+ Invariants.checkState(!stop); // cannot currently restart safely
+ long nowMicros = node.unix(MICROSECONDS);
+ prevShardSyncTimeMicros = nowMicros;
+ setNextGlobalSyncTime(nowMicros);
+ requeue();
+ }
+
+ public void stop()
+ {
+ stop = true;
+ }
+
+ private void requeue()
+ {
+ if (!stop)
+ node.scheduler().once(this::run, frequencyMicros, MICROSECONDS);
+ }
+
+ /**
+ * Schedule the first CoordinateShardDurable execution for the current round. Sub-steps will be scheduled after
+ * each sub-step completes, and once all are completed scheduleCoordinateShardDurable is called again.
+ */
+ private void run()
+ {
+ if (stop)
+ return;
+
+ try
+ {
+ updateTopology();
+ if (currentGlobalTopology.size() == 0)
+ return;
+
+ long nowMicros = node.unix(MICROSECONDS);
+ if (nextGlobalSyncTimeMicros <= nowMicros)
+ {
+ startGlobalSync();
+ setNextGlobalSyncTime(nowMicros);
+ }
+
+ List<Ranges> coordinate = rangesToShardSync(nowMicros);
+ prevShardSyncTimeMicros = nowMicros;
+ if (coordinate.isEmpty())
+ {
+ logger.trace("Nothing pending in schedule for time slot at {}", nowMicros);
+ return;
+ }
+
+ logger.trace("Scheduling CoordinateShardDurable for {} at {}", coordinate, nowMicros);
+ for (Ranges ranges : coordinate)
+ startShardSync(ranges);
+ }
+ finally
+ {
+ requeue();
+ }
+ }
+
+ /**
+ * The first step for coordinating shard durable is to run an exclusive sync point
+ * the result of which can then be used to run
+ */
+ private void startShardSync(Ranges ranges)
+ {
+ TxnId at = node.nextTxnId(ExclusiveSyncPoint, Domain.Range);
+ node.scheduler().once(() -> node.withEpoch(at.epoch(), () -> {
+ CoordinateSyncPoint.coordinate(node, at, ranges)
+ .addCallback((success, fail) -> {
+ if (fail != null) logger.trace("Exception coordinating exclusive sync point for local shard durability of {}", ranges, fail);
+ else coordinateShardDurableAfterExclusiveSyncPoint(node, success);
+ });
+ }), txnIdLagMicros, MICROSECONDS);
+ }
+
+ private void coordinateShardDurableAfterExclusiveSyncPoint(Node node, SyncPoint exclusiveSyncPoint)
+ {
+ node.scheduler().once(() -> {
+ node.commandStores().any().execute(() -> {
+ CoordinateShardDurable.coordinate(node, exclusiveSyncPoint)
+ .addCallback((success, fail) -> {
+ if (fail != null)
+ {
+ logger.trace("Exception coordinating local shard durability, will retry immediately", fail);
+ coordinateShardDurableAfterExclusiveSyncPoint(node, exclusiveSyncPoint);
+ }
+ });
+ });
+ }, durabilityLagMicros, MICROSECONDS);
+ }
+
+ private void startGlobalSync()
+ {
+ try
+ {
+ long epoch = node.epoch();
+ AsyncChain<AsyncResult<Void>> resultChain = node.withEpoch(epoch, () -> node.commandStores().any().submit(() -> CoordinateGloballyDurable.coordinate(node, epoch)));
+ resultChain.begin((success, fail) -> {
+ if (fail != null) logger.trace("Exception initiating coordination of global durability", fail);
+ else logger.trace("Successful coordination of global durability");
+ });
+ }
+ catch (Exception e)
+ {
+ logger.error("Exception invoking withEpoch to start coordination for global durability", e);
+ }
+ }
+
+ private List<Ranges> rangesToShardSync(long nowMicros)
+ {
+ ShardDistributor distributor = node.commandStores().shardDistributor();
+ List<Ranges> result = new ArrayList<>();
+ for (Map.Entry<Shard, Integer> e : localIndexes.entrySet())
+ {
+ Shard shard = e.getKey();
+ int index = e.getValue();
+ long microsOffset = (index * shardCycleTimeMicros) / shard.rf();
+ int shardCycleTimeMicros = Math.max(this.shardCycleTimeMicros, Ints.saturatedCast(shard.rf() * 3L * frequencyMicros));
+ long prevSyncTimeMicros = Math.max(prevShardSyncTimeMicros, nowMicros - ((shardCycleTimeMicros / shard.rf()) / 2L));
+ int from = (int) ((prevSyncTimeMicros + microsOffset) % shardCycleTimeMicros);
+ int to = (int) ((nowMicros + microsOffset) % shardCycleTimeMicros);
+ List<Range> ranges = new ArrayList<>();
+ if (from > to)
+ {
+ Range range = distributor.splitRange(shard.range, to, shardCycleTimeMicros, shardCycleTimeMicros);
+ if (range != null)
+ ranges.add(range);
+ to = from;
+ from = 0;
+ }
+
+ Range range = distributor.splitRange(shard.range, from, to, shardCycleTimeMicros);
+ if (range != null)
+ ranges.add(range);
+
+ if (!ranges.isEmpty())
+ result.add(Ranges.of(ranges.toArray(new Range[0])));
+ }
+ return result;
+ }
+
+ private void updateTopology()
+ {
+ Topology latestGlobal = node.topology().current();
+ if (latestGlobal == currentGlobalTopology)
+ return;
+
+ Topology latestLocal = latestGlobal.forNode(node.id());
+ if (latestLocal.size() == 0)
+ return;
+
+ currentGlobalTopology = latestGlobal;
+
+ List<Node.Id> ids = new ArrayList<>(latestGlobal.nodes());
+ Collections.sort(ids);
+ globalIndex = ids.indexOf(node.id());
+
+ localIndexes.clear();
+ for (Shard shard : latestLocal.shards())
+ localIndexes.put(shard, shard.sortedNodes.indexOf(node.id()));
+ }
+
+ /**
+ * Based on the current unix time (simulated or otherwise) calculate the wait time in microseconds until the next turn of this
+ * node for some activity with a target gap between nodes doing the activity.
+ *
+ * This is done by taking the index of the node in the current topology and the total number of nodes
+ * and then using the target gap between invocations to calculate a "round" duration and point of time each node
+ * should have its turn in each round based on its index and calculating the time to the next turn for that node will occur.
+ *
+ * It's assumed it is fine if nodes overlap or reorder or skip for whatever activity we are picking turns for as long as it is approximately
+ * the right pacing.
+ */
+ private void setNextGlobalSyncTime(long nowMicros)
+ {
+ if (currentGlobalTopology == null)
+ {
+ nextGlobalSyncTimeMicros = nowMicros;
+ return;
+ }
+
+ // How long it takes for all nodes to go once
+ long totalRoundDuration = currentGlobalTopology.nodes().size() * globalCycleTimeMicros;
+ long startOfCurrentRound = (nowMicros / totalRoundDuration) * totalRoundDuration;
+
+ // In a given round at what time in the round should this node take its turn
+ long ourOffsetInRound = globalIndex * globalCycleTimeMicros;
+
+ long targetTimeInCurrentRound = startOfCurrentRound + ourOffsetInRound;
+ long targetTime = targetTimeInCurrentRound;
+ // If our time to run in the current round already passed then schedule it in the next round
+ if (targetTimeInCurrentRound < nowMicros)
+ targetTime += totalRoundDuration;
+
+ nextGlobalSyncTimeMicros = targetTime - nowMicros;
+ }
+}
\ No newline at end of file
diff --git a/accord-core/src/main/java/accord/impl/InMemorySafeCommand.java b/accord-core/src/main/java/accord/impl/ErasedSafeCommand.java
similarity index 56%
copy from accord-core/src/main/java/accord/impl/InMemorySafeCommand.java
copy to accord-core/src/main/java/accord/impl/ErasedSafeCommand.java
index 53fce388..5aea4367 100644
--- a/accord-core/src/main/java/accord/impl/InMemorySafeCommand.java
+++ b/accord-core/src/main/java/accord/impl/ErasedSafeCommand.java
@@ -19,84 +19,65 @@
package accord.impl;
import java.util.Collection;
-import java.util.function.Supplier;
+import java.util.Collections;
-import accord.impl.InMemoryCommandStore.GlobalCommand;
import accord.local.Command;
import accord.local.SafeCommand;
+import accord.local.SaveStatus;
import accord.primitives.TxnId;
-public class InMemorySafeCommand extends SafeCommand implements SafeState<Command>
-{
- private static final Supplier<GlobalCommand> INVALIDATED = () -> null;
+import static accord.local.Listeners.Immutable.EMPTY;
+import static accord.local.Status.Durability.DurableOrInvalidated;
- private Supplier<GlobalCommand> lazy;
- private GlobalCommand global;
+public class ErasedSafeCommand extends SafeCommand
+{
+ final Command erased;
- public InMemorySafeCommand(TxnId txnId, GlobalCommand global)
+ public ErasedSafeCommand(TxnId txnId)
{
super(txnId);
- this.global = global;
+ this.erased = new Command.Truncated(txnId, SaveStatus.Erased, DurableOrInvalidated, null, null, EMPTY, null, null);
}
- public InMemorySafeCommand(TxnId txnId, Supplier<GlobalCommand> global)
+ @Override
+ public Command current()
{
- super(txnId);
- this.lazy = global;
+ return erased;
}
@Override
- public Command current()
+ public void invalidate()
{
- touch();
- return global.value();
+ throw new UnsupportedOperationException();
}
@Override
- protected void set(Command update)
+ public boolean invalidated()
{
- touch();
- global.value(update);
+ return false;
}
@Override
public void addListener(Command.TransientListener listener)
{
- global.addListener(listener);
+ throw new UnsupportedOperationException();
}
@Override
public boolean removeListener(Command.TransientListener listener)
{
- return global.removeListener(listener);
+ throw new UnsupportedOperationException();
}
@Override
public Collection<Command.TransientListener> transientListeners()
{
- return global.transientListeners();
+ return Collections.emptyList();
}
@Override
- public void invalidate()
- {
- lazy = INVALIDATED;
- }
-
- @Override
- public boolean invalidated()
- {
- return lazy == INVALIDATED;
- }
-
- private void touch()
+ protected void set(Command command)
{
- if (invalidated())
- throw new IllegalStateException("Cannot access invalidated " + this);
- if (lazy != null)
- {
- global = lazy.get();
- lazy = null;
- }
+ throw new UnsupportedOperationException();
}
}
diff --git a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
index f4b160bd..ff9ebdc3 100644
--- a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
+++ b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
@@ -39,6 +39,9 @@ import java.util.function.BiFunction;
import java.util.function.Consumer;
import java.util.function.Function;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import accord.api.Agent;
import accord.api.DataStore;
import accord.api.Key;
@@ -47,7 +50,6 @@ import accord.impl.CommandTimeseries.CommandLoader;
import accord.local.Command;
import accord.local.CommandStore;
import accord.local.CommandStores.RangesForEpoch;
-import accord.local.CommandStores.RangesForEpochHolder;
import accord.local.CommonAttributes;
import accord.local.Listeners;
import accord.local.NodeTimeService;
@@ -73,33 +75,30 @@ import accord.utils.async.AsyncChain;
import accord.utils.async.AsyncChains;
import javax.annotation.Nullable;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
+import static accord.local.Command.NotDefined.uninitialised;
import static accord.local.SafeCommandStore.TestDep.ANY_DEPS;
import static accord.local.SafeCommandStore.TestDep.WITH;
import static accord.local.Status.Committed;
-import static accord.local.Status.PreAccepted;
-import static accord.local.Status.PreCommitted;
+import static accord.local.Status.Truncated;
import static accord.primitives.Routables.Slice.Minimal;
public abstract class InMemoryCommandStore extends CommandStore
{
private static final Logger logger = LoggerFactory.getLogger(InMemoryCommandStore.class);
- private final NavigableMap<TxnId, GlobalCommand> commands = new TreeMap<>();
+ final NavigableMap<TxnId, GlobalCommand> commands = new TreeMap<>();
private final NavigableMap<RoutableKey, GlobalCommandsForKey> commandsForKey = new TreeMap<>();
- private final CFKLoader cfkLoader = new CFKLoader();
// TODO (find library, efficiency): this is obviously super inefficient, need some range map
private final TreeMap<TxnId, RangeCommand> rangeCommands = new TreeMap<>();
private final TreeMap<TxnId, Ranges> historicalRangeCommands = new TreeMap<>();
+ protected Timestamp maxRedundant = Timestamp.NONE;
private InMemorySafeStore current;
- public InMemoryCommandStore(int id, NodeTimeService time, Agent agent, DataStore store, ProgressLog.Factory progressLogFactory, RangesForEpochHolder rangesForEpochHolder)
+ public InMemoryCommandStore(int id, NodeTimeService time, Agent agent, DataStore store, ProgressLog.Factory progressLogFactory, EpochUpdateHolder epochUpdateHolder)
{
- super(id, time, agent, store, progressLogFactory, rangesForEpochHolder);
+ super(id, time, agent, store, progressLogFactory, epochUpdateHolder);
}
@Override
@@ -155,80 +154,6 @@ public abstract class InMemoryCommandStore extends CommandStore
return commandsForKey.containsKey(key);
}
- public CFKLoader cfkLoader()
- {
- return cfkLoader;
- }
-
- public void forEpochCommands(Ranges ranges, long epoch, Consumer<Command> consumer)
- {
- Timestamp minTimestamp = Timestamp.minForEpoch(epoch);
- Timestamp maxTimestamp = Timestamp.maxForEpoch(epoch);
- for (Range range : ranges)
- {
- Iterable<GlobalCommandsForKey> keyCommands = commandsForKey.subMap(range.start(), range.startInclusive(),
- range.end(), range.endInclusive()).values();
-
- for (GlobalCommandsForKey commands : keyCommands)
- {
- if (commands.isEmpty())
- continue;
- commands.value().forWitnessed(minTimestamp, maxTimestamp, txnId -> consumer.accept(command(txnId).value()));
- }
-
- rangeCommands.forEach((txnId, rangeCommand) -> {
- if (!rangeCommand.ranges.intersects(range))
- return;
- Command command = rangeCommand.command.value();
- Invariants.nonNull(command);
- if (!command.hasBeen(PreAccepted))
- return;
-
- Timestamp timestamp = command.hasBeen(PreCommitted) ? command.executeAt() : txnId;
- if (timestamp.compareTo(maxTimestamp) > 0)
- return;
- if (timestamp.compareTo(minTimestamp) < 0)
- return;
- consumer.accept(command);
- });
- }
- }
-
- public void forCommittedInEpoch(Ranges ranges, long epoch, Consumer<Command> consumer)
- {
- Timestamp minTimestamp = Timestamp.minForEpoch(epoch);
- Timestamp maxTimestamp = Timestamp.maxForEpoch(epoch);
- for (Range range : ranges)
- {
- Iterable<GlobalCommandsForKey> keyCommands = commandsForKey.subMap(range.start(),
- range.startInclusive(),
- range.end(),
- range.endInclusive()).values();
- for (GlobalCommandsForKey commands : keyCommands)
- {
- if (commands.isEmpty())
- continue;
- commands.value().byExecuteAt()
- .between(minTimestamp, maxTimestamp, status -> status.hasBeen(Committed))
- .forEach(txnId -> consumer.accept(command(txnId).value()));
- }
-
- rangeCommands.forEach((txnId, rangeCommand) -> {
- if (!rangeCommand.ranges.intersects(range))
- return;
- Command command = rangeCommand.command.value();
- if (command == null || !command.hasBeen(Committed))
- return;
- Timestamp executeAt = command.executeAt();
- if (executeAt.compareTo(maxTimestamp) > 0)
- return;
- if (executeAt.compareTo(minTimestamp) < 0)
- return;
- consumer.accept(command);
- });
- }
- }
-
public CommonAttributes register(InMemorySafeStore safeStore, Seekables<?, ?> keysOrRanges, Ranges slice, SafeCommand command, CommonAttributes attrs)
{
switch (keysOrRanges.domain())
@@ -269,17 +194,17 @@ public abstract class InMemoryCommandStore extends CommandStore
return attrs;
}
- private <O> O mapReduceForKey(InMemorySafeStore safeStore, Routables<?, ?> keysOrRanges, Ranges slice, BiFunction<CommandsForKey, O, O> map, O accumulate, O terminalValue)
+ private <O> O mapReduceForKey(InMemorySafeStore safeStore, Routables<?> keysOrRanges, Ranges slice, BiFunction<CommandsForKey, O, O> map, O accumulate, O terminalValue)
{
switch (keysOrRanges.domain()) {
default:
throw new AssertionError();
case Key:
- AbstractKeys<Key, ?> keys = (AbstractKeys<Key, ?>) keysOrRanges;
+ AbstractKeys<Key> keys = (AbstractKeys<Key>) keysOrRanges;
for (Key key : keys)
{
if (!slice.contains(key)) continue;
- SafeCommandsForKey forKey = safeStore.ifLoaded(key);
+ SafeCommandsForKey forKey = safeStore.ifLoadedAndInitialised(key);
if (forKey.current() == null)
continue;
accumulate = map.apply(forKey.current(), accumulate);
@@ -311,7 +236,7 @@ public abstract class InMemoryCommandStore extends CommandStore
default:
throw new AssertionError();
case Key:
- AbstractKeys<Key, ?> keys = (AbstractKeys<Key, ?>) keysOrRanges;
+ AbstractKeys<Key> keys = (AbstractKeys<Key>) keysOrRanges;
keys.forEach(slice, key -> forEach.accept(key));
break;
case Range:
@@ -342,9 +267,33 @@ public abstract class InMemoryCommandStore extends CommandStore
}
}
+ @Override
+ public void markShardDurable(SafeCommandStore safeStore, TxnId syncId, Ranges ranges)
+ {
+ super.markShardDurable(safeStore, syncId, ranges);
+
+ if (!rangeCommands.containsKey(syncId))
+ historicalRangeCommands.merge(syncId, ranges, Ranges::with);
+
+ // TODO (now): apply on retrieval
+ historicalRangeCommands.entrySet().removeIf(next -> next.getKey().compareTo(syncId) < 0 && next.getValue().intersects(ranges));
+ rangeCommands.entrySet().removeIf(next -> {
+ if (!(next.getKey().compareTo(syncId) < 0 && next.getValue().ranges.intersects(ranges)))
+ return false;
+ maxRedundant = Timestamp.nonNullOrMax(maxRedundant, next.getValue().command.value().executeAt());
+ return true;
+ });
+ ranges.forEach(r -> {
+ commandsForKey.subMap(r.start(), r.startInclusive(), r.end(), r.endInclusive()).values().forEach(forKey -> {
+ if (!forKey.isEmpty())
+ forKey.value(forKey.value().withoutRedundant(syncId));
+ });
+ });
+ }
+
protected InMemorySafeStore createSafeStore(PreLoadContext context, RangesForEpoch ranges, Map<TxnId, InMemorySafeCommand> commands, Map<RoutableKey, InMemorySafeCommandsForKey> commandsForKeys)
{
- return new InMemorySafeStore(this, cfkLoader, ranges, context, commands, commandsForKeys);
+ return new InMemorySafeStore(this, ranges, context, commands, commandsForKeys);
}
protected final InMemorySafeStore createSafeStore(PreLoadContext context, RangesForEpoch ranges)
@@ -373,7 +322,7 @@ public abstract class InMemoryCommandStore extends CommandStore
{
if (current != null)
throw new IllegalStateException("Another operation is in progress or it's store was not cleared");
- current = createSafeStore(context, rangesForEpochHolder.get());
+ current = createSafeStore(context, updateRangesForEpoch());
return current;
}
@@ -442,7 +391,7 @@ public abstract class InMemoryCommandStore extends CommandStore
'}';
}
- class RangeCommand
+ static class RangeCommand
{
final GlobalCommand command;
Ranges ranges;
@@ -459,45 +408,63 @@ public abstract class InMemoryCommandStore extends CommandStore
}
}
- class CFKLoader implements CommandLoader<TxnId>
+ static class CFKEntry extends TxnId
{
- private Command loadForCFK(TxnId data)
+ final boolean uninitialised;
+ public CFKEntry(TxnId copy, boolean uninitialised)
+ {
+ super(copy);
+ this.uninitialised = uninitialised;
+ }
+ }
+
+ class CFKLoader implements CommandLoader<CFKEntry>
+ {
+ final RoutableKey key;
+ CFKLoader(RoutableKey key)
+ {
+ this.key = key;
+ }
+
+ private Command loadForCFK(CFKEntry entry)
{
- GlobalCommand globalCommand = ifPresent(data);
+ GlobalCommand globalCommand = ifPresent(entry);
if (globalCommand != null)
return globalCommand.value();
- throw new IllegalStateException("Could not find command for CFK for " + data);
+ if (entry.uninitialised)
+ return uninitialised(entry);
+ throw new IllegalStateException("Could not find command for CFK for " + entry);
}
@Override
- public TxnId txnId(TxnId txnId)
+ public TxnId txnId(CFKEntry txnId)
{
return loadForCFK(txnId).txnId();
}
@Override
- public Timestamp executeAt(TxnId txnId)
+ public Timestamp executeAt(CFKEntry txnId)
{
return loadForCFK(txnId).executeAt();
}
@Override
- public SaveStatus saveStatus(TxnId txnId)
+ public SaveStatus saveStatus(CFKEntry txnId)
{
return loadForCFK(txnId).saveStatus();
}
@Override
- public List<TxnId> depsIds(TxnId data)
+ public List<TxnId> depsIds(CFKEntry data)
{
PartialDeps deps = loadForCFK(data).partialDeps();
return deps != null ? deps.txnIds() : Collections.emptyList();
}
@Override
- public TxnId saveForCFK(Command command)
+ public CFKEntry saveForCFK(Command command)
{
- return command.txnId();
+ return new CFKEntry(command.txnId(), command.saveStatus().isUninitialised());
}
}
@@ -559,6 +526,12 @@ public abstract class InMemoryCommandStore extends CommandStore
return true;
}
+ @Override
+ public GlobalState<Command> value(Command value)
+ {
+ return super.value(value);
+ }
+
public Collection<Command.TransientListener> transientListeners()
{
return transientListeners == null ? Collections.emptySet() : transientListeners;
@@ -586,16 +559,14 @@ public abstract class InMemoryCommandStore extends CommandStore
private final Map<TxnId, InMemorySafeCommand> commands;
private final Map<RoutableKey, InMemorySafeCommandsForKey> commandsForKey;
private final RangesForEpoch ranges;
- private final CFKLoader cfkLoader;
- public InMemorySafeStore(InMemoryCommandStore commandStore, CFKLoader cfkLoader, RangesForEpoch ranges, PreLoadContext context, Map<TxnId, InMemorySafeCommand> commands, Map<RoutableKey, InMemorySafeCommandsForKey> commandsForKey)
+ public InMemorySafeStore(InMemoryCommandStore commandStore, RangesForEpoch ranges, PreLoadContext context, Map<TxnId, InMemorySafeCommand> commands, Map<RoutableKey, InMemorySafeCommandsForKey> commandsForKey)
{
super(context);
this.commandStore = commandStore;
this.commands = commands;
this.commandsForKey = commandsForKey;
this.ranges = Invariants.nonNull(ranges);
- this.cfkLoader = cfkLoader;
}
@Override
@@ -619,7 +590,7 @@ public abstract class InMemoryCommandStore extends CommandStore
@Override
protected void addCommandsForKeyInternal(InMemorySafeCommandsForKey cfk)
{
- commandsForKey.put(cfk.key(), (InMemorySafeCommandsForKey) cfk);
+ commandsForKey.put(cfk.key(), cfk);
}
@Override
@@ -669,28 +640,35 @@ public abstract class InMemoryCommandStore extends CommandStore
@Override
public Timestamp maxConflict(Seekables<?, ?> keysOrRanges, Ranges slice)
{
- Timestamp timestamp = commandStore.mapReduceForKey(this, keysOrRanges, slice, (forKey, prev) -> Timestamp.max(forKey.max(), prev), Timestamp.NONE, null);
+ Timestamp timestamp = commandStore.mapReduceForKey(this, keysOrRanges, slice, (forKey, prev) -> Timestamp.nonNullOrMax(forKey.max(), prev), Timestamp.NONE, null);
Seekables<?, ?> sliced = keysOrRanges.slice(slice, Minimal);
for (RangeCommand command : commandStore.rangeCommands.values())
{
if (command.ranges.intersects(sliced))
- timestamp = Timestamp.max(timestamp, command.command.value().executeAt());
+ timestamp = Timestamp.nonNullOrMax(timestamp, command.command.value().executeAt());
}
- return timestamp;
+ return Timestamp.nonNullOrMax(timestamp, commandStore.maxRedundant);
+ }
+
+ @Override
+ public void erase(SafeCommand command)
+ {
+ commands.remove(command.txnId());
}
+ // TODO (preferable): this can have protected visibility if under CommandStore, and this is perhaps a better place to put it also
@Override
public void registerHistoricalTransactions(Deps deps)
{
- RangesForEpochHolder rangesForEpochHolder = commandStore.rangesForEpochHolder();
- Ranges allRanges = rangesForEpochHolder.get().all();
+ RangesForEpoch rangesForEpoch = commandStore.rangesForEpoch;
+ Ranges allRanges = rangesForEpoch.all();
deps.keyDeps.keys().forEach(allRanges, key -> {
SafeCommandsForKey cfk = commandsForKey(key);
deps.keyDeps.forEach(key, txnId -> {
// TODO (desired, efficiency): this can be made more efficient by batching by epoch
- if (rangesForEpochHolder.get().coordinates(txnId).contains(key))
+ if (rangesForEpoch.coordinates(txnId).contains(key))
return; // already coordinates, no need to replicate
- if (!rangesForEpochHolder.get().allBefore(txnId.epoch()).contains(key))
+ if (!rangesForEpoch.allBefore(txnId.epoch()).contains(key))
return;
cfk.registerNotWitnessed(txnId);
@@ -705,9 +683,9 @@ public abstract class InMemoryCommandStore extends CommandStore
return;
Ranges ranges = deps.rangeDeps.ranges(txnId);
- if (rangesForEpochHolder.get().coordinates(txnId).intersects(ranges))
+ if (rangesForEpoch.coordinates(txnId).intersects(ranges))
return; // already coordinates, no need to replicate
- if (!rangesForEpochHolder.get().allBefore(txnId.epoch()).intersects(ranges))
+ if (!rangesForEpoch.allBefore(txnId.epoch()).intersects(ranges))
return;
historicalRangeCommands.merge(txnId, ranges.slice(allRanges), Ranges::with);
@@ -759,6 +737,10 @@ public abstract class InMemoryCommandStore extends CommandStore
Map<Range, List<Map.Entry<TxnId, Timestamp>>> collect = new TreeMap<>(Range::compare);
commandStore.rangeCommands.forEach(((txnId, rangeCommand) -> {
Command command = rangeCommand.command.value();
+ // TODO (now): probably this isn't safe - want to ensure we take dependency on any relevant syncId
+ if (command.is(Truncated))
+ return;
+
Invariants.nonNull(command);
switch (testTimestamp)
{
@@ -773,7 +755,7 @@ public abstract class InMemoryCommandStore extends CommandStore
if (command.executeAt().compareTo(timestamp) < 0) return;
else break;
case MAY_EXECUTE_BEFORE:
- Timestamp compareTo = command.known().executeAt.hasDecidedExecuteAt() ? command.executeAt() : command.txnId();
+ Timestamp compareTo = command.executeAtIfKnownElseTxnId();
if (compareTo.compareTo(timestamp) > 0) return;
else break;
}
@@ -810,7 +792,6 @@ public abstract class InMemoryCommandStore extends CommandStore
if (minStatus == null && testDep == ANY_DEPS)
{
-
commandStore.historicalRangeCommands.forEach(((txnId, ranges) -> {
switch (testTimestamp)
{
@@ -866,9 +847,9 @@ public abstract class InMemoryCommandStore extends CommandStore
}
@Override
- public CommandLoader<?> cfkLoader()
+ public CommandLoader<?> cfkLoader(RoutableKey key)
{
- return cfkLoader;
+ return commandStore.new CFKLoader(key);
}
@Override
@@ -891,9 +872,9 @@ public abstract class InMemoryCommandStore extends CommandStore
Runnable active = null;
final Queue<Runnable> queue = new ConcurrentLinkedQueue<>();
- public Synchronized(int id, NodeTimeService time, Agent agent, DataStore store, ProgressLog.Factory progressLogFactory, RangesForEpochHolder rangesForEpochHolder)
+ public Synchronized(int id, NodeTimeService time, Agent agent, DataStore store, ProgressLog.Factory progressLogFactory, EpochUpdateHolder epochUpdateHolder)
{
- super(id, time, agent, store, progressLogFactory, rangesForEpochHolder);
+ super(id, time, agent, store, progressLogFactory, epochUpdateHolder);
}
private synchronized void maybeRun()
@@ -983,9 +964,9 @@ public abstract class InMemoryCommandStore extends CommandStore
private Thread thread; // when run in the executor this will be non-null, null implies not running in this store
private final ExecutorService executor;
- public SingleThread(int id, NodeTimeService time, Agent agent, DataStore store, ProgressLog.Factory progressLogFactory, RangesForEpochHolder rangesForEpochHolder)
+ public SingleThread(int id, NodeTimeService time, Agent agent, DataStore store, ProgressLog.Factory progressLogFactory, EpochUpdateHolder epochUpdateHolder)
{
- super(id, time, agent, store, progressLogFactory, rangesForEpochHolder);
+ super(id, time, agent, store, progressLogFactory, epochUpdateHolder);
this.executor = Executors.newSingleThreadExecutor(r -> {
Thread thread = new Thread(r);
thread.setName(CommandStore.class.getSimpleName() + '[' + time.id() + ']');
@@ -1042,30 +1023,23 @@ public abstract class InMemoryCommandStore extends CommandStore
{
class DebugSafeStore extends InMemorySafeStore
{
- public DebugSafeStore(InMemoryCommandStore commandStore, CFKLoader cfkLoader, RangesForEpoch ranges, PreLoadContext context, Map<TxnId, InMemorySafeCommand> commands, Map<RoutableKey, InMemorySafeCommandsForKey> commandsForKey)
+ public DebugSafeStore(InMemoryCommandStore commandStore, RangesForEpoch ranges, PreLoadContext context, Map<TxnId, InMemorySafeCommand> commands, Map<RoutableKey, InMemorySafeCommandsForKey> commandsForKey)
{
- super(commandStore, cfkLoader, ranges, context, commands, commandsForKey);
- }
-
- @Override
- public InMemorySafeCommand ifPresent(TxnId txnId)
- {
- assertThread();
- return super.ifPresent(txnId);
+ super(commandStore, ranges, context, commands, commandsForKey);
}
@Override
- public InMemorySafeCommand ifLoaded(TxnId txnId)
+ public InMemorySafeCommand getInternalIfLoadedAndInitialised(TxnId txnId)
{
assertThread();
- return super.ifLoaded(txnId);
+ return super.getInternalIfLoadedAndInitialised(txnId);
}
@Override
- public InMemorySafeCommand command(TxnId txnId)
+ public InMemorySafeCommand getInternal(TxnId txnId)
{
assertThread();
- return super.command(txnId);
+ return super.getInternal(txnId);
}
@Override
@@ -1083,15 +1057,15 @@ public abstract class InMemoryCommandStore extends CommandStore
}
}
- public Debug(int id, NodeTimeService time, Agent agent, DataStore store, ProgressLog.Factory progressLogFactory, RangesForEpochHolder rangesForEpochHolder)
+ public Debug(int id, NodeTimeService time, Agent agent, DataStore store, ProgressLog.Factory progressLogFactory, EpochUpdateHolder epochUpdateHolder)
{
- super(id, time, agent, store, progressLogFactory, rangesForEpochHolder);
+ super(id, time, agent, store, progressLogFactory, epochUpdateHolder);
}
@Override
protected InMemorySafeStore createSafeStore(PreLoadContext context, RangesForEpoch ranges, Map<TxnId, InMemorySafeCommand> commands, Map<RoutableKey, InMemorySafeCommandsForKey> commandsForKeys)
{
- return new DebugSafeStore(this, cfkLoader(), ranges, context, commands, commandsForKeys);
+ return new DebugSafeStore(this, ranges, context, commands, commandsForKeys);
}
}
@@ -1164,8 +1138,8 @@ public abstract class InMemoryCommandStore extends CommandStore
if (level == 0 || verbose || !committed.isWaitingOnDependency())
log(prefix, suffix, command);
- Set<TxnId> waitingOnCommit = committed.waitingOnCommit();
- Set<TxnId> waitingOnApply = new HashSet<>(committed.waitingOnApply().values());
+ Set<TxnId> waitingOnCommit = committed.waitingOn.computeWaitingOnCommit();
+ Set<TxnId> waitingOnApply = committed.waitingOn.computeWaitingOnApply();
if (committed.isWaitingOnDependency() && !previouslyVisited)
deps.forEach(depId -> logDependencyGraph(commandStore, depId, visited, verbose, level+1, waitingOnCommit.contains(depId), waitingOnApply.contains(depId)));
diff --git a/accord-core/src/main/java/accord/impl/InMemoryCommandStores.java b/accord-core/src/main/java/accord/impl/InMemoryCommandStores.java
index 5fe87a56..a92565d3 100644
--- a/accord-core/src/main/java/accord/impl/InMemoryCommandStores.java
+++ b/accord-core/src/main/java/accord/impl/InMemoryCommandStores.java
@@ -22,7 +22,6 @@ import accord.local.*;
import accord.api.Agent;
import accord.api.DataStore;
import accord.api.ProgressLog;
-import accord.local.CommandStore;
import accord.utils.RandomSource;
public class InMemoryCommandStores
diff --git a/accord-core/src/main/java/accord/impl/InMemorySafeCommand.java b/accord-core/src/main/java/accord/impl/InMemorySafeCommand.java
index 53fce388..e8de7d89 100644
--- a/accord-core/src/main/java/accord/impl/InMemorySafeCommand.java
+++ b/accord-core/src/main/java/accord/impl/InMemorySafeCommand.java
@@ -99,4 +99,10 @@ public class InMemorySafeCommand extends SafeCommand implements SafeState<Comman
lazy = null;
}
}
+
+ GlobalCommand global()
+ {
+ touch();
+ return global;
+ }
}
diff --git a/accord-core/src/main/java/accord/impl/SafeCommandsForKey.java b/accord-core/src/main/java/accord/impl/SafeCommandsForKey.java
index 6ba8d900..06b5a542 100644
--- a/accord-core/src/main/java/accord/impl/SafeCommandsForKey.java
+++ b/accord-core/src/main/java/accord/impl/SafeCommandsForKey.java
@@ -29,7 +29,7 @@ import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static accord.local.Command.NotWitnessed.notWitnessed;
+import static accord.local.Command.NotDefined.uninitialised;
public abstract class SafeCommandsForKey implements SafeState<CommandsForKey>
{
@@ -80,7 +80,7 @@ public abstract class SafeCommandsForKey implements SafeState<CommandsForKey>
return update(new CommandsForKey(current.key(),
updateMax(current, timestamp),
current.lastExecutedTimestamp(),
- current.lastExecutedMicros(),
+ current.rawLastExecutedHlc(),
current.lastWriteTimestamp(),
(CommandTimeseries<D>) current().byId(),
(CommandTimeseries<D>) current().byExecuteAt()));
@@ -94,7 +94,7 @@ public abstract class SafeCommandsForKey implements SafeState<CommandsForKey>
return update(new CommandsForKey(current.key(),
updateMax(current, command.executeAt()),
current.lastExecutedTimestamp(),
- current.lastExecutedMicros(),
+ current.lastExecutedHlc(),
current.lastWriteTimestamp(),
byId.add(command.txnId(), command).build(),
byExecuteAt.add(command.txnId(), command).build() ));
@@ -111,10 +111,10 @@ public abstract class SafeCommandsForKey implements SafeState<CommandsForKey>
return update(new CommandsForKey(current.key(),
updateMax(current, txnId),
current.lastExecutedTimestamp(),
- current.lastExecutedMicros(),
+ current.lastExecutedHlc(),
current.lastWriteTimestamp(),
- byId.add(txnId, notWitnessed(txnId)).build(),
- byExecuteAt.add(txnId, notWitnessed(txnId)).build()));
+ byId.add(txnId, uninitialised(txnId)).build(),
+ byExecuteAt.add(txnId, uninitialised(txnId)).build()));
}
public <D> CommandsForKey listenerUpdate(Command command)
@@ -133,7 +133,7 @@ public abstract class SafeCommandsForKey implements SafeState<CommandsForKey>
{
default: throw new AssertionError();
case PreAccepted:
- case NotWitnessed:
+ case NotDefined:
case Accepted:
case AcceptedInvalidate:
case PreCommitted:
@@ -151,13 +151,14 @@ public abstract class SafeCommandsForKey implements SafeState<CommandsForKey>
case Invalidated:
byId.remove(command.txnId());
byExecuteAt.remove(command.txnId());
+ case Truncated:
break;
}
return update(new CommandsForKey(current.key(),
updateMax(current, command.executeAt()),
current.lastExecutedTimestamp(),
- current.lastExecutedMicros(),
+ current.lastExecutedHlc(),
current.lastWriteTimestamp(),
byId.build(),
byExecuteAt.build()));
@@ -182,16 +183,16 @@ public abstract class SafeCommandsForKey implements SafeState<CommandsForKey>
throw new IllegalArgumentException(String.format("%s is less than the most recent executed timestamp %s", executeAt, lastExecuted));
long micros = executeAt.hlc();
- long lastMicros = current.lastExecutedMicros();
+ long lastMicros = current.lastExecutedHlc();
Timestamp lastExecutedTimestamp = executeAt;
- long lastExecutedMicros = Math.max(micros, lastMicros + 1);
+ long lastExecutedHlc = micros > lastMicros ? Long.MIN_VALUE : lastMicros + 1;
Timestamp lastWriteTimestamp = isForWriteTxn ? executeAt : current.lastWriteTimestamp();
return update(new CommandsForKey(current.key(),
current.max(),
lastExecutedTimestamp,
- lastExecutedMicros,
+ lastExecutedHlc,
lastWriteTimestamp,
(CommandTimeseries<D>) current.byId(),
(CommandTimeseries<D>) current.byExecuteAt()));
diff --git a/accord-core/src/main/java/accord/impl/SimpleProgressLog.java b/accord-core/src/main/java/accord/impl/SimpleProgressLog.java
index c672ab3a..a6779a2c 100644
--- a/accord-core/src/main/java/accord/impl/SimpleProgressLog.java
+++ b/accord-core/src/main/java/accord/impl/SimpleProgressLog.java
@@ -18,54 +18,59 @@
package accord.impl;
-import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Set;
+import java.util.TreeMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeUnit;
import java.util.function.BiConsumer;
-import javax.annotation.Nullable;
-
+import accord.api.ProgressLog;
+import accord.coordinate.FetchData;
+import accord.coordinate.Invalidate;
+import accord.coordinate.Outcome;
+import accord.local.Command;
+import accord.local.CommandStore;
+import accord.local.Commands;
+import accord.local.Node;
+import accord.local.SafeCommand;
+import accord.local.SafeCommandStore;
+import accord.local.SaveStatus;
+import accord.local.SaveStatus.LocalExecution;
+import accord.local.Status;
+import accord.local.Status.Known;
+import accord.primitives.Participants;
+import accord.primitives.ProgressToken;
+import accord.primitives.Route;
+import accord.primitives.Timestamp;
+import accord.primitives.TxnId;
+import accord.primitives.Unseekables;
import accord.utils.IntrusiveLinkedList;
import accord.utils.IntrusiveLinkedListNode;
-import accord.coordinate.*;
-import accord.local.*;
-import accord.local.Status.Known;
-import accord.primitives.*;
import accord.utils.Invariants;
import accord.utils.async.AsyncChain;
import accord.utils.async.AsyncResult;
+import javax.annotation.Nullable;
-import accord.api.ProgressLog;
-import accord.api.RoutingKey;
-import accord.local.Node.Id;
-import accord.messages.Callback;
-import accord.messages.InformDurable;
-import accord.messages.SimpleReply;
-import accord.topology.Topologies;
-
-import static accord.api.ProgressLog.ProgressShard.Home;
import static accord.api.ProgressLog.ProgressShard.Unsure;
import static accord.coordinate.InformHomeOfTxn.inform;
import static accord.impl.SimpleProgressLog.CoordinateStatus.ReadyToExecute;
import static accord.impl.SimpleProgressLog.CoordinateStatus.Uncommitted;
-import static accord.impl.SimpleProgressLog.DisseminateStatus.NotExecuted;
import static accord.impl.SimpleProgressLog.Progress.Done;
import static accord.impl.SimpleProgressLog.Progress.Expected;
import static accord.impl.SimpleProgressLog.Progress.Investigating;
import static accord.impl.SimpleProgressLog.Progress.NoProgress;
import static accord.impl.SimpleProgressLog.Progress.NoneExpected;
import static accord.local.PreLoadContext.contextFor;
-import static accord.local.Status.Durability.Durable;
-import static accord.local.Status.Known.Nothing;
+import static accord.local.PreLoadContext.empty;
+import static accord.local.SaveStatus.LocalExecution.NotReady;
+import static accord.local.SaveStatus.LocalExecution.WaitingToApply;
+import static accord.local.Status.Durability.Majority;
import static accord.local.Status.PreApplied;
import static accord.local.Status.PreCommitted;
-import static accord.primitives.Route.isFullRoute;
// TODO (desired, consider): consider propagating invalidations in the same way as we do applied
+// TODO (expected): report long-lived recurring transactions / operations
public class SimpleProgressLog implements ProgressLog.Factory
{
enum Progress { NoneExpected, Expected, NoProgress, Investigating, Done }
@@ -85,8 +90,6 @@ public class SimpleProgressLog implements ProgressLog.Factory
}
}
- enum DisseminateStatus { NotExecuted, Durable, Done }
-
final Node node;
final List<Instance> instances = new CopyOnWriteArrayList<>();
@@ -124,6 +127,12 @@ public class SimpleProgressLog implements ProgressLog.Factory
}
}
+ void ensureDone()
+ {
+ if (progress != Done)
+ setProgress(Done);
+ }
+
boolean shouldRun()
{
switch (progress)
@@ -144,7 +153,7 @@ public class SimpleProgressLog implements ProgressLog.Factory
}
}
- abstract void run(Command command);
+ abstract void run(SafeCommandStore safeStore, SafeCommand safeCommand);
Progress progress()
{
@@ -208,8 +217,10 @@ public class SimpleProgressLog implements ProgressLog.Factory
}
@Override
- void run(Command command)
+ void run(SafeCommandStore safeStore, SafeCommand safeCommand)
{
+ Command command = safeCommand.current();
+ Invariants.checkState(!safeStore.isTruncated(command), "Command %s is truncated", command);
setProgress(Investigating);
switch (status)
{
@@ -219,17 +230,31 @@ public class SimpleProgressLog implements ProgressLog.Factory
case Done: // shouldn't be trying to make progress, as we're done
throw new IllegalStateException("Unexpected status: " + status);
- case Uncommitted:
case ReadyToExecute:
+ if (command.durability().isDurableOrInvalidated())
+ {
+ // should not reach here with an invalidated state
+ Invariants.checkState(command.durability().isDurable(), "Command %s is not durable", command);
+ durableGlobal();
+ return;
+ }
+
+ case Uncommitted:
{
- if (status.isAtLeastCommitted() && command.durability().isDurable())
+ Invariants.checkState(token.status != Status.Invalidated, "ProgressToken is invalidated, but we have not cleared the ProgressLog");
+ // TODO (expected): this should be encapsulated in Recover/FetchData
+ if (!command.durability().isDurable() && token.durability.isDurableOrInvalidated() && (token.durability.isDurable() || command.hasBeen(PreCommitted)))
+ command = Commands.setDurability(safeStore, safeCommand, Majority);
+
+ if (command.durability().isDurable() || token.durability.isDurableOrInvalidated())
{
- // must also be committed, as at the time of writing we do not guarantee dissemination of Commit
- // records to the home shard, so we only know the executeAt shards will have witnessed this
- // if the home shard is at an earlier phase, it must run recovery
- long epoch = command.executeAt().epoch();
- node.withEpoch(epoch, () -> debugInvestigating = FetchData.fetch(PreApplied.minKnown, node, txnId, command.route(), epoch, (success, fail) -> {
- commandStore.execute(PreLoadContext.empty(), ignore -> {
+ // not guaranteed to know the execution epoch, as might have received durability info obliquely
+ Timestamp executeAt = command.executeAtIfKnown();
+ long epoch = executeAt == null ? txnId.epoch() : executeAt.epoch();
+ Route<?> route = command.route();
+
+ node.withEpoch(epoch, () -> debugInvestigating = FetchData.fetch(PreApplied.minKnown, node, txnId, route, executeAt, (success, fail) -> {
+ commandStore.execute(empty(), ignore -> {
// should have found enough information to apply the result, but in case we did not reset progress
if (progress() == Investigating)
setProgress(Expected);
@@ -238,30 +263,21 @@ public class SimpleProgressLog implements ProgressLog.Factory
}
else
{
- RoutingKey homeKey = command.homeKey();
+ Route<?> route = Invariants.nonNull(command.route()).withHomeKey();
node.withEpoch(txnId.epoch(), () -> {
-
- AsyncResult<? extends Outcome> recover = node.maybeRecover(txnId, homeKey, command.route(), token);
+ AsyncResult<? extends Outcome> recover = node.maybeRecover(txnId, route, token);
recover.addCallback((success, fail) -> {
- commandStore.execute(PreLoadContext.empty(), ignore -> {
+ // TODO (expected): callback should be on safeStore, and should provide safeStore as a parameter
+ commandStore.execute(contextFor(txnId), safeStore0 -> {
if (status.isAtMostReadyToExecute() && progress() == Investigating)
{
setProgress(Expected);
if (fail != null)
return;
- ProgressToken token = success.asProgressToken();
- if (token.durability.isDurable())
- {
- commandStore.execute(contextFor(txnId), safeStore -> {
- Command cmd = Commands.setDurability(safeStore, txnId, token.durability, homeKey, null);
- safeStore.progressLog().durable(txnId, cmd.maxUnseekables(), null);
- }).begin(commandStore.agent());
- }
-
- updateMax(token);
+ updateMax(success.asProgressToken());
}
- }).begin(commandStore.agent());
+ });
});
debugInvestigating = recover;
@@ -278,195 +294,27 @@ public class SimpleProgressLog implements ProgressLog.Factory
}
}
- // exists only on home shard
- class DisseminateState extends State.Monitoring
- {
- class CoordinateAwareness implements Callback<SimpleReply>
- {
- @Override
- public void onSuccess(Id from, SimpleReply reply)
- {
- // TODO (required, efficiency): callbacks should be associated with a commandStore for processing to avoid this
- commandStore.execute(PreLoadContext.empty(), ignore -> {
- if (progress() == Done)
- return;
-
- notAwareOfDurability.remove(from);
- maybeDone();
- }).begin(commandStore.agent());
- }
-
- @Override
- public void onFailure(Id from, Throwable failure)
- {
- }
-
- @Override
- public void onCallbackFailure(Id from, Throwable failure)
- {
- }
- }
-
- DisseminateStatus status = NotExecuted;
- Set<Id> notAwareOfDurability; // TODO (easy, efficiency): use Agrona's IntHashSet as soon as Node.Id switches from long to int
- Set<Id> notPersisted; // TODO (easy, efficiency): use Agrona's IntHashSet as soon as Node.Id switches from long to int
-
- List<Runnable> whenReady;
-
- CoordinateAwareness investigating;
-
- private void whenReady(Node node, Command command, Runnable runnable)
- {
- if (notAwareOfDurability != null || maybeReady(node, command))
- {
- runnable.run();
- }
- else
- {
- if (whenReady == null)
- whenReady = new ArrayList<>();
- whenReady.add(runnable);
- }
- }
-
- private void whenReady(Runnable runnable)
- {
- if (notAwareOfDurability != null)
- {
- runnable.run();
- }
- else
- {
- if (whenReady == null)
- whenReady = new ArrayList<>();
- whenReady.add(runnable);
- }
- }
-
- // must know the epoch information, and have a valid Route
- private boolean maybeReady(Node node, Command command)
- {
- if (!command.status().hasBeen(Status.PreCommitted))
- return false;
-
- if (!isFullRoute(command.route()))
- return false;
-
- if (!node.topology().hasEpoch(command.executeAt().epoch()))
- return false;
-
- Topologies topology = node.topology().preciseEpochs(command.route(), command.txnId().epoch(), command.executeAt().epoch());
- notAwareOfDurability = topology.copyOfNodes();
- notPersisted = topology.copyOfNodes();
- if (whenReady != null)
- {
- whenReady.forEach(Runnable::run);
- whenReady = null;
- }
-
- return shouldRun();
- }
-
- private void maybeDone()
- {
- if (progress() != Done && notAwareOfDurability.isEmpty())
- {
- status = DisseminateStatus.Done;
- setProgress(Done);
- }
- }
-
- void durableGlobal(Node node, Command command, @Nullable Set<Id> persistedOn)
- {
- if (status == DisseminateStatus.Done)
- return;
-
- status = DisseminateStatus.Durable;
- setProgress(Expected);
- if (persistedOn == null)
- return;
-
- whenReady(node, command, () -> {
- notPersisted.removeAll(persistedOn);
- notAwareOfDurability.removeAll(persistedOn);
- maybeDone();
- });
- }
-
- void durableLocal(Node node)
- {
- if (status == DisseminateStatus.Done)
- return;
-
- status = DisseminateStatus.Durable;
- setProgress(Expected);
-
- whenReady(() -> {
- notPersisted.remove(node.id());
- notAwareOfDurability.remove(node.id());
- maybeDone();
- });
- }
-
- @Override
- void run(Command command)
- {
- switch (status)
- {
- default: throw new IllegalStateException();
- case NotExecuted:
- case Done:
- return;
- case Durable:
- }
-
- if (notAwareOfDurability == null && !maybeReady(node, command))
- return;
-
- // whenReady callbacks may run in maybeReady method, and those callbacks may set the progress to Done,
- // in which case this logic should no-op
- if (progress() == Done)
- return;
-
- setProgress(Investigating);
- if (notAwareOfDurability.isEmpty())
- {
- // TODO (required, consider): also track actual durability
- status = DisseminateStatus.Done;
- setProgress(Done);
- return;
- }
-
- FullRoute<?> route = Route.castToFullRoute(command.route());
- Timestamp executeAt = command.executeAt();
- investigating = new CoordinateAwareness();
- Topologies topologies = node.topology().preciseEpochs(route, txnId.epoch(), executeAt.epoch());
- node.send(notAwareOfDurability, to -> new InformDurable(to, topologies, route, txnId, executeAt, Durable), investigating);
- }
-
- @Override
- public String toString()
- {
- return "{" + status + ',' + progress() + '}';
- }
- }
-
class BlockingState extends State.Monitoring
{
- Known blockedUntil = Nothing;
+ LocalExecution blockedUntil = NotReady;
- Unseekables<?, ?> blockedOn;
+ Route<?> route;
+ Participants<?> participants;
Object debugInvestigating;
- void recordBlocking(Known blockedUntil, Unseekables<?, ?> blockedOn)
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ void recordBlocking(LocalExecution blockedUntil, @Nullable Route<?> route, @Nullable Participants<?> participants)
{
- Invariants.checkState(!blockedOn.isEmpty());
- if (this.blockedOn == null) this.blockedOn = blockedOn;
- else this.blockedOn = Unseekables.merge(this.blockedOn, (Unseekables)blockedOn);
- if (!blockedUntil.isSatisfiedBy(this.blockedUntil))
+ Invariants.checkState(route != null || participants != null, "Route and participants are both undefined");
+ Invariants.checkState(participants == null || !participants.isEmpty(), "participants is empty");
+ Invariants.checkState(route == null || route.hasParticipants(), "Route %s does not have participants", route);
+
+ this.route = Route.merge(this.route, (Route)route);
+ this.participants = Participants.merge(this.participants, (Participants) participants);
+ if (blockedUntil.compareTo(this.blockedUntil) > 0)
{
- this.blockedUntil = this.blockedUntil.merge(blockedUntil);
+ this.blockedUntil = blockedUntil;
setProgress(Expected);
}
}
@@ -483,9 +331,10 @@ public class SimpleProgressLog implements ProgressLog.Factory
}
@Override
- void run(Command command)
+ void run(SafeCommandStore safeStore, SafeCommand safeCommand)
{
- if (command.has(blockedUntil))
+ Command command = safeCommand.current();
+ if (command.isAtLeast(blockedUntil))
{
setProgress(NoneExpected);
return;
@@ -493,49 +342,62 @@ public class SimpleProgressLog implements ProgressLog.Factory
setProgress(Investigating);
// first make sure we have enough information to obtain the command locally
- Timestamp executeAt = command.hasBeen(PreCommitted) ? command.executeAt() : null;
- long srcEpoch = (executeAt != null ? executeAt : txnId).epoch();
- // TODO (desired, consider): compute fromEpoch, the epoch we already have this txn replicated until
- long toEpoch = Math.max(srcEpoch, node.topology().epoch());
- Unseekables<?, ?> someKeys = unseekables(command);
+ Timestamp executeAt = command.executeAtIfKnown();
+ Participants<?> maxParticipants = maxParticipants(command);
+ // we want to fetch a route if we have it, so that we can go to our neighbouring shards for info
+ // (rather than the home shard, which may have GC'd its state if the result is durable)
+ Unseekables<?> fetchKeys = maxContact(command);
BiConsumer<Known, Throwable> callback = (success, fail) -> {
- commandStore.execute(PreLoadContext.empty(), ignore -> {
+ // TODO (expected): this should be invoked on this commandStore; also do not need to load txn unless in DEBUG mode
+ commandStore.execute(contextFor(txnId), safeStore0 -> {
if (progress() != Investigating)
return;
setProgress(Expected);
- if (fail == null)
+ if (fail == null && blockedUntil.isSatisfiedBy(success))
{
- if (!success.isDefinitionKnown()) invalidate(node, txnId, someKeys);
- else record(success);
+ Command test = safeStore0.ifInitialised(txnId).current();
+ Invariants.checkState(test.has(success), "Command %s was expected to have known %s, but had %s", test, success, test.known());
+ record(success);
}
}).begin(commandStore.agent());
};
- node.withEpoch(toEpoch, () -> {
- debugInvestigating = FetchData.fetch(blockedUntil, node, txnId, someKeys, executeAt, toEpoch, callback);
+ node.withEpoch(blockedUntil.fetchEpoch(txnId, executeAt), () -> {
+ debugInvestigating = FetchData.fetch(blockedUntil.requires, node, txnId, fetchKeys, executeAt, callback);
});
}
- private Unseekables<?, ?> unseekables(Command command)
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ private Unseekables<?> maxContact(Command command)
{
- return Unseekables.merge((Route)command.route(), blockedOn);
+ Route<?> route = Route.merge(command.route(), (Route)this.route);
+ if (route != null) route = route.withHomeKey();
+ return Unseekables.merge(route == null ? null : route.withHomeKey(), (Unseekables)participants);
}
- private void invalidate(Node node, TxnId txnId, Unseekables<?, ?> someKeys)
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ private Participants<?> maxParticipants(Command command)
+ {
+ Route<?> route = Route.merge(command.route(), (Route)this.route);
+ return Participants.merge(route == null ? null : route.participants(), (Participants) participants);
+ }
+
+ private void invalidate(Node node, TxnId txnId, Participants<?> participants)
{
setProgress(Investigating);
- RoutingKey someKey = Route.isRoute(someKeys) ? (Route.castToRoute(someKeys)).homeKey() : someKeys.get(0).someIntersectingRoutingKey(null);
- someKeys = someKeys.with(someKey);
- debugInvestigating = Invalidate.invalidate(node, txnId, someKeys, (success, fail) -> {
- commandStore.execute(PreLoadContext.empty(), ignore -> {
+ // TODO (now): we should avoid invalidating with home key, as this simplifies erasing of invalidated transactions
+ // TODO (required): exponential back-off or time-slicing
+ debugInvestigating = Invalidate.invalidate(node, txnId, participants, (success, fail) -> {
+ commandStore.execute(contextFor(txnId), safeStore -> {
if (progress() != Investigating)
return;
setProgress(Expected);
- if (fail == null && success.asProgressToken().durability.isDurable())
- setProgress(NoneExpected);
+ SafeCommand safeCommand = safeStore.ifInitialised(txnId);
+ Invariants.checkState(safeCommand != null, "No initialised command for %s but have active progress log", txnId);
+ record(safeCommand.current().known());
}).begin(commandStore.agent());
});
}
@@ -561,12 +423,13 @@ public class SimpleProgressLog implements ProgressLog.Factory
}
@Override
- void run(Command command)
+ void run(SafeCommandStore safeStore, SafeCommand safeCommand)
{
+ Command command = safeCommand.current();
// make sure a quorum of the home shard is aware of the transaction, so we can rely on it to ensure progress
- AsyncChain<Void> inform = inform(node, txnId, command.homeKey());
+ AsyncChain<Void> inform = inform(node, txnId, command.route());
inform.begin((success, fail) -> {
- commandStore.execute(PreLoadContext.empty(), ignore -> {
+ commandStore.execute(empty(), ignore -> {
if (progress() == Done)
return;
@@ -585,7 +448,6 @@ public class SimpleProgressLog implements ProgressLog.Factory
final TxnId txnId;
CoordinateState coordinateState;
- DisseminateState disseminateState;
NonHomeState nonHomeState;
BlockingState blockingState;
@@ -594,12 +456,12 @@ public class SimpleProgressLog implements ProgressLog.Factory
this.txnId = txnId;
}
- void recordBlocking(TxnId txnId, Known waitingFor, Unseekables<?, ?> routables)
+ void recordBlocking(TxnId txnId, LocalExecution waitingFor, Route<?> route, Participants<?> participants)
{
Invariants.checkArgument(txnId.equals(this.txnId));
if (blockingState == null)
blockingState = new BlockingState();
- blockingState.recordBlocking(waitingFor, routables);
+ blockingState.recordBlocking(waitingFor, route, participants);
}
CoordinateState local()
@@ -609,13 +471,6 @@ public class SimpleProgressLog implements ProgressLog.Factory
return coordinateState;
}
- DisseminateState global()
- {
- if (disseminateState == null)
- disseminateState = new DisseminateState();
- return disseminateState;
- }
-
void ensureAtLeast(Command command, CoordinateStatus newStatus, Progress newProgress)
{
local().ensureAtLeast(command, newStatus, newProgress);
@@ -650,7 +505,7 @@ public class SimpleProgressLog implements ProgressLog.Factory
}
final CommandStore commandStore;
- final Map<TxnId, State> stateMap = new HashMap<>();
+ final Map<TxnId, State> stateMap = new TreeMap<>();
boolean isScheduled;
Instance(CommandStore commandStore)
@@ -703,7 +558,7 @@ public class SimpleProgressLog implements ProgressLog.Factory
}
@Override
- public void unwitnessed(TxnId txnId, RoutingKey homeKey, ProgressShard shard)
+ public void unwitnessed(TxnId txnId, ProgressShard shard)
{
if (shard.isHome())
ensure(txnId).ensureAtLeast(Uncommitted, Expected);
@@ -735,9 +590,15 @@ public class SimpleProgressLog implements ProgressLog.Factory
}
@Override
- public void readyToExecute(Command command, ProgressShard shard)
+ public void readyToExecute(Command command)
{
- ensureSafeOrAtLeast(command, shard, ReadyToExecute, Expected);
+ State state = stateMap.get(command.txnId());
+ if (state == null)
+ return; // not progress shard, and nothing blocking
+
+ Invariants.checkState(state.nonHomeState == null || state.nonHomeState.progress() == Done, "nonHomeState should have been set safe by call to committed");
+ if (state.coordinateState != null)
+ state.coordinateState.ensureAtLeast(command, ReadyToExecute, Expected);
}
@Override
@@ -749,53 +610,43 @@ public class SimpleProgressLog implements ProgressLog.Factory
}
@Override
- public void invalidated(Command command, ProgressShard shard)
+ public void clear(TxnId txnId)
{
- State state = recordApply(command.txnId());
-
- Invariants.checkState(shard == Home || state == null || state.coordinateState == null);
-
- // note: we permit Unsure here, so we check if we have any local home state
- if (shard.isProgress())
- {
- state = ensure(command.txnId(), state);
-
- if (shard.isHome()) state.ensureAtLeast(command, CoordinateStatus.Done, Done);
- else ensure(command.txnId()).setSafe();
- }
- }
+ State state = stateMap.remove(txnId);
+ if (state == null)
+ return;
- @Override
- public void durableLocal(TxnId txnId)
- {
- State state = ensure(txnId);
- state.global().durableLocal(node);
+ if (state.blockingState != null)
+ state.blockingState.ensureDone();
+ if (state.nonHomeState != null)
+ state.nonHomeState.ensureDone();
+ if (state.coordinateState != null)
+ state.coordinateState.durableGlobal();
}
@Override
- public void durable(Command command, @Nullable Set<Id> persistedOn)
+ public void durable(Command command)
{
State state = ensure(command.txnId());
- if (!command.status().hasBeen(PreApplied))
- state.recordBlocking(command.txnId(), PreApplied.minKnown, command.maxUnseekables());
- state.local().durableGlobal();
- state.global().durableGlobal(node, command, persistedOn);
+ // if we participate in the transaction and its outcome hasn't been recorded locally then fetch it
+ // TODO (expected): we should delete the in-memory state once we reach Done, and guard against backward progress with Command state
+ // however, we need to be careful:
+ // - we might participate in the execution epoch so need to be sure we have received a route covering both
+ if (!command.status().hasBeen(PreApplied) && command.route() != null && command.route().hasParticipants())
+ state.recordBlocking(command.txnId(), WaitingToApply, command.route(), null);
+ if (state.coordinateState != null)
+ state.coordinateState.durableGlobal();
}
@Override
- public void durable(TxnId txnId, Unseekables<?, ?> unseekables, ProgressShard shard)
+ public void waiting(SafeCommand blockedBy, LocalExecution blockedUntil, Route<?> blockedOnRoute, Participants<?> blockedOnParticipants)
{
- State state = ensure(txnId);
- // TODO (desirable, efficiency): we can probably simplify things by requiring (empty) Apply messages to be sent also to the coordinating topology
- state.recordBlocking(txnId, PreApplied.minKnown, unseekables);
- }
-
- @Override
- public void waiting(TxnId blockedBy, Known blockedUntil, Unseekables<?, ?> blockedOn)
- {
- if (blockedBy.rw().isLocal())
+ if (blockedBy.txnId().rw().isLocal())
return;
+ // ensure we have a record to work with later; otherwise may think has been truncated
+ blockedBy.initialise();
+
// TODO (consider): consider triggering a preemption of existing coordinator (if any) in some circumstances;
// today, an LWT can pre-empt more efficiently (i.e. instantly) a failed operation whereas Accord will
// wait for some progress interval before taking over; there is probably some middle ground where we trigger
@@ -803,7 +654,7 @@ public class SimpleProgressLog implements ProgressLog.Factory
// TODO (desirable, efficiency): forward to local progress shard for processing (if known)
// TODO (desirable, efficiency): if we are co-located with the home shard, don't need to do anything unless we're in a
// later topology that wasn't covered by its coordination
- ensure(blockedBy).recordBlocking(blockedBy, blockedUntil, blockedOn);
+ ensure(blockedBy.txnId()).recordBlocking(blockedBy.txnId(), blockedUntil, blockedOnRoute, blockedOnParticipants);
}
@Override
@@ -825,7 +676,7 @@ public class SimpleProgressLog implements ProgressLog.Factory
return;
isScheduled = true;
- node.scheduler().once(() -> commandStore.execute(PreLoadContext.empty(), ignore -> run()).begin(commandStore.agent()), 1L, TimeUnit.SECONDS);
+ node.scheduler().once(() -> commandStore.execute(empty(), ignore -> run()).begin(commandStore.agent()), 1L, TimeUnit.SECONDS);
}
@Override
@@ -840,7 +691,11 @@ public class SimpleProgressLog implements ProgressLog.Factory
{
commandStore.execute(contextFor(run.txnId()), safeStore -> {
if (run.shouldRun()) // could have been completed by a callback
- run.run(safeStore.command(run.txnId()).current());
+ {
+ SafeCommand safeCommand = safeStore.ifInitialised(run.txnId());
+ if (safeCommand != null && run.shouldRun()) // could be truncated on access
+ run.run(safeStore, safeCommand);
+ }
}).begin(commandStore.agent());
}
}
diff --git a/accord-core/src/main/java/accord/local/Bootstrap.java b/accord-core/src/main/java/accord/local/Bootstrap.java
index b66a0d9a..638e5ea8 100644
--- a/accord-core/src/main/java/accord/local/Bootstrap.java
+++ b/accord-core/src/main/java/accord/local/Bootstrap.java
@@ -119,8 +119,8 @@ class Bootstrap
void start(SafeCommandStore safeStore0)
{
- globalSyncId = new TxnId(node.uniqueNowWithStaleEpoch(epoch), ExclusiveSyncPoint, Routable.Domain.Range);
- localSyncId = globalSyncId.as(LocalOnly);
+ globalSyncId = node.nextTxnId(ExclusiveSyncPoint, Routable.Domain.Range);
+ localSyncId = globalSyncId.as(LocalOnly).withStaleEpoch(epoch);
if (!node.topology().hasEpoch(globalSyncId.epoch()))
{
@@ -141,8 +141,6 @@ class Bootstrap
return AsyncResults.success(Ranges.EMPTY);
Commands.commitRecipientLocalSyncPoint(safeStore1, localSyncId, syncPoint, valid);
- // TODO (now): this should use a dedicated local id, distinct from the one we use to coordinate globally, as this may also be committed and applied locally
- // TODO (now): should we even be putting any partialDeps here? Doesn't seem like it, as they're handled on source nodes.
safeStore1.registerHistoricalTransactions(syncPoint.waitFor);
return fetch = safeStore1.dataStore().fetch(node, safeStore1, valid, syncPoint, this);
})))
@@ -164,7 +162,7 @@ class Bootstrap
if (!valid.intersects(invalidate))
return;
- valid = valid.difference(invalidate);
+ valid = valid.subtract(invalidate);
abort = fetch;
// only cancel the outer future if we have no more ranges to fetch
cancel = valid.isEmpty() ? this.cancel : null;
@@ -326,7 +324,7 @@ class Bootstrap
if (newFailures.isEmpty())
return;
- valid = valid.difference(newFailures);
+ valid = valid.subtract(newFailures);
hasFailed = valid.isEmpty();
}
@@ -349,7 +347,7 @@ class Bootstrap
if (state.safeToReadAt == null)
return;
- Ranges newDone = fetched.slice(state.ranges.difference(fetchedAndSafeToRead), Minimal);
+ Ranges newDone = fetched.slice(state.ranges.subtract(fetchedAndSafeToRead), Minimal);
if (newDone.isEmpty())
return;
@@ -359,7 +357,7 @@ class Bootstrap
// then we are not a definitive bound for safely starting reads, so remove the range
if (overlap.startedAt > state.startedAt)
{
- newDone = newDone.difference(overlap.ranges);
+ newDone = newDone.subtract(overlap.ranges);
if (newDone.isEmpty())
return;
}
@@ -402,7 +400,7 @@ class Bootstrap
if (!fetchedAndSafeToRead.containsAll(fetchCompleted ? fetched : valid))
return;
- retry = valid.difference(fetchedAndSafeToRead);
+ retry = valid.subtract(fetchedAndSafeToRead);
completed = true;
}
@@ -461,7 +459,7 @@ class Bootstrap
Invariants.checkArgument(inProgress.contains(attempt));
Invariants.checkArgument(attempt.fetched.equals(attempt.fetchedAndSafeToRead));
inProgress.remove(attempt);
- remaining = remaining.difference(attempt.fetched);
+ remaining = remaining.subtract(attempt.fetched);
if (inProgress.isEmpty() && remaining.isEmpty())
{
// TODO (now): this waits too long?
@@ -475,8 +473,8 @@ class Bootstrap
// distinct from abort as triggered by ourselves when we no longer own the range
synchronized void invalidate(Ranges invalidate)
{
- allValid = allValid.difference(invalidate);
- remaining = remaining.difference(invalidate);
+ allValid = allValid.subtract(invalidate);
+ remaining = remaining.subtract(invalidate);
for (Attempt attempt : inProgress)
attempt.invalidate(invalidate);
}
diff --git a/accord-core/src/main/java/accord/local/Command.java b/accord-core/src/main/java/accord/local/Command.java
index c9990b0c..dfad5f68 100644
--- a/accord-core/src/main/java/accord/local/Command.java
+++ b/accord-core/src/main/java/accord/local/Command.java
@@ -18,33 +18,42 @@
package accord.local;
+import java.util.Objects;
+
+import com.google.common.collect.ImmutableSortedSet;
+
import accord.api.Data;
+import accord.api.Result;
+import accord.api.RoutingKey;
+import accord.api.VisibleForImplementation;
import accord.primitives.Ballot;
+import accord.primitives.Deps;
import accord.primitives.Keys;
import accord.primitives.PartialDeps;
-import accord.primitives.PartialRoute;
import accord.primitives.PartialTxn;
+import accord.primitives.Ranges;
import accord.primitives.Route;
import accord.primitives.Timestamp;
+import accord.primitives.Txn;
import accord.primitives.TxnId;
import accord.primitives.Unseekables;
import accord.primitives.Writes;
+import accord.utils.ImmutableBitSet;
+import accord.utils.IndexedQuadConsumer;
import accord.utils.Invariants;
-
-import javax.annotation.Nullable;
-
-import accord.api.Result;
-import accord.api.RoutingKey;
-import accord.api.VisibleForImplementation;
+import accord.utils.SimpleBitSet;
import accord.utils.async.AsyncChain;
-import com.google.common.collect.ImmutableSortedMap;
-import com.google.common.collect.ImmutableSortedSet;
-
-import java.util.*;
+import javax.annotation.Nullable;
+import static accord.local.Listeners.Immutable.EMPTY;
+import static accord.local.SaveStatus.Uninitialised;
+import static accord.local.Status.Durability.DurableOrInvalidated;
import static accord.local.Status.Durability.Local;
import static accord.local.Status.Durability.NotDurable;
-import static accord.utils.Utils.*;
+import static accord.local.Status.KnownExecuteAt.ExecuteAtKnown;
+import static accord.utils.SortedArrays.forEachIntersection;
+import static accord.utils.Utils.ensureImmutable;
+import static accord.utils.Utils.ensureMutable;
import static java.lang.String.format;
public abstract class Command implements CommonAttributes
@@ -105,7 +114,7 @@ public abstract class Command implements CommonAttributes
@Override
public void onChange(SafeCommandStore safeStore, SafeCommand safeCommand)
{
- Commands.listenerUpdate(safeStore, safeStore.command(listenerId), safeCommand);
+ Commands.listenerUpdate(safeStore, safeStore.get(listenerId), safeCommand);
}
@Override
@@ -131,9 +140,9 @@ public abstract class Command implements CommonAttributes
@VisibleForImplementation
public static class SerializerSupport
{
- public static NotWitnessed notWitnessed(CommonAttributes attributes, Ballot promised)
+ public static NotDefined notDefined(CommonAttributes attributes, Ballot promised)
{
- return NotWitnessed.notWitnessed(attributes, promised);
+ return NotDefined.notDefined(attributes, promised);
}
public static PreAccepted preaccepted(CommonAttributes common, Timestamp executeAt, Ballot promised)
@@ -146,14 +155,24 @@ public abstract class Command implements CommonAttributes
return Accepted.accepted(common, status, executeAt, promised, accepted);
}
- public static Committed committed(CommonAttributes common, SaveStatus status, Timestamp executeAt, Ballot promised, Ballot accepted, ImmutableSortedSet<TxnId> waitingOnCommit, ImmutableSortedMap<Timestamp, TxnId> waitingOnApply)
+ public static Committed committed(CommonAttributes common, SaveStatus status, Timestamp executeAt, Ballot promised, Ballot accepted, WaitingOn waitingOn)
+ {
+ return Committed.committed(common, status, executeAt, promised, accepted, waitingOn);
+ }
+
+ public static Executed executed(CommonAttributes common, SaveStatus status, Timestamp executeAt, Ballot promised, Ballot accepted, WaitingOn waitingOn, Writes writes, Result result)
{
- return Committed.committed(common, status, executeAt, promised, accepted, waitingOnCommit, waitingOnApply);
+ return Executed.executed(common, status, executeAt, promised, accepted, waitingOn, writes, result);
}
- public static Executed executed(CommonAttributes common, SaveStatus status, Timestamp executeAt, Ballot promised, Ballot accepted, ImmutableSortedSet<TxnId> waitingOnCommit, ImmutableSortedMap<Timestamp, TxnId> waitingOnApply, Writes writes, Result result)
+ public static Truncated invalidated(TxnId txnId, Listeners.Immutable durableListeners)
{
- return Executed.executed(common, status, executeAt, promised, accepted, waitingOnCommit, waitingOnApply, writes, result);
+ return Truncated.invalidated(txnId, durableListeners);
+ }
+
+ public static Truncated truncatedApply(CommonAttributes common,SaveStatus saveStatus, Timestamp executeAt, Writes writes, Result result)
+ {
+ return Truncated.truncatedApply(common, saveStatus, executeAt, writes, result);
}
}
@@ -171,8 +190,8 @@ public abstract class Command implements CommonAttributes
{
switch (status.status)
{
- case NotWitnessed:
- return validateCommandClass(status, NotWitnessed.class, klass);
+ case NotDefined:
+ return validateCommandClass(status, NotDefined.class, klass);
case PreAccepted:
return validateCommandClass(status, PreAccepted.class, klass);
case AcceptedInvalidate:
@@ -184,8 +203,10 @@ public abstract class Command implements CommonAttributes
return validateCommandClass(status, Committed.class, klass);
case PreApplied:
case Applied:
- case Invalidated:
return validateCommandClass(status, Executed.class, klass);
+ case Invalidated:
+ case Truncated:
+ return validateCommandClass(status, Truncated.class, klass);
default:
throw new IllegalStateException("Unhandled status " + status);
}
@@ -196,19 +217,15 @@ public abstract class Command implements CommonAttributes
private final TxnId txnId;
private final SaveStatus status;
private final Status.Durability durability;
- private final RoutingKey homeKey;
- private final RoutingKey progressKey;
private final Route<?> route;
private final Ballot promised;
private final Listeners.Immutable listeners;
- private AbstractCommand(TxnId txnId, SaveStatus status, Status.Durability durability, RoutingKey homeKey, RoutingKey progressKey, Route<?> route, Ballot promised, Listeners.Immutable listeners)
+ private AbstractCommand(TxnId txnId, SaveStatus status, Status.Durability durability, Route<?> route, Ballot promised, Listeners.Immutable listeners)
{
this.txnId = txnId;
this.status = validateCommandClass(status, getClass());
this.durability = durability;
- this.homeKey = homeKey;
- this.progressKey = progressKey;
this.route = route;
this.promised = promised;
this.listeners = listeners;
@@ -219,8 +236,6 @@ public abstract class Command implements CommonAttributes
this.txnId = common.txnId();
this.status = validateCommandClass(status, getClass());
this.durability = common.durability();
- this.homeKey = common.homeKey();
- this.progressKey = common.progressKey();
this.route = common.route();
this.promised = promised;
this.listeners = common.durableListeners();
@@ -235,8 +250,6 @@ public abstract class Command implements CommonAttributes
return txnId.equals(command.txnId())
&& status == command.saveStatus()
&& durability == command.durability()
- && Objects.equals(homeKey, command.homeKey())
- && Objects.equals(progressKey, command.progressKey())
&& Objects.equals(route, command.route())
&& Objects.equals(promised, command.promised())
&& listeners.equals(command.durableListeners());
@@ -254,18 +267,6 @@ public abstract class Command implements CommonAttributes
return txnId;
}
- @Override
- public final RoutingKey homeKey()
- {
- return homeKey;
- }
-
- @Override
- public final RoutingKey progressKey()
- {
- return progressKey;
- }
-
@Override
public final Route<?> route()
{
@@ -288,7 +289,7 @@ public abstract class Command implements CommonAttributes
public Listeners.Immutable durableListeners()
{
if (listeners == null)
- return Listeners.Immutable.EMPTY;
+ return EMPTY;
return listeners;
}
@@ -306,38 +307,45 @@ public abstract class Command implements CommonAttributes
}
/**
- * If this is the home shard, we require that this is a Route for all states > NotWitnessed;
- * otherwise for the local progress shard this is ordinarily a PartialRoute, and for other shards this is not set,
- * so that there is only one copy per node that can be consulted to construct the full set of involved keys.
+ * We require that this is a FullRoute for all states where isDefinitionKnown().
+ * In some cases, the home shard will contain an arbitrary slice of the Route where !isDefinitionKnown(),
+ * i.e. when a non-home shard informs the home shards of a transaction to ensure forward progress.
*
* If hasBeen(Committed) this must contain the keys for both txnId.epoch and executeAt.epoch
+ *
+ * TODO (required): audit uses; do not assume null means it is a complete route for the shard;
+ * preferably introduce two variations so callers can declare whether they need the full shard's route
+ * or any route will do
*/
+ @Override
public abstract Route<?> route();
/**
- * A key nominated to be the primary shard within this node for managing progress of the command.
- * It is nominated only as of txnId.epoch, and may be null (indicating that this node does not monitor
- * the progress of this command).
- *
- * Preferentially, this is homeKey on nodes that replicate it, and otherwise any key that is replicated, as of txnId.epoch
+ * The command may have an incomplete route when this is false
*/
- public abstract RoutingKey progressKey();
+ public boolean hasFullRoute()
+ {
+ return route() != null && route().kind().isFullRoute();
+ }
/**
* homeKey is a global value that defines the home shard - the one tasked with ensuring the transaction is finished.
* progressKey is a local value that defines the local shard responsible for ensuring progress on the transaction.
* This will be homeKey if it is owned by the node, and some other key otherwise. If not the home shard, the progress
* shard has much weaker responsibilities, only ensuring that the home shard has durably witnessed the txnId.
- *
- * TODO (expected, efficiency): we probably do not want to save this on its own, as we probably want to
- * minimize IO interactions and discrete registers, so will likely reference commit log entries directly
- * At which point we may impose a requirement that only a Route can be saved, not a homeKey on its own.
- * Once this restriction is imposed, we no longer need to pass around Routable.Domain with TxnId.
*/
- public abstract RoutingKey homeKey();
+ public RoutingKey homeKey()
+ {
+ Route<?> route = route();
+ return route == null ? null : route.homeKey();
+ }
+
+ @Override
public abstract TxnId txnId();
public abstract Ballot promised();
+ @Override
public abstract Status.Durability durability();
+ @Override
public abstract Listeners.Immutable durableListeners();
public abstract SaveStatus saveStatus();
@@ -368,36 +376,52 @@ public abstract class Command implements CommonAttributes
throw new IllegalArgumentException(errorMsg + format(" expected %s got %s", klass.getSimpleName(), command.getClass().getSimpleName()));
}
- // TODO (low priority, progress): callers should try to consult the local progress shard (if any) to obtain the full set of keys owned locally
- public final Route<?> someRoute()
+ public PreLoadContext contextForSelf()
{
- if (route() != null)
- return route();
+ return contextForCommand(this);
+ }
+
+ public abstract Timestamp executeAt();
+ public abstract Ballot accepted();
+
+ @Override
+ public abstract PartialTxn partialTxn();
+
+ @Override
+ public abstract @Nullable PartialDeps partialDeps();
- if (homeKey() != null)
- return PartialRoute.empty(txnId().domain(), homeKey());
+ public @Nullable Writes writes() { return null; }
+ public @Nullable Result result() { return null; }
- return null;
+ public final Timestamp executeAtIfKnownElseTxnId()
+ {
+ if (known().executeAt == ExecuteAtKnown)
+ return executeAt();
+ return txnId();
}
- public Unseekables<?, ?> maxUnseekables()
+ public final Timestamp executeAtIfKnown()
{
- Route<?> route = someRoute();
- if (route == null)
- return null;
+ return executeAtIfKnown(null);
+ }
- return route.toMaximalUnseekables();
+ public final Timestamp executeAtIfKnown(Timestamp orElse)
+ {
+ if (known().executeAt == ExecuteAtKnown)
+ return executeAt();
+ return orElse;
}
- public PreLoadContext contextForSelf()
+ public final boolean executesInFutureEpoch()
{
- return contextForCommand(this);
+ return known().executeAt == ExecuteAtKnown && executeAt().epoch() > txnId().epoch();
}
- public abstract Timestamp executeAt();
- public abstract Ballot accepted();
- public abstract PartialTxn partialTxn();
- public abstract @Nullable PartialDeps partialDeps();
+ public final Timestamp executeAtOrTxnId()
+ {
+ Timestamp executeAt = executeAt();
+ return executeAt == null || executeAt.equals(Timestamp.NONE) ? txnId() : executeAt;
+ }
public final Status status()
{
@@ -419,6 +443,11 @@ public abstract class Command implements CommonAttributes
return known.isSatisfiedBy(saveStatus().known);
}
+ public boolean isAtLeast(SaveStatus.LocalExecution execution)
+ {
+ return saveStatus().execution.compareTo(execution) >= 0;
+ }
+
public boolean has(Status.Definition definition)
{
return known().definition.compareTo(definition) >= 0;
@@ -439,10 +468,12 @@ public abstract class Command implements CommonAttributes
return new ProxyListener(txnId());
}
- public final boolean isWitnessed()
+ public final boolean isDefined()
{
+ if (status().hasBeen(Status.Truncated))
+ return false;
boolean result = status().hasBeen(Status.PreAccepted);
- Invariants.checkState(result == (this instanceof PreAccepted));
+ Invariants.checkState(result == (this instanceof PreAccepted), "Unexpected type: %s, %s", this, this.getClass().getCanonicalName());
return result;
}
@@ -454,7 +485,7 @@ public abstract class Command implements CommonAttributes
public final boolean isAccepted()
{
boolean result = status().hasBeen(Status.AcceptedInvalidate);
- Invariants.checkState(result == (this instanceof Accepted));
+ Invariants.checkState(result == (this instanceof Accepted), "Unexpected type: %s, %s", this, this.getClass().getCanonicalName());
return result;
}
@@ -466,7 +497,7 @@ public abstract class Command implements CommonAttributes
public final boolean isCommitted()
{
boolean result = status().hasBeen(Status.Committed);
- Invariants.checkState(result == (this instanceof Committed));
+ Invariants.checkState(result == (this instanceof Committed), "Unexpected type: %s, %s", this, this.getClass().getCanonicalName());
return result;
}
@@ -475,16 +506,16 @@ public abstract class Command implements CommonAttributes
return Invariants.cast(this, Committed.class);
}
- public final boolean isExecuted()
+ public final Executed asExecuted()
{
- boolean result = status().hasBeen(Status.PreApplied);
- Invariants.checkState(result == (this instanceof Executed));
- return result;
+ return Invariants.cast(this, Executed.class);
}
- public final Executed asExecuted()
+ public final boolean isTruncated()
{
- return Invariants.cast(this, Executed.class);
+ boolean result = status().hasBeen(Status.Truncated);
+ Invariants.checkState(result == (this instanceof Truncated), "Unexpected type: %s, %s", this, this.getClass().getCanonicalName());
+ return result;
}
public abstract Command updateAttributes(CommonAttributes attrs, Ballot promised);
@@ -499,14 +530,14 @@ public abstract class Command implements CommonAttributes
return updateAttributes(this, promised);
}
- public static final class NotWitnessed extends AbstractCommand
+ public static final class NotDefined extends AbstractCommand
{
- NotWitnessed(TxnId txnId, SaveStatus status, Status.Durability durability, RoutingKey homeKey, RoutingKey progressKey, Route<?> route, Ballot promised, Listeners.Immutable listeners)
+ NotDefined(TxnId txnId, SaveStatus status, Status.Durability durability, Route<?> route, Ballot promised, Listeners.Immutable listeners)
{
- super(txnId, status, durability, homeKey, progressKey, route, promised, listeners);
+ super(txnId, status, durability, route, promised, listeners);
}
- NotWitnessed(CommonAttributes common, SaveStatus status, Ballot promised)
+ NotDefined(CommonAttributes common, SaveStatus status, Ballot promised)
{
super(common, status, promised);
}
@@ -514,24 +545,17 @@ public abstract class Command implements CommonAttributes
@Override
public Command updateAttributes(CommonAttributes attrs, Ballot promised)
{
- return new NotWitnessed(attrs, saveStatus(), promised);
+ return new NotDefined(attrs, initialise(saveStatus()), promised);
}
- public static NotWitnessed notWitnessed(CommonAttributes common, Ballot promised)
+ public static NotDefined notDefined(CommonAttributes common, Ballot promised)
{
- return new NotWitnessed(common, SaveStatus.NotWitnessed, promised);
+ return new NotDefined(common, SaveStatus.NotDefined, promised);
}
- public static NotWitnessed notWitnessed(TxnId txnId)
+ public static NotDefined uninitialised(TxnId txnId)
{
- return new NotWitnessed(txnId, SaveStatus.NotWitnessed, NotDurable, null, null, null, Ballot.ZERO, null);
- }
-
- public static NotWitnessed notWitnessed(NotWitnessed command, CommonAttributes common, Ballot promised)
- {
- checkSameClass(command, NotWitnessed.class, "Cannot update");
- Invariants.checkArgument(command.txnId().equals(common.txnId()));
- return new NotWitnessed(common, command.saveStatus(), promised);
+ return new NotDefined(txnId, Uninitialised, NotDurable, null, Ballot.ZERO, null);
}
@Override
@@ -564,6 +588,113 @@ public abstract class Command implements CommonAttributes
{
return null;
}
+
+ private static SaveStatus initialise(SaveStatus saveStatus)
+ {
+ return saveStatus == Uninitialised ? SaveStatus.NotDefined : saveStatus;
+ }
+ }
+
+ public static final class Truncated extends AbstractCommand
+ {
+ @Nullable final Timestamp executeAt;
+ @Nullable final Writes writes;
+ @Nullable final Result result;
+ public Truncated(CommonAttributes commonAttributes, SaveStatus saveStatus, @Nullable Timestamp executeAt, @Nullable Writes writes, @Nullable Result result)
+ {
+ super(commonAttributes, saveStatus, Ballot.MAX);
+ this.executeAt = executeAt;
+ this.writes = writes;
+ this.result = result;
+ }
+
+ public Truncated(TxnId txnId, SaveStatus saveStatus, Status.Durability durability, Route<?> route, @Nullable Timestamp executeAt, Listeners.Immutable listeners, @Nullable Writes writes, @Nullable Result result)
+ {
+ super(txnId, saveStatus, durability, route, Ballot.MAX, listeners);
+ this.executeAt = executeAt;
+ this.writes = writes;
+ this.result = result;
+ }
+
+ public static Truncated erased(Command command)
+ {
+ return new Truncated(command.txnId(), SaveStatus.Erased, command.durability(), command.route(), null, EMPTY, null, null);
+ }
+
+ public static Truncated truncatedApply(Command command)
+ {
+ Invariants.checkArgument(command.known().executeAt.hasDecidedExecuteAt());
+ return new Truncated(command.txnId(), SaveStatus.TruncatedApply, command.durability(), command.route(), command.executeAtIfKnown(), EMPTY, null, null);
+ }
+
+ public static Truncated truncatedApplyWithOutcome(Executed command)
+ {
+ return new Truncated(command.txnId(), SaveStatus.TruncatedApplyWithOutcome, command.durability(), command.route(), command.executeAt(), EMPTY, command.writes, command.result);
+ }
+
+ public static Truncated truncatedApply(CommonAttributes common, SaveStatus saveStatus, Timestamp executeAt, Writes writes, Result result)
+
+ {
+ Invariants.checkArgument(executeAt != null);
+ Invariants.checkArgument(saveStatus == SaveStatus.TruncatedApply || saveStatus == SaveStatus.TruncatedApplyWithDeps || saveStatus == SaveStatus.TruncatedApplyWithOutcome);
+ // TODO review Is this correctly handling all three versions of truncatedApplyStatus? Should writes be null some of the time?
+ return new Truncated(common.txnId(), saveStatus, common.durability(), common.route(), executeAt, EMPTY, writes, result);
+ }
+
+ public static Truncated invalidated(Command command)
+ {
+ Invariants.checkState(!command.hasBeen(Status.PreCommitted));
+ // TODO (now): we shouldn't need to propagate these
+ return invalidated(command.txnId(), command.durableListeners());
+ }
+
+ public static Truncated invalidated(TxnId txnId, Listeners.Immutable durableListeners)
+ {
+ return new Truncated(txnId, SaveStatus.Invalidated, DurableOrInvalidated, null, Timestamp.NONE, durableListeners, null, null);
+ }
+
+ @Override
+ public Timestamp executeAt()
+ {
+ return executeAt;
+ }
+
+ @Override
+ public @Nullable Writes writes()
+ {
+ return writes;
+ }
+
+ @Override
+ public @Nullable Result result()
+ {
+ return result;
+ }
+
+ @Override
+ public Ballot accepted()
+ {
+ return Ballot.MAX;
+ }
+
+ @Override
+ public PartialTxn partialTxn()
+ {
+ return null;
+ }
+
+ @Override
+ public @Nullable PartialDeps partialDeps()
+ {
+ return null;
+ }
+
+ @Override
+ public Command updateAttributes(CommonAttributes attrs, Ballot promised)
+ {
+ // TODO (now): invoke listeners precisely once when we adopt this state, then we can simply return `this`
+ return new Truncated(txnId(), saveStatus(), attrs.durability(), attrs.route(), executeAt, attrs.durableListeners(), writes, result);
+ }
}
public static class PreAccepted extends AbstractCommand
@@ -685,25 +816,20 @@ public abstract class Command implements CommonAttributes
public static class Committed extends Accepted
{
- private final ImmutableSortedSet<TxnId> waitingOnCommit;
- private final ImmutableSortedMap<Timestamp, TxnId> waitingOnApply;
-
- private Committed(CommonAttributes common, SaveStatus status, Timestamp executeAt, Ballot promised, Ballot accepted, ImmutableSortedSet<TxnId> waitingOnCommit, ImmutableSortedMap<Timestamp, TxnId> waitingOnApply)
- {
- super(common, status, executeAt, promised, accepted);
- this.waitingOnCommit = waitingOnCommit;
- this.waitingOnApply = waitingOnApply;
- }
+ public final WaitingOn waitingOn;
private Committed(CommonAttributes common, SaveStatus status, Timestamp executeAt, Ballot promised, Ballot accepted, WaitingOn waitingOn)
{
- this(common, status, executeAt, promised, accepted, waitingOn.waitingOnCommit, waitingOn.waitingOnApply);
+ super(common, status, executeAt, promised, accepted);
+ this.waitingOn = waitingOn;
+ Invariants.checkState(common.route().kind().isFullRoute(), "Expected a full route but given %s", common.route().kind());
+ Invariants.checkState(waitingOn.deps.equals(common.partialDeps()), "Deps do not match; expected %s == %s", waitingOn.deps, common.partialDeps());
}
@Override
public Command updateAttributes(CommonAttributes attrs, Ballot promised)
{
- return new Committed(attrs, saveStatus(), executeAt(), promised, accepted(), waitingOnCommit(), waitingOnApply());
+ return new Committed(attrs, saveStatus(), executeAt(), promised, accepted(), waitingOn());
}
@Override
@@ -713,35 +839,34 @@ public abstract class Command implements CommonAttributes
if (o == null || getClass() != o.getClass()) return false;
if (!super.equals(o)) return false;
Committed committed = (Committed) o;
- return Objects.equals(waitingOnCommit, committed.waitingOnCommit)
- && Objects.equals(waitingOnApply, committed.waitingOnApply);
+ return Objects.equals(waitingOn, committed.waitingOn);
}
- private static Committed committed(Committed command, CommonAttributes common, Ballot promised, SaveStatus status, ImmutableSortedSet<TxnId> waitingOnCommit, ImmutableSortedMap<Timestamp, TxnId> waitingOnApply)
+ private static Committed committed(Committed command, CommonAttributes common, Ballot promised, SaveStatus status, WaitingOn waitingOn)
{
checkPromised(command, promised);
checkSameClass(command, Committed.class, "Cannot update");
- return new Committed(common, status, command.executeAt(), promised, command.accepted(), waitingOnCommit, waitingOnApply);
+ return new Committed(common, status, command.executeAt(), promised, command.accepted(), waitingOn);
}
static Committed committed(Committed command, CommonAttributes common, Ballot promised)
{
- return committed(command, common, promised, command.saveStatus(), command.waitingOnCommit(), command.waitingOnApply());
+ return committed(command, common, promised, command.saveStatus(), command.waitingOn());
}
static Committed committed(Committed command, CommonAttributes common, SaveStatus status)
{
- return committed(command, common, command.promised(), status, command.waitingOnCommit(), command.waitingOnApply());
+ return committed(command, common, command.promised(), status, command.waitingOn());
}
static Committed committed(Committed command, CommonAttributes common, WaitingOn waitingOn)
{
- return committed(command, common, command.promised(), command.saveStatus(), waitingOn.waitingOnCommit, waitingOn.waitingOnApply);
+ return committed(command, common, command.promised(), command.saveStatus(), waitingOn);
}
- static Committed committed(CommonAttributes common, SaveStatus status, Timestamp executeAt, Ballot promised, Ballot accepted, ImmutableSortedSet<TxnId> waitingOnCommit, ImmutableSortedMap<Timestamp, TxnId> waitingOnApply)
+ static Committed committed(CommonAttributes common, SaveStatus status, Timestamp executeAt, Ballot promised, Ballot accepted, WaitingOn waitingOn)
{
- return new Committed(common, status, executeAt, promised, accepted, waitingOnCommit, waitingOnApply);
+ return new Committed(common, status, executeAt, promised, accepted, waitingOn);
}
public AsyncChain<Data> read(SafeCommandStore safeStore)
@@ -751,37 +876,17 @@ public abstract class Command implements CommonAttributes
public WaitingOn waitingOn()
{
- return new WaitingOn(waitingOnCommit, waitingOnApply);
- }
-
- public ImmutableSortedSet<TxnId> waitingOnCommit()
- {
- return waitingOnCommit;
+ return waitingOn;
}
public boolean isWaitingOnCommit()
{
- return waitingOnCommit != null && !waitingOnCommit.isEmpty();
- }
-
- public TxnId firstWaitingOnCommit()
- {
- return isWaitingOnCommit() ? waitingOnCommit.first() : null;
- }
-
- public ImmutableSortedMap<Timestamp, TxnId> waitingOnApply()
- {
- return waitingOnApply;
+ return waitingOn.isWaitingOnCommit();
}
public boolean isWaitingOnApply()
{
- return waitingOnApply != null && !waitingOnApply.isEmpty();
- }
-
- public TxnId firstWaitingOnApply()
- {
- return isWaitingOnApply() ? waitingOnApply.firstEntry().getValue() : null;
+ return waitingOn.isWaitingOnApply();
}
public boolean isWaitingOnDependency()
@@ -795,16 +900,10 @@ public abstract class Command implements CommonAttributes
private final Writes writes;
private final Result result;
- public Executed(CommonAttributes common, SaveStatus status, Timestamp executeAt, Ballot promised, Ballot accepted, ImmutableSortedSet<TxnId> waitingOnCommit, ImmutableSortedMap<Timestamp, TxnId> waitingOnApply, Writes writes, Result result)
- {
- super(common, status, executeAt, promised, accepted, waitingOnCommit, waitingOnApply);
- this.writes = writes;
- this.result = result;
- }
-
public Executed(CommonAttributes common, SaveStatus status, Timestamp executeAt, Ballot promised, Ballot accepted, WaitingOn waitingOn, Writes writes, Result result)
{
super(common, status, executeAt, promised, accepted, waitingOn);
+ Invariants.checkState(txnId().rw() != Txn.Kind.Write || writes != null);
this.writes = writes;
this.result = result;
}
@@ -812,7 +911,7 @@ public abstract class Command implements CommonAttributes
@Override
public Command updateAttributes(CommonAttributes attrs, Ballot promised)
{
- return new Executed(attrs, saveStatus(), executeAt(), promised, accepted(), waitingOnCommit(), waitingOnApply(), writes, result);
+ return new Executed(attrs, saveStatus(), executeAt(), promised, accepted(), waitingOn(), writes, result);
}
@Override
@@ -826,35 +925,30 @@ public abstract class Command implements CommonAttributes
&& Objects.equals(result, executed.result);
}
- public static Executed executed(Executed command, CommonAttributes common, SaveStatus status, Ballot promised, ImmutableSortedSet<TxnId> waitingOnCommit, ImmutableSortedMap<Timestamp, TxnId> waitingOnApply)
+ public static Executed executed(Executed command, CommonAttributes common, SaveStatus status, Ballot promised, WaitingOn waitingOn)
{
checkSameClass(command, Executed.class, "Cannot update");
- return new Executed(common, status, command.executeAt(), promised, command.accepted(), waitingOnCommit, waitingOnApply, command.writes(), command.result());
+ return new Executed(common, status, command.executeAt(), promised, command.accepted(), waitingOn, command.writes(), command.result());
}
public static Executed executed(Executed command, CommonAttributes common, SaveStatus status)
{
- return executed(command, common, status, command.promised(), command.waitingOnCommit(), command.waitingOnApply());
+ return executed(command, common, status, command.promised(), command.waitingOn());
}
public static Executed executed(Executed command, CommonAttributes common, WaitingOn waitingOn)
{
- return executed(command, common, command.saveStatus(), command.promised(), waitingOn.waitingOnCommit, waitingOn.waitingOnApply);
+ return executed(command, common, command.saveStatus(), command.promised(), waitingOn);
}
public static Executed executed(Executed command, CommonAttributes common, Ballot promised)
{
- return executed(command, common, command.saveStatus(), promised, command.waitingOnCommit(), command.waitingOnApply());
- }
-
- public static Executed executed(CommonAttributes common, SaveStatus status, Timestamp executeAt, Ballot promised, Ballot accepted, ImmutableSortedSet<TxnId> waitingOnCommit, ImmutableSortedMap<Timestamp, TxnId> waitingOnApply, Writes writes, Result result)
- {
- return new Executed(common, status, executeAt, promised, accepted, waitingOnCommit, waitingOnApply, writes, result);
+ return executed(command, common, command.saveStatus(), promised, command.waitingOn());
}
public static Executed executed(CommonAttributes common, SaveStatus status, Timestamp executeAt, Ballot promised, Ballot accepted, WaitingOn waitingOn, Writes writes, Result result)
{
- return new Executed(common, status, executeAt, promised, accepted, waitingOn.waitingOnCommit, waitingOn.waitingOnApply, writes, result);
+ return new Executed(common, status, executeAt, promised, accepted, waitingOn, writes, result);
}
public Writes writes()
@@ -870,88 +964,324 @@ public abstract class Command implements CommonAttributes
public static class WaitingOn
{
- public static final WaitingOn EMPTY = new WaitingOn(ImmutableSortedSet.of(), ImmutableSortedMap.of());
- public final ImmutableSortedSet<TxnId> waitingOnCommit;
- public final ImmutableSortedMap<Timestamp, TxnId> waitingOnApply;
+ public static final WaitingOn EMPTY = new WaitingOn(Deps.NONE, ImmutableBitSet.EMPTY, ImmutableBitSet.EMPTY, ImmutableBitSet.EMPTY);
+
+ public final Deps deps;
+ // note that transactions default to waitingOnCommit, so presence in the set does not mean the transaction is uncommitted
+ public final ImmutableBitSet waitingOnCommit, waitingOnApply, appliedOrInvalidated;
- public WaitingOn(ImmutableSortedSet<TxnId> waitingOnCommit, ImmutableSortedMap<Timestamp, TxnId> waitingOnApply)
+ public WaitingOn(Deps deps, ImmutableBitSet waitingOnCommit, ImmutableBitSet waitingOnApply, ImmutableBitSet appliedOrInvalidated)
{
+ this.deps = deps;
this.waitingOnCommit = waitingOnCommit;
this.waitingOnApply = waitingOnApply;
+ this.appliedOrInvalidated = appliedOrInvalidated;
}
- public static class Update
+ public boolean isWaitingOnCommit()
{
- private boolean hasChanges = false;
- private NavigableSet<TxnId> waitingOnCommit;
- private NavigableMap<Timestamp, TxnId> waitingOnApply;
+ return !waitingOnCommit.isEmpty();
+ }
- public Update()
- {
+ public boolean isWaitingOnApply()
+ {
+ return !waitingOnApply.isEmpty();
+ }
- }
+ public boolean isWaitingOn(TxnId txnId)
+ {
+ int index = deps.indexOf(txnId);
+ return index >= 0 && (waitingOnCommit.get(index) || waitingOnApply.get(index));
+ }
+
+ public TxnId nextWaitingOnCommit()
+ {
+ int i = waitingOnCommit.lastSetBit();
+ return i < 0 ? null : deps.txnId(i);
+ }
+
+ public TxnId nextWaitingOnApply()
+ {
+ int i = waitingOnApply.lastSetBit();
+ return i < 0 ? null : deps.txnId(i);
+ }
+
+ public TxnId nextWaitingOn()
+ {
+ TxnId next = nextWaitingOnApply();
+ return next != null ? next : nextWaitingOnCommit();
+ }
+
+ public boolean isAppliedOrInvalidatedRangeIdx(int i)
+ {
+ return appliedOrInvalidated.get(i + deps.keyDeps.txnIdCount());
+ }
+
+ public TxnId minWaitingOnTxnId()
+ {
+ return minWaitingOnTxnId(deps, waitingOnCommit, waitingOnApply);
+ }
+
+ static TxnId minWaitingOnTxnId(Deps deps, SimpleBitSet waitingOnCommit, SimpleBitSet waitingOnApply)
+ {
+ int keyDepsCount = deps.keyDeps.txnIdCount();
+ int minWaitingOnKeys = Math.min(waitingOnCommit.firstSetBitBefore(keyDepsCount, Integer.MAX_VALUE), waitingOnApply.nextSetBitBefore(0, keyDepsCount, Integer.MAX_VALUE));
+ int minWaitingOnRanges = Math.min(waitingOnCommit.nextSetBit(keyDepsCount, Integer.MAX_VALUE), waitingOnApply.nextSetBit(keyDepsCount, Integer.MAX_VALUE));
+ return TxnId.nonNullOrMin(minWaitingOnKeys == Integer.MAX_VALUE ? null : deps.txnId(minWaitingOnKeys),
+ minWaitingOnRanges == Integer.MAX_VALUE ? null : deps.txnId(minWaitingOnRanges));
+ }
+
+ static TxnId minWaitingOn(Deps deps, SimpleBitSet waitingOn)
+ {
+ int keyDepsCount = deps.keyDeps.txnIdCount();
+ int minWaitingOnKeys = waitingOn.firstSetBitBefore(keyDepsCount, -1);
+ int minWaitingOnRanges = waitingOn.nextSetBit(keyDepsCount, -1);
+ return TxnId.nonNullOrMin(minWaitingOnKeys < 0 ? null : deps.keyDeps.txnId(minWaitingOnKeys),
+ minWaitingOnRanges < 0 ? null : deps.rangeDeps.txnId(minWaitingOnRanges - keyDepsCount));
+ }
+
+ static TxnId maxWaitingOn(Deps deps, SimpleBitSet waitingOn)
+ {
+ int keyDepsCount = deps.keyDeps.txnIdCount();
+ int maxWaitingOnRanges = waitingOn.lastSetBitNotBefore(keyDepsCount);
+ int maxWaitingOnKeys = waitingOn.prevSetBit(keyDepsCount);
+ return TxnId.nonNullOrMax(maxWaitingOnKeys < 0 ? null : deps.keyDeps.txnId(maxWaitingOnKeys),
+ maxWaitingOnRanges < 0 ? null : deps.rangeDeps.txnId(maxWaitingOnRanges - keyDepsCount));
+ }
+
+ public ImmutableSortedSet<TxnId> computeWaitingOnCommit()
+ {
+ return computeWaitingOnCommit(deps, waitingOnCommit);
+ }
+
+ public ImmutableSortedSet<TxnId> computeWaitingOnApply()
+ {
+ return computeWaitingOnApply(deps, waitingOnCommit, waitingOnApply);
+ }
+
+ private static ImmutableSortedSet<TxnId> computeWaitingOnCommit(Deps deps, SimpleBitSet waitingOnCommit)
+ {
+ ImmutableSortedSet.Builder<TxnId> builder = new ImmutableSortedSet.Builder<>(TxnId::compareTo);
+ waitingOnCommit.forEach(builder, deps, (b, d, i) -> b.add(d.txnId(i)));
+ return builder.build();
+ }
+
+ private static ImmutableSortedSet<TxnId> computeWaitingOnApply(Deps deps, SimpleBitSet waitingOnCommit, SimpleBitSet waitingOnApply)
+ {
+ ImmutableSortedSet.Builder<TxnId> builder = new ImmutableSortedSet.Builder<>(TxnId::compareTo);
+ waitingOnApply.forEach(builder, deps, waitingOnCommit, (b, d, s, i) -> {
+ if (!s.get(i))
+ b.add(d.txnId(i));
+ });
+ return builder.build();
+ }
+
+ private static String toString(Deps deps, SimpleBitSet waitingOnCommit, SimpleBitSet waitingOnApply)
+ {
+ return "onApply=" + computeWaitingOnApply(deps, waitingOnCommit, waitingOnApply).descendingSet() + ", onCommit=" + computeWaitingOnCommit(deps, waitingOnCommit).descendingSet();
+ }
+
+ @Override
+ public String toString()
+ {
+ return toString(deps, waitingOnCommit, waitingOnApply);
+ }
+
+ @Override
+ public boolean equals(Object other)
+ {
+ return other instanceof WaitingOn && this.equals((WaitingOn) other);
+ }
+
+ public boolean equals(WaitingOn other)
+ {
+ return this.deps.equals(other.deps)
+ && this.waitingOnCommit.equals(other.waitingOnCommit)
+ && this.waitingOnApply.equals(other.waitingOnApply)
+ && this.appliedOrInvalidated.equals(other.appliedOrInvalidated);
+ }
+
+ public static class Update
+ {
+ final Deps deps;
+ private SimpleBitSet waitingOnCommit, waitingOnApply, appliedOrInvalidated;
public Update(WaitingOn waitingOn)
{
+ this.deps = waitingOn.deps;
this.waitingOnCommit = waitingOn.waitingOnCommit;
this.waitingOnApply = waitingOn.waitingOnApply;
+ this.appliedOrInvalidated = waitingOn.appliedOrInvalidated;
}
public Update(Committed committed)
{
- this.waitingOnCommit = committed.waitingOnCommit();
- this.waitingOnApply = committed.waitingOnApply();
+ this(committed.waitingOn);
+ }
+
+ public Update(Ranges ranges, Unseekables<?> participants, Deps deps)
+ {
+ this.deps = deps;
+ this.waitingOnCommit = new SimpleBitSet(deps.txnIdCount(), false);
+ deps.keyDeps.forEach(ranges, 0, deps.keyDeps.txnIdCount(), this, null, (u, v, i) -> {
+ u.waitingOnCommit.set(i);
+ });
+ // we select range deps on actual participants rather than covered ranges,
+ // since we may otherwise adopt false dependencies for range txns
+ deps.rangeDeps.forEach(participants, this, (u, i) -> {
+ u.waitingOnCommit.set(u.deps.keyDeps.txnIdCount() + i);
+ });
+ this.waitingOnApply = new SimpleBitSet(deps.txnIdCount(), false);
+ this.appliedOrInvalidated = new SimpleBitSet(deps.txnIdCount(), false);
}
public boolean hasChanges()
{
- return hasChanges;
+ return !(waitingOnCommit instanceof ImmutableBitSet)
+ || !(waitingOnApply instanceof ImmutableBitSet)
+ || !(appliedOrInvalidated instanceof ImmutableBitSet);
+ }
+
+ public boolean removeWaitingOnCommit(TxnId txnId)
+ {
+ int index = deps.indexOf(txnId);
+ if (!waitingOnCommit.get(index))
+ return false;
+
+ waitingOnCommit = ensureMutable(waitingOnCommit);
+ waitingOnCommit.unset(index);
+ return true;
+ }
+
+ public boolean isWaitingOnApply(TxnId txnId)
+ {
+ int index = deps.indexOf(txnId);
+ return waitingOnApply.get(index);
+ }
+
+ public boolean isWaitingOn(TxnId txnId)
+ {
+ int index = deps.indexOf(txnId);
+ return waitingOnApply.get(index) || waitingOnCommit.get(index);
+ }
+
+ public boolean addWaitingOnApply(TxnId txnId)
+ {
+ int index = deps.indexOf(txnId);
+ if (waitingOnApply.get(index))
+ return false;
+
+ waitingOnApply = ensureMutable(waitingOnApply);
+ waitingOnApply.set(index);
+ return true;
+ }
+
+ public boolean removeWaitingOnApply(TxnId txnId)
+ {
+ int index = deps.indexOf(txnId);
+ if (!waitingOnApply.get(index))
+ return false;
+
+ waitingOnApply = ensureMutable(waitingOnApply);
+ waitingOnApply.unset(index);
+ return true;
+ }
+
+ public boolean removeWaitingOn(TxnId txnId)
+ {
+ return removeWaitingOnCommit(txnId) || removeWaitingOnApply(txnId);
}
- public void addWaitingOnCommit(TxnId txnId)
+ public boolean removeInvalidatedAppliedOrTruncated(TxnId txnId)
{
- waitingOnCommit = ensureSortedMutable(waitingOnCommit);
- waitingOnCommit.add(txnId);
- hasChanges = true;
+ int index = this.deps.indexOf(txnId);
+ return setAppliedOrInvalidated(index);
}
- public void removeWaitingOnCommit(TxnId txnId)
+ public boolean isEmpty()
{
- if (waitingOnApply == null)
- return;
- waitingOnCommit = ensureSortedMutable(waitingOnCommit);
- waitingOnCommit.remove(txnId);
- hasChanges = true;
+ return waitingOnApply.isEmpty() && waitingOnCommit.isEmpty();
}
- public void addWaitingOnApply(TxnId txnId, Timestamp executeAt)
+ public boolean removeApplied(TxnId txnId, WaitingOn propagate)
{
- waitingOnApply = ensureSortedMutable(waitingOnApply);
- waitingOnApply.put(executeAt, txnId);
- hasChanges = true;
+ int index = this.deps.indexOf(txnId);
+ if (!setAppliedOrInvalidated(index))
+ return false;
+
+ if (!propagate.appliedOrInvalidated.isEmpty())
+ {
+ forEachIntersection(propagate.deps.keyDeps.txnIds(), deps.keyDeps.txnIds(),
+ (from, to, ignore, i1, i2) -> {
+ if (from.get(i1))
+ to.setAppliedOrInvalidated(i2);
+ }, propagate.appliedOrInvalidated, this, null);
+
+ forEachIntersection(propagate.deps.rangeDeps.txnIds(), deps.rangeDeps.txnIds(),
+ (from, to, ignore, i1, i2) -> {
+ if (from.isAppliedOrInvalidatedRangeIdx(i1))
+ to.setAppliedOrInvalidatedRangeIdx(i2);
+ }, propagate, this, null);
+ }
+
+ return true;
}
- public void removeWaitingOnApply(TxnId txnId, Timestamp executeAt)
+ public TxnId minWaitingOnTxnId()
{
- if (waitingOnApply == null)
- return;
- waitingOnApply = ensureSortedMutable(waitingOnApply);
- waitingOnApply.remove(executeAt);
- hasChanges = true;
+ return WaitingOn.minWaitingOnTxnId(deps, waitingOnCommit, waitingOnApply);
}
- public void removeWaitingOn(TxnId txnId, Timestamp executeAt)
+ boolean setAppliedOrInvalidatedRangeIdx(int i)
{
- removeWaitingOnCommit(txnId);
- removeWaitingOnApply(txnId, executeAt);
- hasChanges = true;
+ return setAppliedOrInvalidated(i + deps.keyDeps.txnIdCount());
+ }
+
+ boolean setAppliedOrInvalidated(int i)
+ {
+ if (appliedOrInvalidated.get(i))
+ return false;
+
+ if (waitingOnCommit.get(i))
+ {
+ waitingOnCommit = ensureMutable(waitingOnCommit);
+ waitingOnCommit.unset(i);
+ }
+ else if (waitingOnApply.get(i))
+ {
+ waitingOnApply = ensureMutable(waitingOnApply);
+ waitingOnApply.unset(i);
+ }
+ else
+ {
+ return false;
+ }
+
+ appliedOrInvalidated = ensureMutable(appliedOrInvalidated);
+ appliedOrInvalidated.set(i);
+
+ return true;
+ }
+
+ public <P1, P2, P3, P4> void forEachWaitingOnCommit(P1 p1, P2 p2, P3 p3, P4 p4, IndexedQuadConsumer<P1, P2, P3, P4> forEach)
+ {
+ waitingOnCommit.reverseForEach(p1, p2, p3, p4, forEach);
+ }
+
+ public <P1, P2, P3, P4> void forEachWaitingOnApply(P1 p1, P2 p2, P3 p3, P4 p4, IndexedQuadConsumer<P1, P2, P3, P4> forEach)
+ {
+ waitingOnApply.reverseForEach(p1, p2, p3, p4, forEach);
}
public WaitingOn build()
{
- if ((waitingOnCommit == null || waitingOnCommit.isEmpty()) && (waitingOnApply == null || waitingOnApply.isEmpty()))
- return EMPTY;
- return new WaitingOn(ensureSortedImmutable(waitingOnCommit), ensureSortedImmutable(waitingOnApply));
+ return new WaitingOn(deps, ensureImmutable(waitingOnCommit), ensureImmutable(waitingOnApply), ensureImmutable(appliedOrInvalidated));
+ }
+
+ @Override
+ public String toString()
+ {
+ return WaitingOn.toString(deps, waitingOnCommit, waitingOnApply);
}
}
}
@@ -980,7 +1310,7 @@ public abstract class Command implements CommonAttributes
static Command.PreAccepted preaccept(Command command, CommonAttributes attrs, Timestamp executeAt, Ballot ballot)
{
- if (command.status() == Status.NotWitnessed)
+ if (command.status() == Status.NotDefined)
{
return Command.PreAccepted.preAccepted(attrs, executeAt, ballot);
}
@@ -1010,23 +1340,18 @@ public abstract class Command implements CommonAttributes
static Command.Accepted acceptInvalidated(Command command, Ballot ballot)
{
- Timestamp executeAt = command.isWitnessed() ? command.asWitnessed().executeAt() : null;
+ Timestamp executeAt = command.isDefined() ? command.asWitnessed().executeAt() : null;
return new Command.Accepted(command, SaveStatus.get(Status.AcceptedInvalidate, command.known()), executeAt, ballot, ballot);
}
static Command.Committed commit(Command command, CommonAttributes attrs, Timestamp executeAt, Command.WaitingOn waitingOn)
{
- return Command.Committed.committed(attrs, SaveStatus.get(Status.Committed, command.known()), executeAt, command.promised(), command.accepted(), waitingOn.waitingOnCommit, waitingOn.waitingOnApply);
+ return Command.Committed.committed(attrs, SaveStatus.get(Status.Committed, command.known()), executeAt, command.promised(), command.accepted(), waitingOn);
}
- static Command precommit(Command command, Timestamp executeAt)
+ static Command precommit(CommonAttributes attrs, Command command, Timestamp executeAt)
{
- return new Command.Accepted(command, SaveStatus.get(Status.PreCommitted, command.known()), executeAt, command.promised(), command.accepted());
- }
-
- static Command.Committed commitInvalidated(Command command, CommonAttributes attrs, Timestamp executeAt)
- {
- return Command.Executed.executed(attrs, SaveStatus.get(Status.Invalidated, command.known()), executeAt, command.promised(), command.accepted(), Command.WaitingOn.EMPTY, null, null);
+ return new Command.Accepted(attrs, SaveStatus.get(Status.PreCommitted, command.known()), executeAt, command.promised(), command.accepted());
}
static Command.Committed readyToExecute(Command.Committed command)
@@ -1039,8 +1364,13 @@ public abstract class Command implements CommonAttributes
return Command.Executed.executed(attrs, SaveStatus.get(Status.PreApplied, command.known()), executeAt, command.promised(), command.accepted(), waitingOn, writes, result);
}
+ static Command.Executed applying(Command.Executed command)
+ {
+ return Command.Executed.executed(command, command, SaveStatus.Applying);
+ }
+
static Command.Executed applied(Command.Executed command)
{
- return Command.Executed.executed(command, command, SaveStatus.get(Status.Applied, command.known()));
+ return Command.Executed.executed(command, command, SaveStatus.Applied);
}
}
diff --git a/accord-core/src/main/java/accord/local/CommandStore.java b/accord-core/src/main/java/accord/local/CommandStore.java
index becc7538..9a0f8468 100644
--- a/accord-core/src/main/java/accord/local/CommandStore.java
+++ b/accord-core/src/main/java/accord/local/CommandStore.java
@@ -18,48 +18,51 @@
package accord.local;
-import accord.api.ProgressLog;
-import accord.api.DataStore;
-import accord.coordinate.CollectDeps;
-import accord.local.CommandStores.RangesForEpochHolder;
-import accord.primitives.Timestamp;
-
-import javax.annotation.Nullable;
-import accord.api.Agent;
-
-import accord.utils.async.AsyncChain;
-
-import accord.api.ConfigurationService.EpochReady;
-import accord.primitives.*;
-import accord.utils.DeterministicIdentitySet;
-import accord.utils.IndexedRangeTriConsumer;
-import accord.utils.Invariants;
-import accord.utils.ReducingRangeMap;
-import accord.utils.SortedArrays.SortedArrayList;
-import accord.utils.async.AsyncResult;
-
import java.util.AbstractMap.SimpleImmutableEntry;
import java.util.Collections;
import java.util.Map;
import java.util.NavigableMap;
import java.util.Objects;
-
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.Callable;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.Supplier;
+import javax.annotation.Nullable;
+
+import com.google.common.collect.ImmutableSortedMap;
+import accord.api.Agent;
+import accord.api.ConfigurationService.EpochReady;
+import accord.api.DataStore;
+import accord.api.ProgressLog;
+import accord.coordinate.CollectDeps;
+import accord.local.Command.WaitingOn;
+import accord.local.CommandStores.RangesForEpoch;
+import accord.primitives.FullRoute;
+import accord.primitives.KeyDeps;
+import accord.primitives.Participants;
+import accord.primitives.RangeDeps;
+import accord.primitives.Ranges;
import accord.primitives.Seekables;
+import accord.primitives.Timestamp;
import accord.primitives.TxnId;
+import accord.primitives.Unseekables;
+import accord.utils.DeterministicIdentitySet;
+import accord.utils.Invariants;
+import accord.utils.ReducingRangeMap;
+import accord.utils.RelationMultiMap.SortedRelationList;
+import accord.utils.async.AsyncChain;
+import accord.utils.async.AsyncResult;
import accord.utils.async.AsyncResults;
-import com.google.common.collect.ImmutableSortedMap;
-
import static accord.api.ConfigurationService.EpochReady.DONE;
import static accord.local.PreLoadContext.contextFor;
import static accord.local.PreLoadContext.empty;
+import static accord.primitives.AbstractRanges.UnionMode.MERGE_ADJACENT;
+import static accord.primitives.Routables.Slice.Minimal;
import static accord.primitives.Txn.Kind.ExclusiveSyncPoint;
/**
@@ -67,6 +70,53 @@ import static accord.primitives.Txn.Kind.ExclusiveSyncPoint;
*/
public abstract class CommandStore implements AgentExecutor
{
+ static class EpochUpdate
+ {
+ final RangesForEpoch newRangesForEpoch;
+ final RedundantBefore addRedundantBefore;
+ final Ranges addGlobalRanges;
+
+ EpochUpdate(RangesForEpoch newRangesForEpoch, RedundantBefore addRedundantBefore, Ranges addGlobalRanges)
+ {
+ this.newRangesForEpoch = newRangesForEpoch;
+ this.addRedundantBefore = addRedundantBefore;
+ this.addGlobalRanges = addGlobalRanges;
+ }
+ }
+
+ public static class EpochUpdateHolder extends AtomicReference<EpochUpdate>
+ {
+ // TODO (required, eventually): support removing ranges
+ public void updateGlobal(Ranges addGlobalRanges)
+ {
+ EpochUpdate baseUpdate = new EpochUpdate(null, RedundantBefore.EMPTY, addGlobalRanges);
+ EpochUpdate cur = get();
+ if (cur == null || !compareAndSet(cur, new EpochUpdate(cur.newRangesForEpoch, cur.addRedundantBefore, cur.addGlobalRanges.with(addGlobalRanges))))
+ set(baseUpdate);
+ }
+
+ // TODO (desired): can better encapsulate by accepting only the newRangesForEpoch and deriving the add/remove ranges
+ public void add(long epoch, RangesForEpoch newRangesForEpoch, Ranges addRanges)
+ {
+ RedundantBefore addRedundantBefore = RedundantBefore.create(addRanges, epoch, Long.MAX_VALUE, TxnId.NONE, TxnId.minForEpoch(epoch));
+ update(newRangesForEpoch, addRedundantBefore);
+ }
+
+ public void remove(long epoch, RangesForEpoch newRangesForEpoch, Ranges removeRanges)
+ {
+ RedundantBefore addRedundantBefore = RedundantBefore.create(removeRanges, Long.MIN_VALUE, epoch, TxnId.NONE, TxnId.NONE);
+ update(newRangesForEpoch, addRedundantBefore);
+ }
+
+ private void update(RangesForEpoch newRangesForEpoch, RedundantBefore addRedundantBefore)
+ {
+ EpochUpdate baseUpdate = new EpochUpdate(newRangesForEpoch, addRedundantBefore, Ranges.EMPTY);
+ EpochUpdate cur = get();
+ if (cur == null || !compareAndSet(cur, new EpochUpdate(newRangesForEpoch, RedundantBefore.merge(cur.addRedundantBefore, addRedundantBefore), cur.addGlobalRanges)))
+ set(baseUpdate);
+ }
+ }
+
public interface Factory
{
CommandStore create(int id,
@@ -74,7 +124,7 @@ public abstract class CommandStore implements AgentExecutor
Agent agent,
DataStore store,
ProgressLog.Factory progressLogFactory,
- RangesForEpochHolder rangesForEpoch);
+ EpochUpdateHolder rangesForEpoch);
}
private static final ThreadLocal<CommandStore> CURRENT_STORE = new ThreadLocal<>();
@@ -84,24 +134,30 @@ public abstract class CommandStore implements AgentExecutor
protected final Agent agent;
protected final DataStore store;
protected final ProgressLog progressLog;
- protected final RangesForEpochHolder rangesForEpochHolder;
+ protected final EpochUpdateHolder epochUpdateHolder;
// TODO (expected): schedule regular pruning of these collections
- // TODO (required): by itself this doesn't handle re-bootstrapping of a range previously owned by this command store
- private NavigableMap<TxnId, Ranges> bootstrapBeganAt = ImmutableSortedMap.of(TxnId.NONE, Ranges.EMPTY);
+ // bootstrapBeganAt and shardDurableAt are both canonical data sets mostly used for debugging / constructing
+ private NavigableMap<TxnId, Ranges> bootstrapBeganAt = ImmutableSortedMap.of(TxnId.NONE, Ranges.EMPTY); // additive (i.e. once inserted, rolled-over until invalidated, and the floor entry contains additions)
+ private RedundantBefore redundantBefore = RedundantBefore.EMPTY;
+ // TODO (expected): store this only once per node
+ private DurableBefore durableBefore = DurableBefore.EMPTY;
+ protected RangesForEpoch rangesForEpoch;
+
+ // TODO (desired): merge with redundantBefore?
private NavigableMap<Timestamp, Ranges> safeToRead = ImmutableSortedMap.of(Timestamp.NONE, Ranges.EMPTY);
private final Set<Bootstrap> bootstraps = Collections.synchronizedSet(new DeterministicIdentitySet<>());
- private long maxBootstrapEpoch;
+ // TODO (required): we must synchronise this on joining, prior to marking ourselves ready to coordinate
@Nullable private ReducingRangeMap<Timestamp> rejectBefore;
- protected CommandStore(int id, NodeTimeService time, Agent agent, DataStore store, ProgressLog.Factory progressLogFactory, RangesForEpochHolder rangesForEpochHolder)
+ protected CommandStore(int id, NodeTimeService time, Agent agent, DataStore store, ProgressLog.Factory progressLogFactory, EpochUpdateHolder epochUpdateHolder)
{
this.id = id;
this.time = time;
this.agent = agent;
this.store = store;
this.progressLog = progressLogFactory.create(this);
- this.rangesForEpochHolder = rangesForEpochHolder;
+ this.epochUpdateHolder = epochUpdateHolder;
}
public final int id()
@@ -115,9 +171,25 @@ public abstract class CommandStore implements AgentExecutor
return agent;
}
- public RangesForEpochHolder rangesForEpochHolder()
+ public RangesForEpoch updateRangesForEpoch()
{
- return rangesForEpochHolder;
+ EpochUpdate update = epochUpdateHolder.get();
+ if (update == null)
+ return rangesForEpoch;
+
+ update = epochUpdateHolder.getAndSet(null);
+ if (!update.addGlobalRanges.isEmpty())
+ setDurableBefore(DurableBefore.merge(durableBefore, DurableBefore.create(update.addGlobalRanges, TxnId.NONE, TxnId.NONE)));
+ if (update.addRedundantBefore.size() > 0)
+ setRedundantBefore(RedundantBefore.merge(redundantBefore, update.addRedundantBefore));
+ if (update.newRangesForEpoch != null)
+ rangesForEpoch = update.newRangesForEpoch;
+ return rangesForEpoch;
+ }
+
+ public RangesForEpoch unsafeRangesForEpoch()
+ {
+ return rangesForEpoch;
}
public abstract boolean inStore();
@@ -125,7 +197,6 @@ public abstract class CommandStore implements AgentExecutor
public abstract AsyncChain<Void> execute(PreLoadContext context, Consumer<? super SafeCommandStore> consumer);
public abstract <T> AsyncChain<T> submit(PreLoadContext context, Function<? super SafeCommandStore, T> apply);
-
public abstract void shutdown();
private static Timestamp maxApplied(SafeCommandStore safeStore, Seekables<?, ?> keysOrRanges, Ranges slice)
@@ -161,6 +232,28 @@ public abstract class CommandStore implements AgentExecutor
this.bootstrapBeganAt = newBootstrapBeganAt;
}
+ public DurableBefore durableBefore()
+ {
+ return durableBefore;
+ }
+
+ /**
+ * To be overridden by implementations, to ensure the new state is persisted.
+ */
+ public void setDurableBefore(DurableBefore durableBefore)
+ {
+ this.durableBefore = durableBefore;
+ }
+
+
+ /**
+ * To be overridden by implementations, to ensure the new state is persisted.
+ */
+ protected void setRedundantBefore(RedundantBefore newRedundantBefore)
+ {
+ this.redundantBefore = newRedundantBefore;
+ }
+
/**
* This method may be invoked on a non-CommandStore thread
*/
@@ -179,24 +272,20 @@ public abstract class CommandStore implements AgentExecutor
return safeToRead;
}
- public long maxBootstrapEpoch()
- {
- return maxBootstrapEpoch;
- }
-
- public void markExclusiveSyncPoint(TxnId txnId, Ranges ranges, SafeCommandStore safeStore)
+ public final void markExclusiveSyncPoint(SafeCommandStore safeStore, TxnId txnId, Ranges ranges)
{
+ // TODO (desired): narrow ranges to those that are owned
Invariants.checkArgument(txnId.rw() == ExclusiveSyncPoint);
- ReducingRangeMap<Timestamp> newRejectBefore = rejectBefore != null ? rejectBefore : new ReducingRangeMap<>(Timestamp.NONE);
+ ReducingRangeMap<Timestamp> newRejectBefore = rejectBefore != null ? rejectBefore : new ReducingRangeMap<>();
newRejectBefore = ReducingRangeMap.add(newRejectBefore, ranges, txnId, Timestamp::max);
setRejectBefore(newRejectBefore);
}
- Timestamp preaccept(TxnId txnId, Seekables<?, ?> keys, SafeCommandStore safeStore)
+ final Timestamp preaccept(TxnId txnId, Seekables<?, ?> keys, SafeCommandStore safeStore, boolean permitFastPath)
{
NodeTimeService time = safeStore.time();
boolean isExpired = agent().isExpired(txnId, safeStore.time().now());
- if (rejectBefore != null && !isExpired && txnId.rw().isWrite())
+ if (rejectBefore != null && !isExpired)
isExpired = null == rejectBefore.foldl(keys, (rejectIfBefore, test) -> rejectIfBefore.compareTo(test) > 0 ? null : test, txnId, Objects::isNull);
if (isExpired)
@@ -204,12 +293,12 @@ public abstract class CommandStore implements AgentExecutor
if (txnId.rw() == ExclusiveSyncPoint)
{
- markExclusiveSyncPoint(txnId, (Ranges)keys, safeStore);
+ markExclusiveSyncPoint(safeStore, txnId, (Ranges)keys);
return txnId;
}
Timestamp maxConflict = safeStore.maxConflict(keys, safeStore.ranges().coordinates(txnId));
- if (txnId.compareTo(maxConflict) > 0 && txnId.epoch() >= time.epoch())
+ if (permitFastPath && txnId.compareTo(maxConflict) > 0 && txnId.epoch() >= time.epoch())
return txnId;
return time.uniqueNow(maxConflict);
@@ -293,7 +382,7 @@ public abstract class CommandStore implements AgentExecutor
* So, the outer future's success is sufficient for the topology to be acknowledged, and the inner future for the
* bootstrap to be complete.
*/
- Supplier<EpochReady> bootstrapper(Node node, Ranges newRanges, long epoch)
+ final Supplier<EpochReady> bootstrapper(Node node, Ranges newRanges, long epoch)
{
return () -> {
AsyncResult<EpochReady> metadata = submit(empty(), safeStore -> {
@@ -349,16 +438,13 @@ public abstract class CommandStore implements AgentExecutor
});
}
- Supplier<EpochReady> interruptBootstraps(long epoch, Ranges allRanges)
+ Supplier<EpochReady> unbootstrap(long epoch, Ranges removedRanges)
{
return () -> {
AsyncResult<Void> done = this.<Void>submit(empty(), safeStore -> {
- // TODO (required): to enable this we just need to ensure we have a chain of bootstrapped nodes
- // linking each epoch, else new owners may not finish before their ownership changes, so that if they
- // cancel their bootstrap and there's no chain bridging old to new the system gets stalled
for (Bootstrap prev : bootstraps)
{
- Ranges abort = prev.allValid.difference(allRanges);
+ Ranges abort = prev.allValid.subtract(removedRanges);
if (!abort.isEmpty())
prev.invalidate(abort);
}
@@ -369,25 +455,39 @@ public abstract class CommandStore implements AgentExecutor
};
}
- void complete(Bootstrap bootstrap)
+ final void complete(Bootstrap bootstrap)
{
bootstraps.remove(bootstrap);
}
- void markBootstrapping(SafeCommandStore safeStore, TxnId globalSyncId, Ranges ranges)
+ final void markBootstrapping(SafeCommandStore safeStore, TxnId globalSyncId, Ranges ranges)
{
setBootstrapBeganAt(bootstrap(globalSyncId, ranges, bootstrapBeganAt));
- maxBootstrapEpoch = Math.max(maxBootstrapEpoch, globalSyncId.epoch());
+ RedundantBefore addRedundantBefore = RedundantBefore.create(ranges, Long.MIN_VALUE, Long.MAX_VALUE, TxnId.NONE, globalSyncId);
+ setRedundantBefore(RedundantBefore.merge(redundantBefore, addRedundantBefore));
+ DurableBefore addDurableBefore = DurableBefore.create(ranges, TxnId.NONE, TxnId.NONE);
+ setDurableBefore(DurableBefore.merge(durableBefore, addDurableBefore));
+ }
+
+ // TODO (expected): we can immediately truncate dependencies locally once an exclusiveSyncPoint applies, we don't need to wait for the whole shard
+ public void markShardDurable(SafeCommandStore safeStore, TxnId globalSyncId, Ranges ranges)
+ {
+ // TODO (now): IMPORTANT we should insert additional entries for each epoch our ownership differs
+ ranges = ranges.slice(safeStore.ranges().allUntil(globalSyncId.epoch()), Minimal);
+ RedundantBefore addRedundantBefore = RedundantBefore.create(ranges, Long.MIN_VALUE, Long.MAX_VALUE, globalSyncId, TxnId.NONE);
+ setRedundantBefore(RedundantBefore.merge(redundantBefore, addRedundantBefore));
+ DurableBefore addDurableBefore = DurableBefore.create(ranges, globalSyncId, globalSyncId);
+ setDurableBefore(DurableBefore.merge(durableBefore, addDurableBefore));
}
- // TODO (required): load from disk and restore bootstraps
- EpochReady initialise(long epoch, Ranges ranges)
+ // MUST be invoked before CommandStore reference leaks to anyone
+ Supplier<EpochReady> initialise(long epoch, Ranges ranges)
{
- AsyncResult<Void> done = execute(empty(), safeStore -> {
- setBootstrapBeganAt(ImmutableSortedMap.of(TxnId.NONE, ranges));
- setSafeToRead(ImmutableSortedMap.of(Timestamp.NONE, ranges));
- }).beginAsResult();
- return new EpochReady(epoch, done, done, done, done);
+ DurableBefore addDurableBefore = DurableBefore.create(ranges, TxnId.NONE, TxnId.NONE);
+ setDurableBefore(DurableBefore.merge(durableBefore, addDurableBefore));
+ setBootstrapBeganAt(ImmutableSortedMap.of(TxnId.NONE, ranges));
+ setSafeToRead(ImmutableSortedMap.of(Timestamp.NONE, ranges));
+ return () -> new EpochReady(epoch, DONE, DONE, DONE, DONE);
}
final Ranges safeToReadAt(Timestamp at)
@@ -395,71 +495,78 @@ public abstract class CommandStore implements AgentExecutor
return safeToRead.floorEntry(at).getValue();
}
- /**
- * True iff one or more of these dependencies may need to be expunged due to occurring prior to a bootstrap event,
- * indicating that they may not be executed locally.
- */
- public final boolean isAffectedByBootstrap(Deps deps)
+ // TODO (desired): Commands.durability() can use this to upgrade to Majority without further info
+ public final Status.Durability globalDurability(TxnId txnId)
{
- return !deps.isEmpty() && isAffectedByBootstrap(deps.minTxnId());
+ return durableBefore.min(txnId);
}
- /**
- * True iff this transaction may itself not execute locally
- */
- public final boolean isAffectedByBootstrap(TxnId txnId)
+ public final RedundantBefore redundantBefore()
{
- // TODO (required): consider race conditions when bootstrapping into an active command store, that may have seen a higher txnId than this?
- // might benefit from maintaining a per-CommandStore largest TxnId register to ensure we allocate a higher TxnId for our ExclSync,
- // or from using whatever summary records we have for the range, once we maintain them
- return txnId.epoch() <= maxBootstrapEpoch && bootstrapBeganAt.higherEntry(txnId) != null;
+ return redundantBefore;
}
- /**
- * True iff this transaction's execution overlaps a range that is unsafe to read.
- * This permits us to ignore its dependencies for these keys, as we cannot locally execute an overlapping read,
- * so a write may execute at any time.
- */
- public final boolean isPartiallyInvisibleToReads(Timestamp executeAt, Seekables<?, ?> readScope)
+ public final boolean isRejectedIfNotPreAccepted(TxnId txnId, Unseekables<?> participants)
{
- Map.Entry<Timestamp, Ranges> entry = safeToRead.higherEntry(executeAt);
- if (entry == null)
+ if (rejectBefore == null)
return false;
- // TODO (desired): this is potentially pessimistic, depending on how we eventually manage loss of ownership of a range
- // i.e., if we remove from safeToRead when we lose ownership (or, if we remove due to a network partition causing inconsistency,
- // and then do not regain it)
- return !readScope.containsAll(entry.getValue());
+ return null != rejectBefore.foldl(participants, (rejectIfBefore, test) -> rejectIfBefore.compareTo(test) > 0 ? null : test, txnId, Objects::isNull);
}
- final <P1> void forEachBootstrapRange(SortedArrayList<TxnId> txnIds, IndexedRangeTriConsumer<P1, TxnId, Ranges> forEach, P1 p1)
+ public final void removeRedundantDependencies(Unseekables<?> participants, WaitingOn.Update builder)
{
- if (txnIds.isEmpty())
- return;
-
- int i = 0;
- while (i < txnIds.size())
- {
- Map.Entry<TxnId, Ranges> from = bootstrapBeganAt.floorEntry(txnIds.get(i));
- Map.Entry<TxnId, Ranges> to = bootstrapBeganAt.higherEntry(txnIds.get(i));
- int nexti = to == null ? txnIds.size() : txnIds.findNext(i, to.getKey());
- if (nexti < 0) nexti = -1 - nexti;
- if (from != null && !from.getValue().isEmpty())
+ // Note: we do not need to track the bootstraps we implicitly depend upon, because we will not serve any read requests until this has completed
+ // and since we are a timestamp store, and we write only this will sort itself out naturally
+ // TODO (required): make sure we have no races on HLC around SyncPoint else this resolution may not work (we need to know the micros equivalent timestamp of the snapshot)
+ KeyDeps keyDeps = builder.deps.keyDeps;
+ redundantBefore().foldl(keyDeps.keys(), (e, b, d, p2, ki, kj) -> {
+ int txnIdx = d.txnIds().find(e.locallyRedundantBefore());
+ if (txnIdx < 0) txnIdx = -1 - txnIdx;
+ while (ki < kj)
{
- TxnId bootstrapId = i == 0 && from.getKey() == TxnId.NONE ? null : from.getKey();
- forEach.accept(p1, bootstrapId, from.getValue(), i, nexti);
+ SortedRelationList<TxnId> txnIdsForKey = d.txnIdsForKeyIndex(ki++);
+ int ti = txnIdsForKey.findNext(0, txnIdx);
+ if (ti < 0)
+ ti = -1 - ti;
+ for (int i = 0 ; i < ti ; ++i)
+ b.setAppliedOrInvalidated(txnIdsForKey.getValueIndex(i));
}
- i = nexti;
- }
+ return b;
+ }, builder, keyDeps, null, ignore -> false);
+ RangeDeps rangeDeps = builder.deps.rangeDeps;
+ // TODO (now): slice to only those ranges we own, maybe don't even construct rangeDeps.covering()
+ redundantBefore().foldl(participants, (e, b, d, rs, pi, pj) -> {
+ // TODO (now): foldlInt so we can track the lower rangeidx bound and not revisit unnecessarily
+ boolean useBootstrap = e.isLocalRedundancyViaBootstrap();
+ int txnIdx;
+ {
+ int tmp = d.txnIds().find(useBootstrap ? e.bootstrappedAt : e.redundantBefore);
+ txnIdx = tmp < 0 ? -1 - tmp : tmp;
+ }
+ rangeDeps.forEach(participants, e.range, pi, pj, d, (d0, txnIdx0) -> {
+ if (txnIdx0 < txnIdx)
+ b.setAppliedOrInvalidatedRangeIdx(txnIdx0);
+ });
+ return b;
+ }, builder, rangeDeps, participants, ignore -> false);
+ }
+
+ public final boolean hasLocallyRedundantDependencies(TxnId minimumDependencyId, Timestamp executeAt, Participants<?> participantsOfWaitingTxn)
+ {
+ // TODO (required): consider race conditions when bootstrapping into an active command store, that may have seen a higher txnId than this?
+ // might benefit from maintaining a per-CommandStore largest TxnId register to ensure we allocate a higher TxnId for our ExclSync,
+ // or from using whatever summary records we have for the range, once we maintain them
+ return redundantBefore.status(minimumDependencyId, executeAt, participantsOfWaitingTxn).compareTo(RedundantStatus.PARTIALLY_PRE_BOOTSTRAP) >= 0;
}
- synchronized void markUnsafeToRead(Ranges ranges)
+ final synchronized void markUnsafeToRead(Ranges ranges)
{
if (safeToRead.values().stream().anyMatch(r -> r.intersects(ranges)))
setSafeToRead(purgeHistory(safeToRead, ranges));
}
- synchronized void markSafeToRead(Timestamp at, Ranges ranges)
+ final synchronized void markSafeToRead(Timestamp at, Ranges ranges)
{
setSafeToRead(purgeAndInsert(safeToRead, at, ranges));
}
@@ -475,8 +582,8 @@ public abstract class CommandStore implements AgentExecutor
private static <T extends Timestamp> ImmutableSortedMap<T, Ranges> purgeAndInsert(NavigableMap<T, Ranges> in, T insertAt, Ranges insert)
{
TreeMap<T, Ranges> build = new TreeMap<>(in);
- build.headMap(insertAt, false).entrySet().forEach(e -> e.setValue(e.getValue().difference(insert)));
- build.tailMap(insertAt, true).entrySet().forEach(e -> e.setValue(e.getValue().with(insert)));
+ build.headMap(insertAt, false).entrySet().forEach(e -> e.setValue(e.getValue().subtract(insert)));
+ build.tailMap(insertAt, true).entrySet().forEach(e -> e.setValue(e.getValue().union(MERGE_ADJACENT, insert)));
build.entrySet().removeIf(e -> e.getKey().compareTo(Timestamp.NONE) > 0 && e.getValue().isEmpty());
Map.Entry<T, Ranges> prev = build.floorEntry(insertAt);
build.putIfAbsent(insertAt, prev.getValue().with(insert));
@@ -498,7 +605,7 @@ public abstract class CommandStore implements AgentExecutor
private static <T extends Timestamp> Map.Entry<T, Ranges> without(Map.Entry<T, Ranges> in, Ranges remove)
{
- Ranges without = in.getValue().difference(remove);
+ Ranges without = in.getValue().subtract(remove);
if (without == in.getValue())
return in;
return new SimpleImmutableEntry<>(in.getKey(), without);
diff --git a/accord-core/src/main/java/accord/local/CommandStores.java b/accord-core/src/main/java/accord/local/CommandStores.java
index 2cb4eef1..dea108cc 100644
--- a/accord-core/src/main/java/accord/local/CommandStores.java
+++ b/accord-core/src/main/java/accord/local/CommandStores.java
@@ -18,22 +18,6 @@
package accord.local;
-import accord.api.*;
-import accord.api.ConfigurationService.EpochReady;
-import accord.primitives.*;
-import accord.api.RoutingKey;
-import accord.topology.Topology;
-import accord.utils.MapReduce;
-import accord.utils.MapReduceConsume;
-
-import com.google.common.annotations.VisibleForTesting;
-
-import accord.utils.RandomSource;
-import org.agrona.collections.Hashing;
-import org.agrona.collections.Int2ObjectHashMap;
-import accord.utils.async.AsyncChain;
-import accord.utils.async.AsyncChains;
-
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -45,7 +29,32 @@ import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
+import com.google.common.annotations.VisibleForTesting;
+
+import accord.api.Agent;
+import accord.api.ConfigurationService.EpochReady;
+import accord.api.DataStore;
+import accord.api.Key;
+import accord.api.ProgressLog;
+import accord.api.RoutingKey;
+import accord.local.CommandStore.EpochUpdateHolder;
+import accord.primitives.Range;
+import accord.primitives.Ranges;
+import accord.primitives.Routables;
+import accord.primitives.Route;
+import accord.primitives.RoutingKeys;
+import accord.primitives.Timestamp;
+import accord.primitives.TxnId;
+import accord.topology.Topology;
+import accord.utils.Invariants;
+import accord.utils.MapReduce;
+import accord.utils.MapReduceConsume;
+import accord.utils.RandomSource;
+import accord.utils.async.AsyncChain;
+import accord.utils.async.AsyncChains;
import javax.annotation.Nonnull;
+import org.agrona.collections.Hashing;
+import org.agrona.collections.Int2ObjectHashMap;
import static accord.api.ConfigurationService.EpochReady.done;
import static accord.local.PreLoadContext.empty;
@@ -87,47 +96,34 @@ public abstract class CommandStores
this.shardFactory = shardFactory;
}
- CommandStore create(int id, RangesForEpochHolder rangesForEpoch)
+ CommandStore create(int id, EpochUpdateHolder rangesForEpoch)
{
return shardFactory.create(id, time, agent, this.store, progressLogFactory, rangesForEpoch);
}
}
- public static class RangesForEpochHolder
- {
- // no need for safe publication; RangesForEpoch members are final, and will be guarded by other synchronization actions
- protected RangesForEpoch current;
-
- /**
- * This is updated asynchronously, so should only be fetched between executing tasks;
- * otherwise the contents may differ between invocations for the same task.
- * @return the current RangesForEpoch
- */
- public RangesForEpoch get() { return current; }
- }
-
static class ShardHolder
{
final CommandStore store;
- final RangesForEpochHolder ranges;
+ RangesForEpoch ranges;
- ShardHolder(CommandStore store, RangesForEpochHolder ranges)
+ ShardHolder(CommandStore store)
{
this.store = store;
- this.ranges = ranges;
}
RangesForEpoch ranges()
{
- return ranges.current;
+ return ranges;
}
public String toString()
{
- return store.id() + " " + ranges.current;
+ return store.id() + " " + ranges;
}
}
+ // TODO (now): split into deterministic and non-deterministic methods; ensure we only use deterministic methods during commands execution
public static class RangesForEpoch
{
final long[] epochs;
@@ -148,12 +144,15 @@ public abstract class CommandStores
this.store = store;
}
- public RangesForEpoch withRanges(long epoch, Ranges ranges)
+ public RangesForEpoch withRanges(long epoch, Ranges latestRanges)
{
- long[] newEpochs = Arrays.copyOf(this.epochs, this.epochs.length + 1);
- Ranges[] newRanges = Arrays.copyOf(this.ranges, this.ranges.length + 1);
- newEpochs[this.epochs.length] = epoch;
- newRanges[this.ranges.length] = ranges;
+ Invariants.checkArgument(epochs.length == 0 || epochs[epochs.length - 1] <= epoch);
+ int newLength = epochs.length == 0 || epochs[epochs.length - 1] < epoch ? epochs.length + 1 : epochs.length;
+ long[] newEpochs = Arrays.copyOf(epochs, newLength);
+ Ranges[] newRanges = Arrays.copyOf(ranges, newLength);
+ newEpochs[newLength - 1] = epoch;
+ newRanges[newLength - 1] = latestRanges;
+ Invariants.checkState(newEpochs[newLength - 1] == 0 || newEpochs[newLength - 1] == epoch, "Attempted to override historic epoch %d with %d", newEpochs[newLength - 1], epoch);
return new RangesForEpoch(newEpochs, newRanges, store);
}
@@ -169,7 +168,7 @@ public abstract class CommandStores
public @Nonnull Ranges unsafeToReadAt(Timestamp at)
{
- return allAt(at).difference(store.safeToReadAt(at));
+ return allAt(at).subtract(store.safeToReadAt(at));
}
public @Nonnull Ranges allAt(Timestamp at)
@@ -203,53 +202,43 @@ public abstract class CommandStores
if (fromInclusive == toInclusive)
return allAt(fromInclusive);
- int i = Arrays.binarySearch(epochs, fromInclusive);
- if (i < 0) i = -2 - i;
- if (i < 0) i = 0;
-
- int j = Arrays.binarySearch(epochs, toInclusive);
- if (j < 0) j = -2 - j;
- if (i > j) return Ranges.EMPTY;
-
- Ranges result = ranges[i++];
- while (i <= j)
- result = result.with(ranges[i++]);
-
- return result;
+ return allInternal(Math.max(0, floorIndex(fromInclusive)), 1 + floorIndex(toInclusive));
}
- public @Nonnull Ranges allSince(long fromInclusive)
+ public @Nonnull Ranges all()
{
- int i = Arrays.binarySearch(epochs, fromInclusive);
- if (i < 0) i = Math.max(0, -2 -i);
-
- Ranges result = ranges[i++];
- while (i < ranges.length)
- result = ranges[i++].with(result);
-
- return result;
+ return allInternal(0, ranges.length);
}
- public @Nonnull Ranges all()
+ public @Nonnull Ranges allBefore(long toExclusive)
{
- if (ranges.length == 0)
- return Ranges.EMPTY;
+ return allInternal(0, ceilIndex(toExclusive));
+ }
- Ranges result = ranges[0];
- for (int i = 1; i < ranges.length ; ++i)
- result = ranges[i].with(result);
+ public @Nonnull Ranges allUntil(long toInclusive)
+ {
+ return allInternal(0, 1 + floorIndex(toInclusive));
+ }
- return result;
+ private int floorIndex(long epoch)
+ {
+ int i = Arrays.binarySearch(epochs, epoch);
+ if (i < 0) i = -2 - i;
+ return i;
}
- public @Nonnull Ranges allBefore(long toExclusive)
+ private int ceilIndex(long epoch)
{
- int limit = Arrays.binarySearch(epochs, toExclusive);
- if (limit < 0) limit = Math.max(0, -1 -limit);
- if (limit == 0) return Ranges.EMPTY;
+ int i = Arrays.binarySearch(epochs, epoch);
+ if (i < 0) i = -1 - i;
+ return i;
+ }
- Ranges result = ranges[0];
- for (int i = 1 ; i < limit; ++i)
+ private @Nonnull Ranges allInternal(int startIndex, int endIndex)
+ {
+ if (startIndex >= endIndex) return Ranges.EMPTY;
+ Ranges result = ranges[startIndex];
+ for (int i = startIndex + 1 ; i < endIndex; ++i)
result = ranges[i].with(result);
return result;
@@ -258,7 +247,7 @@ public abstract class CommandStores
public @Nonnull Ranges applyRanges(Timestamp executeAt)
{
// Note: we COULD slice to only those ranges we haven't bootstrapped, but no harm redundantly writing
- return allSince(executeAt.epoch());
+ return allAt(executeAt.epoch());
}
public @Nonnull Ranges currentRanges()
@@ -340,9 +329,17 @@ public abstract class CommandStores
return new TopologyUpdate(prev, () -> done(epoch));
Topology newLocalTopology = newTopology.forNode(supplier.time.id()).trim();
- Ranges added = newLocalTopology.ranges().difference(prev.local.ranges());
- Ranges subtracted = prev.local.ranges().difference(newLocalTopology.ranges());
+ Ranges addedGlobal = newTopology.ranges().subtract(prev.global.ranges());
+ if (!addedGlobal.isEmpty())
+ {
+ for (ShardHolder shard : prev.shards)
+ {
+ shard.store.epochUpdateHolder.updateGlobal(addedGlobal);
+ }
+ }
+ Ranges added = newLocalTopology.ranges().subtract(prev.local.ranges());
+ Ranges subtracted = prev.local.ranges().subtract(newLocalTopology.ranges());
if (added.isEmpty() && subtracted.isEmpty())
{
Supplier<EpochReady> epochReady = () -> done(epoch);
@@ -357,11 +354,14 @@ public abstract class CommandStores
List<ShardHolder> result = new ArrayList<>(prev.shards.length + added.size());
for (ShardHolder shard : prev.shards)
{
- if (subtracted.intersects(shard.ranges().currentRanges()))
+ Ranges current = shard.ranges().currentRanges();
+ Ranges removeRanges = subtracted.slice(current, Minimal);
+ if (!removeRanges.isEmpty())
{
- RangesForEpoch newRanges = shard.ranges().withRanges(newTopology.epoch(), shard.ranges().currentRanges().difference(subtracted));
- shard.ranges.current = newRanges;
- bootstrapUpdates.add(shard.store.interruptBootstraps(epoch, newRanges.currentRanges()));
+ // TODO (required): This is updating the a non-volatile field in the previous Snapshot, why modify it at all, even with volatile the guaranteed visibility is weak even with mutual exclusion
+ shard.ranges = shard.ranges().withRanges(newTopology.epoch(), current.subtract(subtracted));
+ shard.store.epochUpdateHolder.remove(epoch, shard.ranges, removeRanges);
+ bootstrapUpdates.add(shard.store.unbootstrap(epoch, removeRanges));
}
// TODO (desired): only sync affected shards
Ranges ranges = shard.ranges().currentRanges();
@@ -374,18 +374,20 @@ public abstract class CommandStores
if (!added.isEmpty())
{
// TODO (required): shards must rebalance
- for (Ranges add : shardDistributor.split(added))
+ for (Ranges addRanges : shardDistributor.split(added))
{
- RangesForEpochHolder rangesHolder = new RangesForEpochHolder();
- ShardHolder shardHolder = new ShardHolder(supplier.create(nextId++, rangesHolder), rangesHolder);
- rangesHolder.current = new RangesForEpoch(epoch, add, shardHolder.store);
+ EpochUpdateHolder updateHolder = new EpochUpdateHolder();
+ ShardHolder shard = new ShardHolder(supplier.create(nextId++, updateHolder));
+ shard.ranges = new RangesForEpoch(epoch, addRanges, shard.store);
+ shard.store.epochUpdateHolder.add(epoch, shard.ranges, addRanges);
+ shard.store.epochUpdateHolder.updateGlobal(newTopology.ranges());
- Map<Boolean, Ranges> partitioned = add.partitioningBy(range -> shouldBootstrap(node, prev.global, newLocalTopology, range));
- if (partitioned.containsKey(true))
- bootstrapUpdates.add(shardHolder.store.bootstrapper(node, partitioned.get(true), newLocalTopology.epoch()));
+ Map<Boolean, Ranges> partitioned = addRanges.partitioningBy(range -> shouldBootstrap(node, prev.global, newLocalTopology, range));
if (partitioned.containsKey(false))
- bootstrapUpdates.add(() -> shardHolder.store.initialise(epoch, partitioned.get(false)));
- result.add(shardHolder);
+ bootstrapUpdates.add(shard.store.initialise(epoch, partitioned.get(false)));
+ if (partitioned.containsKey(true))
+ bootstrapUpdates.add(shard.store.bootstrapper(node, partitioned.get(true), newLocalTopology.epoch()));
+ result.add(shard);
}
}
@@ -423,12 +425,12 @@ public abstract class CommandStores
return forEach(context, RoutingKeys.of(key), minEpoch, maxEpoch, forEach, true);
}
- public AsyncChain<Void> forEach(PreLoadContext context, Routables<?, ?> keys, long minEpoch, long maxEpoch, Consumer<SafeCommandStore> forEach)
+ public AsyncChain<Void> forEach(PreLoadContext context, Routables<?> keys, long minEpoch, long maxEpoch, Consumer<SafeCommandStore> forEach)
{
return forEach(context, keys, minEpoch, maxEpoch, forEach, true);
}
- private AsyncChain<Void> forEach(PreLoadContext context, Routables<?, ?> keys, long minEpoch, long maxEpoch, Consumer<SafeCommandStore> forEach, boolean matchesMultiple)
+ private AsyncChain<Void> forEach(PreLoadContext context, Routables<?> keys, long minEpoch, long maxEpoch, Consumer<SafeCommandStore> forEach, boolean matchesMultiple)
{
return this.mapReduce(context, keys, minEpoch, maxEpoch, new MapReduce<SafeCommandStore, Void>()
{
@@ -467,7 +469,7 @@ public abstract class CommandStores
*
* Implementations are expected to invoke {@link #mapReduceConsume(PreLoadContext, Routables, long, long, MapReduceConsume)}
*/
- protected <O> void mapReduceConsume(PreLoadContext context, Routables<?, ?> keys, long minEpoch, long maxEpoch, MapReduceConsume<? super SafeCommandStore, O> mapReduceConsume)
+ protected <O> void mapReduceConsume(PreLoadContext context, Routables<?> keys, long minEpoch, long maxEpoch, MapReduceConsume<? super SafeCommandStore, O> mapReduceConsume)
{
AsyncChain<O> reduced = mapReduce(context, keys, minEpoch, maxEpoch, mapReduceConsume);
reduced.begin(mapReduceConsume);
@@ -479,7 +481,7 @@ public abstract class CommandStores
reduced.begin(mapReduceConsume);
}
- public <O> AsyncChain<O> mapReduce(PreLoadContext context, Routables<?, ?> keys, long minEpoch, long maxEpoch, MapReduce<? super SafeCommandStore, O> mapReduce)
+ public <O> AsyncChain<O> mapReduce(PreLoadContext context, Routables<?> keys, long minEpoch, long maxEpoch, MapReduce<? super SafeCommandStore, O> mapReduce)
{
AsyncChain<O> chain = null;
BiFunction<O, O, O> reducer = mapReduce::reduce;
@@ -495,9 +497,7 @@ public abstract class CommandStores
AsyncChain<O> next = shard.store.submit(context, mapReduce);
chain = chain != null ? AsyncChains.reduce(chain, next, reducer) : next;
}
- if (chain == null)
- return AsyncChains.success(null);
- return chain;
+ return chain == null ? AsyncChains.success(null) : chain;
}
protected <O> AsyncChain<O> mapReduce(PreLoadContext context, IntStream commandStoreIds, MapReduce<? super SafeCommandStore, O> mapReduce)
@@ -512,9 +512,27 @@ public abstract class CommandStores
AsyncChain<O> next = commandStore.submit(context, mapReduce);
chain = chain != null ? AsyncChains.reduce(chain, next, reducer) : next;
}
- if (chain == null)
- return AsyncChains.success(null);
- return chain;
+ return chain == null ? AsyncChains.success(null) : chain;
+ }
+
+ public <O> void mapReduceConsume(PreLoadContext context, MapReduceConsume<? super SafeCommandStore, O> mapReduceConsume)
+ {
+ AsyncChain<O> reduced = mapReduce(context, mapReduceConsume);
+ reduced.begin(mapReduceConsume);
+ }
+
+ protected <O> AsyncChain<O> mapReduce(PreLoadContext context, MapReduce<? super SafeCommandStore, O> mapReduce)
+ {
+ // TODO (low priority, efficiency): avoid using an array, or use a scratch buffer
+ AsyncChain<O> chain = null;
+ BiFunction<O, O, O> reducer = mapReduce::reduce;
+ for (ShardHolder shardHolder : current.shards)
+ {
+ CommandStore commandStore = shardHolder.store;
+ AsyncChain<O> next = commandStore.submit(context, mapReduce);
+ chain = chain != null ? AsyncChains.reduce(chain, next, reducer) : next;
+ }
+ return chain == null ? AsyncChains.success(null) : chain;
}
public synchronized Supplier<EpochReady> updateTopology(Node node, Topology newTopology, boolean startSync)
@@ -532,7 +550,7 @@ public abstract class CommandStores
public CommandStore select(RoutingKey key)
{
- return select(ranges -> ranges.contains(key));
+ return select(ranges -> ranges.contains(key));
}
public CommandStore select(Route<?> route)
@@ -582,6 +600,11 @@ public abstract class CommandStores
return current.shards.length;
}
+ public ShardDistributor shardDistributor()
+ {
+ return shardDistributor;
+ }
+
@VisibleForTesting
public CommandStore unsafeForKey(Key key)
{
diff --git a/accord-core/src/main/java/accord/local/Commands.java b/accord-core/src/main/java/accord/local/Commands.java
index 5ac578f4..604899bf 100644
--- a/accord-core/src/main/java/accord/local/Commands.java
+++ b/accord-core/src/main/java/accord/local/Commands.java
@@ -19,10 +19,9 @@
package accord.local;
import java.util.Arrays;
-import java.util.BitSet;
import java.util.Collection;
-import java.util.Map;
import java.util.function.Consumer;
+import javax.annotation.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -30,16 +29,19 @@ import org.slf4j.LoggerFactory;
import accord.api.ProgressLog.ProgressShard;
import accord.api.Result;
import accord.api.RoutingKey;
+import accord.coordinate.Infer;
+import accord.local.Command.ProxyListener;
import accord.local.Command.WaitingOn;
+import accord.local.SaveStatus.LocalExecution;
import accord.primitives.Ballot;
import accord.primitives.Deps;
-import accord.primitives.Keys;
+import accord.primitives.EpochSupplier;
+import accord.primitives.FullRoute;
import accord.primitives.PartialDeps;
import accord.primitives.PartialRoute;
import accord.primitives.PartialTxn;
-import accord.primitives.Range;
+import accord.primitives.Participants;
import accord.primitives.Ranges;
-import accord.primitives.RoutableKey;
import accord.primitives.Routables;
import accord.primitives.Route;
import accord.primitives.Seekables;
@@ -50,36 +52,46 @@ import accord.primitives.TxnId;
import accord.primitives.Unseekables;
import accord.primitives.Writes;
import accord.utils.Invariants;
-import accord.utils.SortedArrays;
import accord.utils.async.AsyncChain;
import accord.utils.async.AsyncChains;
-import net.nicoulaj.compilecommand.annotations.Inline;
-
-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
import static accord.api.ProgressLog.ProgressShard.Home;
import static accord.api.ProgressLog.ProgressShard.Local;
import static accord.api.ProgressLog.ProgressShard.No;
import static accord.api.ProgressLog.ProgressShard.UnmanagedHome;
import static accord.api.ProgressLog.ProgressShard.Unsure;
+import static accord.local.Command.Truncated.erased;
+import static accord.local.Command.Truncated.truncatedApply;
+import static accord.local.Command.Truncated.truncatedApplyWithOutcome;
+import static accord.local.Commands.Cleanup.ERASE;
+import static accord.local.Commands.Cleanup.NO;
+import static accord.local.Commands.Cleanup.TRUNCATE;
import static accord.local.Commands.EnsureAction.Add;
import static accord.local.Commands.EnsureAction.Check;
import static accord.local.Commands.EnsureAction.Ignore;
import static accord.local.Commands.EnsureAction.Set;
import static accord.local.Commands.EnsureAction.TrySet;
+import static accord.local.RedundantStatus.NOT_OWNED;
+import static accord.local.RedundantStatus.PRE_BOOTSTRAP;
+import static accord.local.SaveStatus.Applying;
+import static accord.local.SaveStatus.Erased;
+import static accord.local.SaveStatus.TruncatedApply;
+import static accord.local.SaveStatus.TruncatedApplyWithOutcome;
+import static accord.local.SaveStatus.Uninitialised;
import static accord.local.Status.Accepted;
import static accord.local.Status.AcceptedInvalidate;
import static accord.local.Status.Applied;
import static accord.local.Status.Committed;
import static accord.local.Status.Durability;
+import static accord.local.Status.Durability.Majority;
+import static accord.local.Status.Durability.Universal;
import static accord.local.Status.Invalidated;
-import static accord.local.Status.Known;
-import static accord.local.Status.Known.ExecuteAtOnly;
+import static accord.local.Status.NotDefined;
+import static accord.local.Status.PreAccepted;
import static accord.local.Status.PreApplied;
import static accord.local.Status.PreCommitted;
import static accord.local.Status.ReadyToExecute;
-import static accord.primitives.Routables.Slice.Minimal;
+import static accord.local.Status.Truncated;
import static accord.primitives.Route.isFullRoute;
public class Commands
@@ -113,29 +125,29 @@ public class Commands
return safeStore.ranges().allAt(epoch).contains(someKey);
}
- public static RoutingKey noProgressKey()
- {
- return NO_PROGRESS_KEY;
- }
-
- public enum AcceptOutcome {Success, Redundant, RejectedBallot}
+ public enum AcceptOutcome { Success, Redundant, RejectedBallot, Truncated }
- public static AcceptOutcome preaccept(SafeCommandStore safeStore, TxnId txnId, long acceptEpoch, PartialTxn partialTxn, Route<?> route, @Nullable RoutingKey progressKey)
+ public static AcceptOutcome preaccept(SafeCommandStore safeStore, SafeCommand safeCommand, TxnId txnId, long acceptEpoch, PartialTxn partialTxn, FullRoute<?> route, @Nullable RoutingKey progressKey)
{
- return preacceptOrRecover(safeStore, txnId, acceptEpoch, partialTxn, route, progressKey, Ballot.ZERO);
+ return preacceptOrRecover(safeStore, safeCommand, txnId, acceptEpoch, partialTxn, route, progressKey, Ballot.ZERO);
}
- public static AcceptOutcome recover(SafeCommandStore safeStore, TxnId txnId, PartialTxn partialTxn, Route<?> route, @Nullable RoutingKey progressKey, Ballot ballot)
+ public static AcceptOutcome recover(SafeCommandStore safeStore, SafeCommand safeCommand, TxnId txnId, PartialTxn partialTxn, FullRoute<?> route, @Nullable RoutingKey progressKey, Ballot ballot)
{
// for recovery we only ever propose either the original epoch or an Accept that we witness; otherwise we invalidate
- return preacceptOrRecover(safeStore, txnId, txnId.epoch(), partialTxn, route, progressKey, ballot);
+ return preacceptOrRecover(safeStore, safeCommand, txnId, txnId.epoch(), partialTxn, route, progressKey, ballot);
}
- private static AcceptOutcome preacceptOrRecover(SafeCommandStore safeStore, TxnId txnId, long acceptEpoch, PartialTxn partialTxn, Route<?> route, @Nullable RoutingKey progressKey, Ballot ballot)
+ private static AcceptOutcome preacceptOrRecover(SafeCommandStore safeStore, SafeCommand safeCommand, TxnId txnId, long acceptEpoch, PartialTxn partialTxn, FullRoute<?> route, @Nullable RoutingKey progressKey, Ballot ballot)
{
- SafeCommand safeCommand = safeStore.command(txnId);
Command command = safeCommand.current();
+ if (command.hasBeen(Truncated))
+ {
+ logger.trace("{}: skipping preaccept - command is truncated", txnId);
+ return command.is(Invalidated) ? AcceptOutcome.Redundant : AcceptOutcome.Truncated;
+ }
+
int compareBallots = command.promised().compareTo(ballot);
if (compareBallots > 0)
{
@@ -155,12 +167,11 @@ public class Commands
Ranges coordinateRanges = coordinateRanges(safeStore, txnId, acceptEpoch);
Invariants.checkState(!coordinateRanges.isEmpty());
- CommonAttributes attrs = updateHomeAndProgressKeys(safeStore, command.txnId(), command, route, progressKey, coordinateRanges);
- ProgressShard shard = progressShard(attrs, progressKey, coordinateRanges);
- Invariants.checkState(validate(command.status(), attrs, Ranges.EMPTY, coordinateRanges, shard, route, Set, partialTxn, Set, null, Ignore));
+ ProgressShard shard = progressShard(route, progressKey, coordinateRanges);
+ Invariants.checkState(validate(command.status(), command, Ranges.EMPTY, coordinateRanges, shard, route, Set, partialTxn, Set, null, Ignore));
// FIXME: this should go into a consumer method
- attrs = set(safeStore, command, attrs, Ranges.EMPTY, coordinateRanges, shard, route, partialTxn, Set, null, Ignore);
+ CommonAttributes attrs = set(safeStore, command, command, Ranges.EMPTY, coordinateRanges, shard, route, partialTxn, Set, null, Ignore);
if (command.executeAt() == null)
{
// unlike in the Accord paper, we partition shards within a node, so that to ensure a total order we must either:
@@ -168,9 +179,8 @@ public class Commands
// - assign each shard _and_ process a unique id, and use both as components of the timestamp
// if we are performing recovery (i.e. non-zero ballot), do not permit a fast path decision as we want to
// invalidate any transactions that were not completed by their initial coordinator
- Timestamp executeAt = ballot.equals(Ballot.ZERO)
- ? safeStore.commandStore().preaccept(txnId, partialTxn.keys(), safeStore)
- : safeStore.time().uniqueNow(txnId);
+ // TODO (desired): limit preaccept to keys we include, to avoid inflating unnecessary state
+ Timestamp executeAt = safeStore.commandStore().preaccept(txnId, partialTxn.keys(), safeStore, ballot.equals(Ballot.ZERO));
command = safeCommand.preaccept(attrs, executeAt, ballot);
safeStore.progressLog().preaccepted(command, shard);
}
@@ -184,10 +194,18 @@ public class Commands
return AcceptOutcome.Success;
}
- public static boolean preacceptInvalidate(SafeCommandStore safeStore, TxnId txnId, Ballot ballot)
+ public static boolean preacceptInvalidate(SafeCommand safeCommand, Ballot ballot)
{
- SafeCommand safeCommand = safeStore.command(txnId);
Command command = safeCommand.current();
+
+ if (command.hasBeen(Status.Committed))
+ {
+ if (command.is(Truncated)) logger.trace("{}: skipping preacceptInvalidate - already truncated", command.txnId());
+ else if (command.is(Invalidated)) logger.trace("{}: skipping preacceptInvalidate - already invalidated", command.txnId());
+ else logger.trace("{}: skipping preacceptInvalidate - already committed", command.txnId());
+ return false;
+ }
+
if (command.promised().compareTo(ballot) > 0)
{
logger.trace("{}: skipping preacceptInvalidate - witnessed higher ballot ({})", command.txnId(), command.promised());
@@ -199,32 +217,43 @@ public class Commands
public static AcceptOutcome accept(SafeCommandStore safeStore, TxnId txnId, Ballot ballot, PartialRoute<?> route, Seekables<?, ?> keys, @Nullable RoutingKey progressKey, Timestamp executeAt, PartialDeps partialDeps)
{
- SafeCommand safeCommand = safeStore.command(txnId);
+ SafeCommand safeCommand = safeStore.get(txnId, executeAt, route);
Command command = safeCommand.current();
- if (command.promised().compareTo(ballot) > 0)
- {
- logger.trace("{}: skipping accept - witnessed higher ballot ({} > {})", txnId, command.promised(), ballot);
- return AcceptOutcome.RejectedBallot;
- }
-
if (command.hasBeen(PreCommitted))
{
+ if (command.is(Invalidated))
+ {
+ logger.trace("{}: skipping accept - command is invalidated", txnId);
+ return AcceptOutcome.Redundant;
+ }
+
+ if (command.is(Truncated))
+ {
+ logger.trace("{}: skipping accept - command is truncated", txnId);
+ return AcceptOutcome.Truncated;
+ }
+
logger.trace("{}: skipping accept - already committed ({})", txnId, command.status());
return AcceptOutcome.Redundant;
}
+ if (command.promised().compareTo(ballot) > 0)
+ {
+ logger.trace("{}: skipping accept - witnessed higher ballot ({} > {})", txnId, command.promised(), ballot);
+ return AcceptOutcome.RejectedBallot;
+ }
+
Ranges coordinateRanges = coordinateRanges(safeStore, txnId);
Ranges acceptRanges = acceptRanges(safeStore, txnId, executeAt, coordinateRanges);
Invariants.checkState(!acceptRanges.isEmpty());
- CommonAttributes attrs = updateHomeAndProgressKeys(safeStore, command.txnId(), command, route, progressKey, coordinateRanges);
- ProgressShard shard = progressShard(attrs, progressKey, coordinateRanges);
- Invariants.checkState(validate(command.status(), attrs, coordinateRanges, acceptRanges, shard, route, Ignore, null, Ignore, partialDeps, Set));
+ ProgressShard shard = progressShard(route, progressKey, coordinateRanges);
+ Invariants.checkState(validate(command.status(), command, coordinateRanges, acceptRanges, shard, route, Ignore, null, Ignore, partialDeps, Set));
// TODO (desired, clarity/efficiency): we don't need to set the route here, and perhaps we don't even need to
// distributed partialDeps at all, since all we gain is not waiting for these transactions to commit during
// recovery. We probably don't want to directly persist a Route in any other circumstances, either, to ease persistence.
- attrs = set(safeStore, command, attrs, coordinateRanges, acceptRanges, shard, route, null, Ignore, partialDeps, Set);
+ CommonAttributes attrs = set(safeStore, command, command, coordinateRanges, acceptRanges, shard, route, null, Ignore, partialDeps, Set);
// set only registers by transaction keys, which we mightn't already have received
if (!command.known().isDefinitionKnown())
@@ -239,19 +268,32 @@ public class Commands
public static AcceptOutcome acceptInvalidate(SafeCommandStore safeStore, SafeCommand safeCommand, Ballot ballot)
{
+ // TODO (expected): save some partial route we can use to determine if it can be GC'd
Command command = safeCommand.current();
- if (command.promised().compareTo(ballot) > 0)
- {
- logger.trace("{}: skipping accept invalidated - witnessed higher ballot ({} > {})", command.txnId(), command.promised(), ballot);
- return AcceptOutcome.RejectedBallot;
- }
-
if (command.hasBeen(PreCommitted))
{
+ if (command.is(Invalidated))
+ {
+ logger.trace("{}: skipping accept invalidated - already invalidated ({})", command.txnId(), command.status());
+ return AcceptOutcome.Redundant;
+ }
+
+ if (command.is(Truncated))
+ {
+ logger.trace("{}: skipping accept invalidated - already truncated ({})", command.txnId(), command.status());
+ return AcceptOutcome.Truncated;
+ }
+
logger.trace("{}: skipping accept invalidated - already committed ({})", command.txnId(), command.status());
return AcceptOutcome.Redundant;
}
+ if (command.promised().compareTo(ballot) > 0)
+ {
+ logger.trace("{}: skipping accept invalidated - witnessed higher ballot ({} > {})", command.txnId(), command.promised(), ballot);
+ return AcceptOutcome.RejectedBallot;
+ }
+
logger.trace("{}: accepted invalidated", command.txnId());
safeCommand.acceptInvalidated(ballot);
@@ -259,20 +301,26 @@ public class Commands
return AcceptOutcome.Success;
}
- public enum CommitOutcome {Success, Redundant, Insufficient;}
+ public enum CommitOutcome { Success, Redundant, Insufficient }
// relies on mutual exclusion for each key
- public static CommitOutcome commit(SafeCommandStore safeStore, TxnId txnId, Route<?> route, @Nullable RoutingKey progressKey, @Nullable PartialTxn partialTxn, Timestamp executeAt, PartialDeps partialDeps)
+ public static CommitOutcome commit(SafeCommandStore safeStore, SafeCommand safeCommand, TxnId txnId, Route<?> route, @Nullable RoutingKey progressKey, @Nullable PartialTxn partialTxn, Timestamp executeAt, PartialDeps partialDeps)
{
- SafeCommand safeCommand = safeStore.command(txnId);
Command command = safeCommand.current();
if (command.hasBeen(PreCommitted))
{
- logger.trace("{}: skipping commit - already committed ({})", txnId, command.status());
- if (!executeAt.equals(command.executeAt()) || command.status() == Invalidated)
- safeStore.agent().onInconsistentTimestamp(command, (command.status() == Invalidated ? Timestamp.NONE : command.executeAt()), executeAt);
+ if (command.is(Truncated))
+ {
+ logger.trace("{}: skipping commit - already truncated ({})", txnId, command.status());
+ }
+ else
+ {
+ logger.trace("{}: skipping commit - already committed ({})", txnId, command.status());
+ if (!executeAt.equals(command.executeAt()) || command.status() == Invalidated)
+ safeStore.agent().onInconsistentTimestamp(command, (command.status() == Invalidated ? Timestamp.NONE : command.executeAt()), executeAt);
+ }
if (command.hasBeen(Committed))
return CommitOutcome.Redundant;
@@ -283,80 +331,89 @@ public class Commands
// TODO (expected, consider): consider ranges between coordinateRanges and executeRanges? Perhaps don't need them
Ranges executeRanges = executeRanges(safeStore, executeAt);
- CommonAttributes attrs = updateHomeAndProgressKeys(safeStore, command.txnId(), command, route, progressKey, coordinateRanges);
- ProgressShard shard = progressShard(attrs, progressKey, coordinateRanges);
+ ProgressShard shard = progressShard(route, progressKey, coordinateRanges);
- if (!validate(command.status(), attrs, acceptRanges, executeRanges, shard, route, Check, partialTxn, Add, partialDeps, Set))
- {
- safeCommand.updateAttributes(attrs);
+ if (!validate(command.status(), command, acceptRanges, executeRanges, shard, route, Check, partialTxn, Add, partialDeps, Set))
return CommitOutcome.Insufficient;
- }
// FIXME: split up set
- attrs = set(safeStore, command, attrs, acceptRanges, executeRanges, shard, route, partialTxn, Add, partialDeps, Set);
+ CommonAttributes attrs = set(safeStore, command, command, acceptRanges, executeRanges, shard, route, partialTxn, Add, partialDeps, Set);
logger.trace("{}: committed with executeAt: {}, deps: {}", txnId, executeAt, partialDeps);
- WaitingOn waitingOn = populateWaitingOn(safeStore, txnId, executeAt, partialDeps);
+ WaitingOn waitingOn = initialiseWaitingOn(safeStore, txnId, executeAt, attrs.partialDeps(), attrs.route());
command = safeCommand.commit(attrs, executeAt, waitingOn);
-
safeStore.progressLog().committed(command, shard);
// TODO (expected, safety): introduce intermediate status to avoid reentry when notifying listeners (which might notify us)
- maybeExecute(safeStore, safeCommand, shard, true, true);
+ maybeExecute(safeStore, safeCommand, true, true);
return CommitOutcome.Success;
}
// relies on mutual exclusion for each key
- public static void precommit(SafeCommandStore safeStore, TxnId txnId, Timestamp executeAt)
+ public static CommitOutcome precommit(SafeCommandStore safeStore, SafeCommand safeCommand, TxnId txnId, Timestamp executeAt, Route<?> route)
{
- SafeCommand safeCommand = safeStore.command(txnId);
Command command = safeCommand.current();
if (command.hasBeen(PreCommitted))
{
- logger.trace("{}: skipping precommit - already committed ({})", txnId, command.status());
- if (executeAt.equals(command.executeAt()) && command.status() != Invalidated)
- return;
+ if (command.is(Truncated))
+ {
+ logger.trace("{}: skipping commit - already truncated ({})", txnId, command.status());
+ return CommitOutcome.Redundant;
+ }
+ else
+ {
+ logger.trace("{}: skipping precommit - already committed ({})", txnId, command.status());
+ if (executeAt.equals(command.executeAt()) && command.status() != Invalidated)
+ return CommitOutcome.Redundant;
+
+ safeStore.agent().onInconsistentTimestamp(command, (command.status() == Invalidated ? Timestamp.NONE : command.executeAt()), executeAt);
+ }
+ }
- safeStore.agent().onInconsistentTimestamp(command, (command.status() == Invalidated ? Timestamp.NONE : command.executeAt()), executeAt);
+ CommonAttributes attrs = command;
+ if (command.route() == null || !command.route().kind().isFullRoute())
+ {
+ if (!route.kind().isFullRoute())
+ return CommitOutcome.Insufficient;
+ attrs = updateRoute(command, route);
}
- safeCommand.precommit(executeAt);
+ safeCommand.precommit(attrs, executeAt);
safeStore.notifyListeners(safeCommand);
logger.trace("{}: precommitted with executeAt: {}", txnId, executeAt);
+ return CommitOutcome.Success;
}
public static void commitRecipientLocalSyncPoint(SafeCommandStore safeStore, TxnId localSyncId, SyncPoint syncPoint, Seekables<?, ?> keys)
{
- SafeCommand safeCommand = safeStore.command(localSyncId);
+ SafeCommand safeCommand = safeStore.get(localSyncId);
Command command = safeCommand.current();
Invariants.checkState(!command.hasBeen(Committed));
- commitRecipientLocalSyncPoint(safeStore, localSyncId, keys, syncPoint.route);
+ commitRecipientLocalSyncPoint(safeStore, localSyncId, keys, syncPoint.route());
}
private static void commitRecipientLocalSyncPoint(SafeCommandStore safeStore, TxnId localSyncId, Seekables<?, ?> keys, Route<?> route)
{
- SafeCommand safeCommand = safeStore.command(localSyncId);
+ SafeCommand safeCommand = safeStore.get(localSyncId);
Command command = safeCommand.current();
if (command.hasBeen(Committed))
return;
Ranges coordinateRanges = coordinateRanges(safeStore, localSyncId);
// TODO (desired, consider): in the case of sync points, the coordinator is unlikely to be a home shard, do we mind this? should document at least
- ProgressShard progressShard = coordinateRanges.contains(route.homeKey()) ? UnmanagedHome : No;
Txn emptyTxn = safeStore.agent().emptyTxn(localSyncId.rw(), keys);
- CommonAttributes newAttributes = new CommonAttributes.Mutable(command)
- .progressKey(progressShard == UnmanagedHome ? route.homeKey() : NO_PROGRESS_KEY);
+ ProgressShard progressShard = coordinateRanges.contains(route.homeKey()) ? UnmanagedHome : No;
PartialDeps none = Deps.NONE.slice(coordinateRanges);
PartialTxn partialTxn = emptyTxn.slice(coordinateRanges, true);
- Invariants.checkState(validate(command.status(), newAttributes, Ranges.EMPTY, coordinateRanges, progressShard, route, Set, partialTxn, Set, none, Set));
- newAttributes = set(safeStore, command, newAttributes, Ranges.EMPTY, coordinateRanges, progressShard, route, partialTxn, Set, none, Set);
+ Invariants.checkState(validate(command.status(), command, Ranges.EMPTY, coordinateRanges, progressShard, route, Set, partialTxn, Set, none, Set));
+ CommonAttributes newAttributes = set(safeStore, command, command, Ranges.EMPTY, coordinateRanges, progressShard, route, partialTxn, Set, none, Set);
safeCommand.commit(newAttributes, localSyncId, WaitingOn.EMPTY);
safeStore.notifyListeners(safeCommand);
}
public static void applyRecipientLocalSyncPoint(SafeCommandStore safeStore, TxnId localSyncId, Seekables<?, ?> keys)
{
- SafeCommand safeCommand = safeStore.command(localSyncId);
+ SafeCommand safeCommand = safeStore.get(localSyncId);
Command.Committed command = safeCommand.current().asCommitted();
if (command.hasBeen(PreApplied))
return;
@@ -364,40 +421,40 @@ public class Commands
// NOTE: if this is ever made a non-empty txn this will introduce a potential bug where the txn is registered against CommandsForKeys
Txn emptyTxn = safeStore.agent().emptyTxn(localSyncId.rw(), keys);
safeCommand.preapplied(command, command.executeAt(), command.waitingOn(), emptyTxn.execute(localSyncId, localSyncId, null), emptyTxn.result(localSyncId, localSyncId, null));
- maybeExecute(safeStore, safeCommand, Unsure, true, false);
+ maybeExecute(safeStore, safeCommand, true, false);
}
// TODO (expected, ?): commitInvalidate may need to update cfks _if_ possible
- public static void commitInvalidate(SafeCommandStore safeStore, TxnId txnId)
+ public static void commitInvalidate(SafeCommandStore safeStore, SafeCommand safeCommand, Unseekables<?> scope)
{
- SafeCommand safeCommand = safeStore.command(txnId);
Command command = safeCommand.current();
if (command.hasBeen(PreCommitted))
{
- logger.trace("{}: skipping commit invalidated - already committed ({})", txnId, command.status());
- if (!command.hasBeen(Invalidated))
- safeStore.agent().onInconsistentTimestamp(command, Timestamp.NONE, command.executeAt());
-
+ if (command.is(Truncated))
+ {
+ logger.trace("{}: skipping commit invalidated - already truncated ({})", safeCommand.txnId(), command.status());
+ }
+ else
+ {
+ logger.trace("{}: skipping commit invalidated - already committed ({})", safeCommand.txnId(), command.status());
+ if (!command.is(Invalidated) && !(command.is(Truncated) && command.executeAt().equals(Timestamp.NONE)))
+ safeStore.agent().onInconsistentTimestamp(command, Timestamp.NONE, command.executeAt());
+ }
return;
}
+ else if (command.saveStatus().isUninitialised() && !safeStore.ranges().allAt(command.txnId().epoch()).intersects(scope))
+ return; // don't bother propagating the invalidation to future epochs where the replica didn't already witness the command
- ProgressShard shard = progressShard(safeStore, command);
- safeStore.progressLog().invalidated(command, shard);
-
- CommonAttributes attrs = command;
- if (command.partialDeps() == null)
- attrs = attrs.mutable().partialDeps(PartialDeps.NONE);
- safeCommand.commitInvalidated(attrs, txnId);
- logger.trace("{}: committed invalidated", txnId);
-
- safeStore.notifyListeners(safeCommand);
+ safeCommand.commitInvalidated();
+ safeStore.progressLog().clear(command.txnId());
+ logger.trace("{}: committed invalidated", safeCommand.txnId());
+ safeStore.notifyListeners(safeCommand, command, command.durableListeners(), safeCommand.transientListeners());
}
public enum ApplyOutcome {Success, Redundant, Insufficient}
- public static ApplyOutcome apply(SafeCommandStore safeStore, TxnId txnId, long untilEpoch, Route<?> route, Timestamp executeAt, @Nullable PartialDeps partialDeps, Writes writes, Result result)
+ public static ApplyOutcome apply(SafeCommandStore safeStore, SafeCommand safeCommand, TxnId txnId, Route<?> route, @Nullable RoutingKey progressKey, Timestamp executeAt, @Nullable PartialDeps partialDeps, @Nullable PartialTxn partialTxn, Writes writes, Result result)
{
- SafeCommand safeCommand = safeStore.command(txnId);
Command command = safeCommand.current();
if (command.hasBeen(PreApplied) && executeAt.equals(command.executeAt()))
{
@@ -406,35 +463,28 @@ public class Commands
}
else if (command.hasBeen(PreCommitted) && !executeAt.equals(command.executeAt()))
{
+ if (command.is(Truncated) && command.executeAt() == null)
+ return ApplyOutcome.Redundant;
safeStore.agent().onInconsistentTimestamp(command, command.executeAt(), executeAt);
}
Ranges coordinateRanges = coordinateRanges(safeStore, txnId);
Ranges acceptRanges = acceptRanges(safeStore, txnId, executeAt, coordinateRanges);
Ranges executeRanges = executeRanges(safeStore, executeAt);
- if (untilEpoch < safeStore.latestEpoch())
- {
- Ranges expectedRanges = safeStore.ranges().allBetween(executeAt.epoch(), untilEpoch);
- Invariants.checkState(expectedRanges.containsAll(executeRanges));
- }
- CommonAttributes attrs = updateHomeAndProgressKeys(safeStore, command.txnId(), command, route, coordinateRanges);
- ProgressShard shard = progressShard(attrs, coordinateRanges);
+ ProgressShard shard = progressShard(route, progressKey, coordinateRanges);
- if (!validate(command.status(), attrs, acceptRanges, executeRanges, shard, route, Check, null, Check, partialDeps, command.hasBeen(Committed) ? Add : TrySet))
- {
- safeCommand.updateAttributes(attrs);
+ if (!validate(command.status(), command, acceptRanges, executeRanges, shard, route, Check, partialTxn, Add, partialDeps, command.hasBeen(Committed) ? Check : TrySet))
return ApplyOutcome.Insufficient; // TODO (expected, consider): this should probably be an assertion failure if !TrySet
- }
- WaitingOn waitingOn = !command.hasBeen(Committed) ? populateWaitingOn(safeStore, txnId, executeAt, partialDeps) : command.asCommitted().waitingOn();
- attrs = set(safeStore, command, attrs, acceptRanges, executeRanges, shard, route, null, Check, partialDeps, command.hasBeen(Committed) ? Add : TrySet);
+ CommonAttributes attrs = set(safeStore, command, command, acceptRanges, executeRanges, shard, route, partialTxn, Add, partialDeps, command.hasBeen(Committed) ? Check : TrySet);
+ WaitingOn waitingOn = !command.hasBeen(Committed) ? initialiseWaitingOn(safeStore, txnId, executeAt, attrs.partialDeps(), attrs.route()) : command.asCommitted().waitingOn();
safeCommand.preapplied(attrs, executeAt, waitingOn, writes, result);
safeStore.notifyListeners(safeCommand);
logger.trace("{}: apply, status set to Executed with executeAt: {}, deps: {}", txnId, executeAt, partialDeps);
- maybeExecute(safeStore, safeCommand, shard, true, true);
+ maybeExecute(safeStore, safeCommand, true, true);
safeStore.progressLog().executed(safeCommand.current(), shard);
return ApplyOutcome.Success;
@@ -444,13 +494,23 @@ public class Commands
{
Command listener = safeListener.current();
Command updated = safeUpdated.current();
+ if (listener.is(NotDefined) || listener.is(Truncated))
+ {
+ // This listener must be a stale vestige
+ // TODO (desired): would be nice to ensure these are deregistered explicitly, but would be costly
+ Invariants.checkState(listener.saveStatus().isUninitialised() || listener.is(Truncated), "Listener status expected to be Uninitialised or Truncated, but was %s", listener.saveStatus());
+ Invariants.checkState(updated.hasBeen(Applied) || updated.is(NotDefined), "Updated status expected to be Applied or NotDefined, but was %s", updated);
+ safeUpdated.removeListener(listener.asListener());
+ return;
+ }
+
logger.trace("{}: updating as listener in response to change on {} with status {} ({})",
listener.txnId(), updated.txnId(), updated.status(), updated);
switch (updated.status())
{
default:
throw new IllegalStateException("Unexpected status: " + updated.status());
- case NotWitnessed:
+ case NotDefined:
case PreAccepted:
case Accepted:
case AcceptedInvalidate:
@@ -462,6 +522,7 @@ public class Commands
case PreApplied:
case Applied:
case Invalidated:
+ case Truncated:
updateDependencyAndMaybeExecute(safeStore, safeListener, safeUpdated, true);
break;
}
@@ -470,7 +531,7 @@ public class Commands
protected static void postApply(SafeCommandStore safeStore, TxnId txnId)
{
logger.trace("{} applied, setting status to Applied and notifying listeners", txnId);
- SafeCommand safeCommand = safeStore.command(txnId);
+ SafeCommand safeCommand = safeStore.get(txnId);
safeCommand.applied();
safeStore.notifyListeners(safeCommand);
}
@@ -491,11 +552,6 @@ public class Commands
return safeStore.ranges().allBetween(txnId.epoch(), untilEpoch);
}
- private static boolean coordinates(SafeCommandStore safeStore, TxnId txnId, RoutingKey key)
- {
- return coordinateRanges(safeStore, txnId).contains(key);
- }
-
private static Ranges acceptRanges(SafeCommandStore safeStore, TxnId txnId, Timestamp executeAt, Ranges coordinateRanges)
{
return txnId.epoch() == executeAt.epoch() ? coordinateRanges : safeStore.ranges().allBetween(txnId, executeAt);
@@ -503,7 +559,7 @@ public class Commands
private static Ranges executeRanges(SafeCommandStore safeStore, Timestamp executeAt)
{
- return safeStore.ranges().allSince(executeAt.epoch());
+ return safeStore.ranges().allAt(executeAt.epoch());
}
/**
@@ -518,10 +574,13 @@ public class Commands
private static AsyncChain<Void> applyChain(SafeCommandStore safeStore, PreLoadContext context, TxnId txnId)
{
- Command.Executed command = safeStore.command(txnId).current().asExecuted();
+ Command.Executed command = safeStore.get(txnId).current().asExecuted();
if (command.hasBeen(Applied))
return AsyncChains.success(null);
+ // TODO (required): make sure we are correctly handling (esp. C* side with validation logic) executing a transaction
+ // that was pre-bootstrap for some range (so redundant and we may have gone ahead of), but had to be executed locally
+ // for another range
CommandStore unsafeStore = safeStore.commandStore();
return command.writes().apply(safeStore, applyRanges(safeStore, command.executeAt()))
.flatMap(unused -> unsafeStore.submit(context, ss -> {
@@ -550,7 +609,7 @@ public class Commands
}
// TODO (expected, API consistency): maybe split into maybeExecute and maybeApply?
- private static boolean maybeExecute(SafeCommandStore safeStore, SafeCommand safeCommand, ProgressShard shard, boolean alwaysNotifyListeners, boolean notifyWaitingOn)
+ private static boolean maybeExecute(SafeCommandStore safeStore, SafeCommand safeCommand, boolean alwaysNotifyListeners, boolean notifyWaitingOn)
{
Command command = safeCommand.current();
if (logger.isTraceEnabled())
@@ -573,6 +632,7 @@ public class Commands
return false;
}
+ // TODO (required): slice our execute ranges based on any pre-bootstrap state
// FIXME: need to communicate to caller that we didn't execute if we take one of the above paths
switch (command.status())
@@ -581,7 +641,7 @@ public class Commands
// TODO (desirable, efficiency): maintain distinct ReadyToRead and ReadyToWrite states
command = safeCommand.readyToExecute();
logger.trace("{}: set to ReadyToExecute", command.txnId());
- safeStore.progressLog().readyToExecute(command, shard);
+ safeStore.progressLog().readyToExecute(command);
safeStore.notifyListeners(safeCommand);
return true;
@@ -592,6 +652,7 @@ public class Commands
if (intersects)
{
+ safeCommand.applying();
logger.trace("{}: applying", command.txnId());
apply(safeStore, executed);
return true;
@@ -606,276 +667,377 @@ public class Commands
return true;
}
default:
- throw new IllegalStateException();
+ throw new IllegalStateException("Unexpected status: " + command.status());
}
}
- protected static WaitingOn populateWaitingOn(SafeCommandStore safeStore, TxnId txnId, Timestamp executeAt, PartialDeps partialDeps)
+ protected static WaitingOn initialiseWaitingOn(SafeCommandStore safeStore, TxnId waitingId, Timestamp executeWaitingAt, PartialDeps partialDeps, Route<?> route)
{
- Ranges ranges = applyRanges(safeStore, executeAt);
- if (ranges.isEmpty())
- return WaitingOn.EMPTY;
-
- return populateWaitingOn(safeStore, ranges, txnId, executeAt, partialDeps);
+ Ranges ranges = safeStore.ranges().allAt(executeWaitingAt);
+ Unseekables<?> executionParticipants = route.participants().slice(ranges);
+ WaitingOn.Update update = new WaitingOn.Update(ranges, executionParticipants, partialDeps);
+ return updateWaitingOn(safeStore, waitingId, executeWaitingAt, update, route.participants()).build();
}
- protected static <P> void visitWaitingOn(SafeCommandStore safeStore, TxnId txnId, Timestamp waitUntil, Deps waitOn, Timestamp executeAt, WaitingOnVisitor<P> visitor, P param)
+ protected static WaitingOn.Update updateWaitingOn(SafeCommandStore safeStore, TxnId txnId, Timestamp executeAt, WaitingOn.Update update, Participants<?> participants)
{
- Ranges ranges = applyRanges(safeStore, executeAt);
- if (ranges.isEmpty())
- return;
-
- visitWaitingOn(safeStore, ranges, txnId, waitUntil, waitOn, visitor, param);
- }
+ CommandStore commandStore = safeStore.commandStore();
+ TxnId minWaitingOnTxnId = update.minWaitingOnTxnId();
+ if (minWaitingOnTxnId != null && commandStore.hasLocallyRedundantDependencies(update.minWaitingOnTxnId(), executeAt, participants))
+ safeStore.commandStore().removeRedundantDependencies(participants, update);
+
+ update.forEachWaitingOnCommit(safeStore, update, txnId, executeAt, (safeStore0, upd, id, exec, i) -> {
+ // TODO (expected): load read-only to reduce overhead; upgrade only if we need to remove listener
+ SafeCommand dep = safeStore0.ifLoadedAndInitialised(upd.deps.txnId(i));
+ if (dep == null || !dep.current().hasBeen(PreCommitted))
+ return;
+ updateWaitingOn(safeStore0, id, exec, upd, dep);
+ });
- protected static WaitingOn populateWaitingOn(SafeCommandStore safeStore, Ranges ranges, TxnId waitingId, Timestamp executeAt, PartialDeps partialDeps)
- {
- WaitingOn.Update update = new WaitingOn.Update();
- visitWaitingOn(safeStore, ranges, waitingId, executeAt, partialDeps, Commands::populateWaitingOn, update);
- return update.build();
- }
+ update.forEachWaitingOnApply(safeStore, update, txnId, executeAt, (store, upd, id, exec, i) -> {
+ SafeCommand dep = store.ifLoadedAndInitialised(upd.deps.txnId(i));
+ if (dep == null || !dep.current().hasBeen(PreCommitted))
+ return;
+ updateWaitingOn(store, id, exec, upd, dep);
+ });
- public interface WaitingOnVisitor<P>
- {
- void visit(SafeCommandStore safeStore, TxnId waitingId, Timestamp executeAt, TxnId dependencyId, P param);
+ return update;
}
- protected static <P> void visitWaitingOn(SafeCommandStore safeStore, Ranges ranges, TxnId waitingId, Timestamp waitUntil, Deps deps, WaitingOnVisitor<P> visitor, P param)
+ /**
+ * @param dependencySafeCommand is either committed truncated, or invalidated
+ * @return true iff {@code maybeExecute} might now have a different outcome
+ */
+ private static boolean updateWaitingOn(SafeCommandStore safeStore, TxnId waitingId, Timestamp executeWaitingAt, WaitingOn.Update waitingOn, SafeCommand dependencySafeCommand)
{
- boolean isAffectedByBootstrap = safeStore.commandStore().isAffectedByBootstrap(deps);
- if (!isAffectedByBootstrap)
- {
- deps.forEachUniqueTxnId(ranges, dependencyId -> visitor.visit(safeStore, waitingId, waitUntil, dependencyId, param));
- return;
- }
-
- BitSet bits = new BitSet(Math.max(deps.keyDeps.txnIdCount(), deps.rangeDeps.txnIdCount()));
- if (!deps.keyDeps.isEmpty())
+ Command dependency = dependencySafeCommand.current();
+ Invariants.checkState(dependency.hasBeen(PreCommitted));
+ TxnId dependencyId = dependency.txnId();
+ if (dependency.is(Truncated))
{
- SortedArrays.SortedArrayList<TxnId> dependencyIds = deps.keyDeps.txnIds();
- // process each interval of TxnId we have a different incomplete range for in a batch,
- safeStore.commandStore().forEachBootstrapRange(dependencyIds, (kdeps, bootstrapId, rs, start, end) -> {
- // We do not need to depend on the bootstrap transaction for writes, as we have a timestamp store
- // (so any write we perform will persist past the bootstrap completing).
- // For most reads we can rely on safeToRead, but this is not the case for reads that depend on
- // writes that started before our ExclusiveSyncPoint but execute afterwards.
- // We must retain a dependency on these transactions.
-
- rs = ranges.slice(rs, Minimal);
- kdeps.forEach(rs, (bs, txnId, i) -> {
- if (i < end && i >= start)
- bs.set(i);
- }, bits);
- }, deps.keyDeps);
- int i = -1;
- while ((i = bits.nextSetBit(i + 1)) >= 0)
- visitor.visit(safeStore, waitingId, waitUntil, dependencyIds.get(i), param);
+ logger.trace("{}: {} is truncated. Stop listening and removing from waiting on commit set.", waitingId, dependencyId);
+ dependencySafeCommand.removeListener(new ProxyListener(waitingId));
+ return waitingOn.removeInvalidatedAppliedOrTruncated(dependencyId);
}
-
- if (!deps.rangeDeps.isEmpty())
+ else if (dependency.is(Invalidated))
{
- bits.clear();
- SortedArrays.SortedArrayList<TxnId> dependencyIds = deps.rangeDeps.txnIds();
- safeStore.commandStore().forEachBootstrapRange(dependencyIds, (rdeps, bootstrapId, rs, start, end) -> {
- rs = ranges.slice(rs, Minimal);
- rdeps.forEach(rs, (bs, i) -> {
- if (i >= start && i < end)
- bs.set(i);
- }, bits);
- }, deps.rangeDeps);
- int i = -1;
- while ((i = bits.nextSetBit(i + 1)) >= 0)
- visitor.visit(safeStore, waitingId, waitUntil, dependencyIds.get(i), param);
+ logger.trace("{}: {} is invalidated. Stop listening and removing from waiting on commit set.", waitingId, dependencyId);
+ dependencySafeCommand.removeListener(new ProxyListener(waitingId));
+ return waitingOn.removeInvalidatedAppliedOrTruncated(dependencyId);
}
- }
-
- @Inline
- private static void populateWaitingOn(SafeCommandStore safeStore, TxnId waitingId, Timestamp executeAt, TxnId dependencyId, WaitingOn.Update update)
- {
- SafeCommand dependencySafeCommand = safeStore.ifLoaded(dependencyId);
- if (dependencySafeCommand == null)
+ else if (dependency.executeAt().compareTo(executeWaitingAt) > 0 && !waitingId.rw().awaitsFutureDeps())
{
- update.addWaitingOnCommit(dependencyId);
- safeStore.addAndInvokeListener(dependencyId, waitingId);
+ // dependency cannot be a predecessor if it executes later
+ logger.trace("{}: {} executes after us. Stop listening and removing from waiting on apply set.", waitingId, dependencyId);
+ dependencySafeCommand.removeListener(new ProxyListener(waitingId));
+ return waitingOn.removeWaitingOn(dependencyId);
}
- else
+ else if (dependency.hasBeen(Applied))
{
- Command command = dependencySafeCommand.current();
- switch (command.status())
- {
- default:
- throw new IllegalStateException();
- case NotWitnessed:
- case PreAccepted:
- case Accepted:
- case AcceptedInvalidate:
- case PreCommitted:
- // we don't know when these dependencies will execute, and cannot execute until we do
-
- command = dependencySafeCommand.addListener(new Command.ProxyListener(waitingId));
- update.addWaitingOnCommit(command.txnId());
- break;
- case Committed:
- // TODO (desired, efficiency): split into ReadyToRead and ReadyToWrite;
- // the distributed read can be performed as soon as those keys are ready,
- // and in parallel with any other reads. the client can even ACK immediately after;
- // only the write needs to be postponed until other in-progress reads complete
- case ReadyToExecute:
- case PreApplied:
- command = dependencySafeCommand.addListener(new Command.ProxyListener(waitingId));
- insertWaitingOn(waitingId, executeAt, update, command);
- case Applied:
- case Invalidated:
- break;
- }
+ logger.trace("{}: {} has been applied. Stop listening and removing from waiting on apply set.", waitingId, dependencyId);
+ dependencySafeCommand.removeListener(new ProxyListener(waitingId));
+ return waitingOn.removeApplied(dependencyId, dependency.asCommitted().waitingOn());
}
- }
-
- private static void insertWaitingOn(TxnId txnId, Timestamp executeAt, WaitingOn.Update update, Command dependencyId)
- {
- Invariants.checkState(dependencyId.hasBeen(Committed));
- if (dependencyId.hasBeen(Invalidated))
+ else if (waitingOn.removeWaitingOnCommit(dependencyId))
{
- logger.trace("{}: {} is invalidated. Do not insert.", txnId, dependencyId.txnId());
+ logger.trace("{}: adding {} to waiting on apply set.", waitingId, dependencyId);
+ boolean addedWaitingOnApply = waitingOn.addWaitingOnApply(dependencyId);
+ Invariants.checkState(addedWaitingOnApply);
+ return true;
}
- else if (dependencyId.executeAt().compareTo(executeAt) > 0)
+ else if (waitingOn.isWaitingOnApply(dependencyId))
{
- // dependency cannot be a predecessor if it executes later
- logger.trace("{}: {} executes after us. Do not insert.", txnId, dependencyId.txnId());
+ return false;
}
- else if (dependencyId.hasBeen(Applied))
+ else if (safeStore.isFullyPreBootstrap(dependency, waitingOn.deps.participants(dependency.txnId())))
{
- logger.trace("{}: {} has been applied. Do not insert.", txnId, dependencyId.txnId());
+ // TODO (expected): erase or otherwise prevent from executing
+ return false;
}
else
{
- logger.trace("{}: adding {} to waiting on apply set.", txnId, dependencyId.txnId());
- update.addWaitingOnApply(dependencyId.txnId(), dependencyId.executeAt());
+ throw new IllegalStateException("We have a dependency to wait on, but have already finished waiting");
}
}
- /**
- * @param safeDependency is either committed or invalidated
- * @return true iff {@code maybeExecute} might now have a different outcome
- */
- private static boolean updateWaitingOn(SafeCommand dependencySafeCommand, WaitingOn.Update waitingOn, SafeCommand safeDependency)
+ static void updateDependencyAndMaybeExecute(SafeCommandStore safeStore, SafeCommand safeCommand, SafeCommand predecessor, boolean notifyWaitingOn)
{
- Command.Committed command = dependencySafeCommand.current().asCommitted();
- Command dependency = safeDependency.current();
- Invariants.checkState(dependency.hasBeen(PreCommitted));
- if (dependency.hasBeen(Invalidated))
- {
- logger.trace("{}: {} is invalidated. Stop listening and removing from waiting on commit set.", command.txnId(), dependency.txnId());
- safeDependency.removeListener(command.asListener());
- waitingOn.removeWaitingOnCommit(dependency.txnId());
- return true;
- }
- else if (dependency.executeAt().compareTo(command.executeAt()) > 0)
- {
- // dependency cannot be a predecessor if it executes later
- logger.trace("{}: {} executes after us. Stop listening and removing from waiting on apply set.", command.txnId(), dependency.txnId());
- waitingOn.removeWaitingOn(dependency.txnId(), dependency.executeAt());
- safeDependency.removeListener(command.asListener());
- return true;
- }
- else if (dependency.hasBeen(Applied))
+ Command.Committed command = safeCommand.current().asCommitted();
+ if (command.hasBeen(Applied))
+ return;
+
+ WaitingOn.Update waitingOn = new WaitingOn.Update(command);
+ if (updateWaitingOn(safeStore, command.txnId(), command.executeAt(), waitingOn, predecessor))
{
- logger.trace("{}: {} has been applied. Stop listening and removing from waiting on apply set.", command.txnId(), dependency.txnId());
- waitingOn.removeWaitingOn(dependency.txnId(), dependency.executeAt());
- safeDependency.removeListener(command.asListener());
- return true;
+ safeCommand.updateWaitingOn(waitingOn);
+ maybeExecute(safeStore, safeCommand, false, notifyWaitingOn);
}
- else if (command.isWaitingOnDependency())
+ else
{
- logger.trace("{}: adding {} to waiting on apply set.", command.txnId(), dependency.txnId());
- waitingOn.addWaitingOnApply(dependency.txnId(), dependency.executeAt());
- waitingOn.removeWaitingOnCommit(dependency.txnId());
- return false;
+ Command pred = predecessor.current();
+ if (pred.hasBeen(PreCommitted))
+ {
+ TxnId nextWaitingOn = command.waitingOn().nextWaitingOn();
+ if (nextWaitingOn != null && nextWaitingOn.equals(pred.txnId()))
+ safeStore.progressLog().waiting(predecessor, LocalExecution.Applied, pred.route(), null);
+ }
}
- else
+ }
+
+ public enum Cleanup
+ {
+ NO(Uninitialised),
+ TRUNCATE_WITH_OUTCOME(TruncatedApplyWithOutcome),
+ TRUNCATE(TruncatedApply),
+ ERASE(Erased);
+
+ public final SaveStatus appliesIfNot;
+
+ Cleanup(SaveStatus appliesIfNot)
{
- throw new IllegalStateException();
+ this.appliesIfNot = appliesIfNot;
}
}
- static void updateDependencyAndMaybeExecute(SafeCommandStore safeStore, SafeCommand dependencySafeCommand, SafeCommand livePredecessor, boolean notifyWaitingOn)
+ // TODO (now): document and justify all calls
+ public static void setTruncatedApply(SafeCommandStore safeStore, SafeCommand safeCommand)
{
- Command.Committed command = dependencySafeCommand.current().asCommitted();
- if (command.hasBeen(Applied))
- return;
+ cleanup(safeStore, safeCommand, TRUNCATE, true);
+ }
- WaitingOn.Update waitingOn = new WaitingOn.Update(command);
- boolean attemptExecution = updateWaitingOn(dependencySafeCommand, waitingOn, livePredecessor);
- command = dependencySafeCommand.updateWaitingOn(waitingOn);
+ public static void setErased(SafeCommandStore safeStore, SafeCommand safeCommand)
+ {
+ cleanup(safeStore, safeCommand, ERASE, true);
+ }
- if (attemptExecution)
- maybeExecute(safeStore, dependencySafeCommand, progressShard(safeStore, command), false, notifyWaitingOn);
+ public static Command cleanup(SafeCommandStore safeStore, SafeCommand safeCommand, Cleanup cleanup, boolean notifyListeners)
+ {
+ Command command = safeCommand.current();
+
+ // 1) a command has been applied; or
+ // 2) has been coordinated but *will not* be applied (we just haven't witnessed the invalidation yet); or
+ // 3) a command is durably decided and this shard only hosts its home data, so no explicit truncation is necessary to remove it
+ // TODO (desired): consider if there are better invariants we can impose for undecided transactions, to verify they aren't later committed (should be detected already, but more is better)
+ // note that our invariant here is imperfectly applied to keep the code cleaner: we don't verify that the caller was safe to invoke if we don't already have a route in the command and we're only PreCommitted
+ Invariants.checkState(command.hasBeen(Applied) || !command.hasBeen(PreCommitted)
+ || (command.route() == null || Infer.safeToCleanup(safeStore, command, command.route(), command.executeAt()) || safeStore.isFullyPreBootstrap(command, command.route().participants()))
+ , "Command %s could not be truncated", command);
+
+ Command result;
+ switch (cleanup)
+ {
+ default: throw new AssertionError("Unexpected status: " + cleanup);
+ case TRUNCATE_WITH_OUTCOME:
+ Invariants.checkArgument(!command.hasBeen(Truncated));
+ if (command.hasBeen(PreApplied))
+ {
+ result = truncatedApplyWithOutcome(command.asExecuted());
+ break;
+ }
+ // TODO (expected, consider): should we downgrade to no truncation in this case? Or are we stale?
+
+ case TRUNCATE:
+ Invariants.checkState(command.saveStatus().compareTo(TruncatedApply) < 0);
+ if (command.hasBeen(PreCommitted)) result = truncatedApply(command);
+ else result = erased(command); // TODO (expected): check if we are stale; if not, is erase anyway correct outcome?
+ break;
+
+ case ERASE:
+ Invariants.checkState(command.saveStatus().compareTo(Erased) < 0);
+ result = erased(command);
+ break;
+ }
+
+ safeCommand.set(result);
+ safeStore.progressLog().clear(safeCommand.txnId());
+ if (notifyListeners)
+ safeStore.notifyListeners(safeCommand, result, command.durableListeners(), safeCommand.transientListeners());
+ return result;
}
- // TODO (now): check/move methods below
- private static Command setDurability(SafeCommandStore safeStore, SafeCommand dependencySafeCommand, Durability durability, RoutingKey homeKey, @Nullable Timestamp executeAt)
+ public static boolean cleanup(SafeCommandStore safeStore, SafeCommand safeCommand, Command command, EpochSupplier toEpoch, Unseekables<?> maybeFullRoute)
{
- Command command = dependencySafeCommand.current();
- CommonAttributes attrs = updateHomeKey(safeStore, command.txnId(), command, homeKey);
- if (executeAt != null && command.status().hasBeen(Committed) && !command.asCommitted().executeAt().equals(executeAt))
- safeStore.agent().onInconsistentTimestamp(command, command.asCommitted().executeAt(), executeAt);
- attrs = attrs.mutable().durability(durability);
- return dependencySafeCommand.updateAttributes(attrs);
+ Cleanup cleanup = shouldCleanup(safeStore, command, toEpoch, maybeFullRoute);
+ if (command.saveStatus().compareTo(cleanup.appliesIfNot) >= 0)
+ return false;
+
+ cleanup(safeStore, safeCommand, cleanup, true);
+ return true;
}
- public static Command setDurability(SafeCommandStore safeStore, TxnId txnId, Durability durability, RoutingKey homeKey, @Nullable Timestamp executeAt)
+ static Cleanup shouldCleanup(SafeCommandStore safeStore, Command command, EpochSupplier toEpoch, Unseekables<?> maybeFullRoute)
{
- return setDurability(safeStore, safeStore.command(txnId), durability, homeKey, executeAt);
+ if (command.saveStatus() == Erased)
+ return NO; // once erased we no longer have executeAt, and may consider that a transaction is owned by us that should not be (as its txnId is an earlier epoch than we adopted a range)
+
+ Route<?> route = command.route();
+ if (!Route.isFullRoute(route))
+ {
+ if (command.known().hasFullRoute()) throw new IllegalStateException(command.saveStatus() + " should include full route, but only found " + route);
+ else if (Route.isFullRoute(maybeFullRoute)) route = Route.castToFullRoute(maybeFullRoute);
+ else return Cleanup.NO;
+ }
+
+ Timestamp executeAt = command.executeAt();
+ if (toEpoch == null || (executeAt != null && executeAt.epoch() > toEpoch.epoch()))
+ toEpoch = executeAt;
+
+ return shouldCleanup(command.txnId(), command.status(), command.durability(), toEpoch, route,
+ safeStore.commandStore().redundantBefore(), safeStore.commandStore().durableBefore());
}
- private static TxnId firstWaitingOnCommit(Command command)
+ public static Cleanup shouldCleanup(TxnId txnId, Status status, Durability durability, EpochSupplier toEpoch, Route<?> route, RedundantBefore redundantBefore, DurableBefore durableBefore)
{
- if (!command.hasBeen(Committed))
- return null;
+ if (durableBefore.min(txnId) == Universal)
+ {
+ if (status.hasBeen(PreCommitted) && !status.hasBeen(Applied)) // TODO (expected): may be stale
+ throw new IllegalStateException("Loading universally-durable command that has been PreCommitted but not Applied");
+ return Cleanup.ERASE;
+ }
+
+ if (!Route.isFullRoute(route))
+ return NO;
+
+ // We first check if the command is redundant locally, i.e. whether it has been applied to all non-faulty replicas of the local shard
+ // If not, we don't want to truncate its state else we may make catching up for these other replicas much harder
+ RedundantStatus redundant = redundantBefore.status(txnId, toEpoch, route.participants());
+ switch (redundant)
+ {
+ default: throw new AssertionError();
+ case NOT_OWNED:
+ // TODO (expected): we can impose additional validations here IF we receive an epoch upper bound
+ // TODO (expected): we should be more robust to the presence/absence of executeAt
+ if (route.isParticipatingHomeKey() || redundantBefore.get(txnId, toEpoch, route.homeKey()) == NOT_OWNED)
+ throw new IllegalStateException("Command " + txnId + " that is being loaded is not owned by this shard on route " + route);
+ case LIVE:
+ case PARTIALLY_PRE_BOOTSTRAP:
+ case PRE_BOOTSTRAP:
+ return NO;
+ case LOCALLY_REDUNDANT:
+ if (status.hasBeen(PreCommitted) && !status.hasBeen(Applied)) // TODO (required): may be stale
+ throw new IllegalStateException("Loading redundant command that has been PreCommitted but not Applied");
+ }
+
+ Durability min = durableBefore.min(txnId, route);
+ switch (min)
+ {
+ default:
+ case Local:
+ case DurableOrInvalidated:
+ throw new AssertionError("Unexpected durability: " + min);
+ case NotDurable:
+ if (durability == Majority)
+ return Cleanup.TRUNCATE;
+
+ return Cleanup.TRUNCATE_WITH_OUTCOME;
- Command.Committed committed = command.asCommitted();
- return committed.isWaitingOnCommit() ? committed.waitingOnCommit().first() : null;
+ case Majority:
+ return Cleanup.TRUNCATE;
+
+ case Universal:
+ return Cleanup.ERASE;
+ }
}
- private static TxnId firstWaitingOnApply(Command command, @Nullable TxnId ifExecutesBefore)
+ // TODO (now): either ignore this message if we don't have a route, or else require FullRoute requiring route, or else require FullRoute
+ public static Command setDurability(SafeCommandStore safeStore, SafeCommand safeCommand, Durability durability, @Nullable Route<?> route, @Nullable Timestamp executeAt)
{
- if (!command.hasBeen(Committed))
- return null;
+ Command command = safeCommand.current();
+ if (command.is(Truncated))
+ return command;
- Command.Committed committed = command.asCommitted();
- if (!committed.isWaitingOnApply())
- return null;
+ if (command.durability().compareTo(durability) >= 0)
+ return command;
- Map.Entry<Timestamp, TxnId> first = committed.waitingOnApply().firstEntry();
- if (ifExecutesBefore == null || first.getKey().compareTo(ifExecutesBefore) < 0)
- return first.getValue();
+ CommonAttributes attrs = route == null ? command : updateRoute(command, route);
+ if (executeAt != null && command.status().hasBeen(Committed) && !command.asCommitted().executeAt().equals(executeAt))
+ safeStore.agent().onInconsistentTimestamp(command, command.asCommitted().executeAt(), executeAt);
+ attrs = attrs.mutable().durability(durability);
+ command = safeCommand.updateAttributes(attrs);
+
+ if (cleanup(safeStore, safeCommand, command, null, route))
+ return safeCommand.current();
- return null;
+ safeStore.progressLog().durable(command);
+ safeStore.notifyListeners(safeCommand);
+ return command;
}
- // TODO (expected): this should integrate with LocalBarrier so that we do not need to register interest with every transaction
+ public static Command setDurability(SafeCommandStore safeStore, SafeCommand safeCommand, Durability durability)
+ {
+ return setDurability(safeStore, safeCommand, durability, null, null);
+ }
+
+ // TODO (expected): we should have a new SaveState that avoids us duplicating work that indicates we're WaitingToExecute;
+ // this means that once we have begun visiting a transaction on behalf of some later transaction, we don't need to do so again.
static class NotifyWaitingOn implements PreLoadContext, Consumer<SafeCommandStore>
{
- Known[] blockedUntil = new Known[4];
+ LocalExecution[] blockedUntil = new LocalExecution[4];
TxnId[] txnIds = new TxnId[4];
int depth;
public NotifyWaitingOn(SafeCommand root)
{
txnIds[0] = root.txnId();
- blockedUntil[0] = Known.Done;
+ blockedUntil[0] = LocalExecution.Applied;
}
@Override
public void accept(SafeCommandStore safeStore)
{
- SafeCommand prevSafe = get(safeStore, depth - 1);
- while (depth >= 0)
+ // first do a simple loop to skip over txns that have been truncated
+ SafeCommand prevSafe = ifInitialised(safeStore, depth - 1);
+ if (depth > 0 && (prevSafe == null || prevSafe.current().hasBeen(Truncated)))
+ {
+ while (depth > 0 && (prevSafe == null || prevSafe.current().hasBeen(Truncated)))
+ prevSafe = ifInitialised(safeStore, --depth - 1);
+ }
+ else
+ {
+ // we know we loaded cur, so if it's null it's either truncated or we haven't witnessed it and need to initialise;
+ // in this case use our predecessor's intersecting keys to decide which
+ SafeCommand curSafe = ifInitialised(safeStore, depth);
+ if (curSafe == null)
+ {
+ if (prevSafe != null)
+ {
+ RedundantStatus redundantStatus = safeStore.commandStore().redundantBefore().status(txnIds[depth], prevSafe.current().executeAt(), prevSafe.current().partialDeps().participants(txnIds[depth]));
+ switch (redundantStatus)
+ {
+ default: throw new AssertionError("Unexpected redundant status: " + redundantStatus);
+ case NOT_OWNED: throw new AssertionError("Invalid state: waiting for execution of command that is not owned at the execution time");
+ case LOCALLY_REDUNDANT:
+ case PRE_BOOTSTRAP:
+ removeRedundantDependencies(safeStore, prevSafe, txnIds[depth], redundantStatus == PRE_BOOTSTRAP);
+ prevSafe = get(safeStore, --depth - 1);
+ break;
+ case LIVE:
+ case PARTIALLY_PRE_BOOTSTRAP:
+ initialise(safeStore, depth);
+ }
+ }
+ else
+ {
+ do
+ {
+ if (--depth == -1)
+ return; // the command must have been truncated
+ curSafe = prevSafe;
+ prevSafe = ifInitialised(safeStore, depth - 1);
+ } while (curSafe == null);
+ }
+ }
+ }
+
+ Invariants.checkState(depth == 0 || prevSafe != null);
+ loop: while (depth >= 0)
{
+ if (depth > 100000) // TODO (expected): dump more debug info to aid investigation
+ throw new StackOverflowError("Exploring a probably-recursive or otherwise faulty dependency graph from " + txnIds[0]);
Command prev = prevSafe != null ? prevSafe.current() : null;
- SafeCommand curSafe = ifLoaded(safeStore, depth);
+ SafeCommand curSafe = ifLoadedAndInitialised(safeStore, depth);
Command cur = curSafe != null ? curSafe.current() : null;
- Known until = blockedUntil[depth];
+ LocalExecution until = blockedUntil[depth];
if (cur == null)
{
// need to load; schedule execution for later
@@ -885,7 +1047,7 @@ public class Commands
if (prev != null)
{
- if (cur.has(until) || (cur.hasBeen(PreCommitted) && cur.executeAt().compareTo(prev.executeAt()) > 0))
+ if (cur.isAtLeast(until) || (cur.hasBeen(PreCommitted) && cur.executeAt().compareTo(prev.executeAt()) > 0 && !prev.txnId().rw().awaitsFutureDeps()))
{
updateDependencyAndMaybeExecute(safeStore, prevSafe, curSafe, false);
--depth;
@@ -893,57 +1055,101 @@ public class Commands
continue;
}
}
- else if (cur.has(until))
+ else if (cur.isAtLeast(until))
{
- // we're done; have already applied
+ // we're done; already applying
Invariants.checkState(depth == 0);
break;
}
- TxnId directlyBlockedOnCommit = firstWaitingOnCommit(cur);
- TxnId directlyBlockedOnApply = firstWaitingOnApply(cur, directlyBlockedOnCommit);
+ WaitingOn waitingOn = cur.hasBeen(Committed) ? cur.asCommitted().waitingOn() : WaitingOn.EMPTY;
+ TxnId directlyBlockedOnCommit;
+ TxnId directlyBlockedOnApply = waitingOn.nextWaitingOnApply();
if (directlyBlockedOnApply != null)
{
- push(directlyBlockedOnApply, Known.Done);
+ // preferentially block on apply, as this probably saves us additional bookkeeping
+ // by giving other dependencies time to complete
+ push(directlyBlockedOnApply, LocalExecution.Applied);
+ prevSafe = curSafe;
}
- else if (directlyBlockedOnCommit != null)
+ else if ((directlyBlockedOnCommit = waitingOn.nextWaitingOnCommit()) != null)
{
- push(directlyBlockedOnCommit, ExecuteAtOnly);
+ push(directlyBlockedOnCommit, LocalExecution.ReadyToExclude);
+ prevSafe = curSafe;
}
else
{
- if (cur.hasBeen(Committed) && !cur.hasBeen(ReadyToExecute) && !cur.asCommitted().isWaitingOnDependency())
+ if (cur.hasBeen(Committed) && !cur.is(ReadyToExecute) && cur.saveStatus() != Applying && !cur.asCommitted().isWaitingOnDependency())
{
- if (!maybeExecute(safeStore, curSafe, progressShard(safeStore, cur), false, false))
+ if (!maybeExecute(safeStore, curSafe, false, false))
throw new AssertionError("Is able to Apply, but has not done so");
// loop and re-test the command's status; we may still want to notify blocking, esp. if not homeShard
continue;
}
- Unseekables<?, ?> someKeys = cur.maxUnseekables();
- if (someKeys == null && prev != null) someKeys = prev.partialDeps().someUnseekables(cur.txnId());
- Invariants.checkState(someKeys != null);
- logger.trace("{} blocked on {} until {}", txnIds[0], cur.txnId(), until);
- safeStore.progressLog().waiting(cur.txnId(), until, someKeys);
- return;
+ Timestamp executeAt = prev == null ? cur.executeAt() : prev.executeAt();
+ Participants<?> participants = prev != null // TODO (desired): slightly costly to invert a large partialDeps collection
+ ? prev.partialDeps().participants(cur.txnId()) // we do want to limit to the intersection of keys with the waiting transaction
+ : cur.route().participants(); // no need to slice to execution ranges, as implicitly done for us by RedundantBefore
+
+ RedundantStatus redundantStatus = safeStore.commandStore().redundantBefore().status(cur.txnId(), executeAt, participants);
+ switch (redundantStatus)
+ {
+ default: throw new AssertionError("Unknown redundant status: " + redundantStatus);
+ case NOT_OWNED: throw new AssertionError("Invalid state: waiting for execution of command that is not owned at the execution time");
+ case LIVE:
+ case PARTIALLY_PRE_BOOTSTRAP:
+ logger.trace("{} blocked on {} until {}", txnIds[0], cur.txnId(), until);
+ safeStore.progressLog().waiting(curSafe, until, null, participants);
+ break loop;
+
+ case PRE_BOOTSTRAP:
+ case LOCALLY_REDUNDANT:
+ Invariants.checkState(cur.hasBeen(Applied) || !cur.hasBeen(PreCommitted) || redundantStatus == PRE_BOOTSTRAP);
+ if (prev == null)
+ return;
+
+ curSafe.removeListener(prev.asListener());
+
+ // we've been applied, invalidated, or are no longer relevant
+ if (cur.hasBeen(Applied)) removeInvalidatedAppliedOrTruncatedDependency(prevSafe, cur.txnId());
+ else removeRedundantDependencies(safeStore, prevSafe, cur.txnId(), redundantStatus == PRE_BOOTSTRAP);
+
+ --depth;
+ prevSafe = get(safeStore, depth - 1);
+ }
}
- prevSafe = curSafe;
}
+ for (int i = 1 ; i <= depth ; ++i)
+ get(safeStore, i).addListener(get(safeStore, i - 1).current().asListener());
}
- private SafeCommand ifLoaded(SafeCommandStore safeStore, int i)
+ private SafeCommand ifInitialised(SafeCommandStore safeStore, int i)
{
if (i < 0) return null;
- return safeStore.ifLoaded(txnIds[i]);
+ return safeStore.ifInitialised(txnIds[i]);
+ }
+
+ private SafeCommand ifLoadedAndInitialised(SafeCommandStore safeStore, int i)
+ {
+ if (i < 0) return null;
+ return safeStore.ifLoadedAndInitialised(txnIds[i]);
}
private SafeCommand get(SafeCommandStore safeStore, int i)
{
if (i < 0) return null;
- return safeStore.command(txnIds[i]);
+ SafeCommand result = safeStore.ifInitialised(txnIds[i]);
+ Invariants.checkState(result != null);
+ return result;
+ }
+
+ private SafeCommand initialise(SafeCommandStore safeStore, int i)
+ {
+ return safeStore.get(txnIds[i]);
}
- void push(TxnId by, Known until)
+ void push(TxnId by, LocalExecution until)
{
if (++depth == txnIds.length)
{
@@ -965,19 +1171,42 @@ public class Commands
{
return Arrays.asList(txnIds).subList(1, depth + 1);
}
+ }
- @Override
- public Seekables<?, ?> keys()
- {
- return Keys.EMPTY;
- }
+ static Command removeInvalidatedAppliedOrTruncatedDependency(SafeCommand safeCommand, TxnId redundant)
+ {
+ Command.Committed current = safeCommand.current().asCommitted();
+ WaitingOn.Update update = new WaitingOn.Update(current.waitingOn);
+ update.removeInvalidatedAppliedOrTruncated(redundant);
+ return safeCommand.updateWaitingOn(update);
}
- public static Command updateHomeKey(SafeCommandStore safeStore, SafeCommand dependencySafeCommand, RoutingKey homeKey)
+ static Command removeRedundantDependencies(SafeCommandStore safeStore, SafeCommand safeCommand, TxnId redundant, boolean isPreBootstrap)
{
- Command command = dependencySafeCommand.current();
- CommonAttributes attrs = updateHomeKey(safeStore, command.txnId(), command, homeKey);
- return dependencySafeCommand.updateAttributes(attrs);
+ CommandStore commandStore = safeStore.commandStore();
+ Command.Committed current = safeCommand.current().asCommitted();
+ WaitingOn.Update update = new WaitingOn.Update(current.waitingOn);
+ TxnId minWaitingOnTxnId = update.minWaitingOnTxnId();
+ if (minWaitingOnTxnId != null && commandStore.hasLocallyRedundantDependencies(update.minWaitingOnTxnId(), current.executeAt(), current.route().participants()))
+ safeStore.commandStore().removeRedundantDependencies(current.route().participants(), update);
+
+ if (isPreBootstrap) update.removeWaitingOn(redundant); // above operation not guaranteed to remove all redundant dependencies; transaction may
+ else update.removeInvalidatedAppliedOrTruncated(redundant);
+ return safeCommand.updateWaitingOn(update);
+ }
+
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ public static Command informHome(SafeCommandStore safeStore, SafeCommand safeCommand, Route<?> someRoute)
+ {
+ Command command = safeCommand.current();
+ Invariants.checkState(owns(safeStore, command.txnId().epoch(), someRoute.homeKey()));
+ // TODO (expected): tighten up definition of what we know about a Route (pack it into Known) and whether we've been witnessed to decide our action here
+ if (command.hasBeen(PreAccepted))
+ return command;
+
+ Command result = safeCommand.updateAttributes(command.mutable().route(Route.merge((Route)someRoute, command.route())));
+ safeStore.progressLog().unwitnessed(safeCommand.txnId(), Home);
+ return result;
}
/**
@@ -989,95 +1218,38 @@ public class Commands
* Note that for ProgressLog purposes the "home shard" is the shard as of txnId.epoch.
* For recovery purposes the "home shard" is as of txnId.epoch until Committed, and executeAt.epoch once Executed
*/
- public static CommonAttributes updateHomeKey(SafeCommandStore safeStore, TxnId txnId, CommonAttributes attrs, RoutingKey homeKey)
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ public static CommonAttributes updateRoute(Command command, Route<?> route)
{
- if (attrs.homeKey() == null)
- {
- attrs = attrs.mutable().homeKey(homeKey);
- // TODO (low priority, safety): if we're processed on a node that does not know the latest epoch,
- // do we guarantee the home key calculation is unchanged since the prior epoch?
- if (attrs.progressKey() == null && owns(safeStore, txnId.epoch(), homeKey))
- attrs = attrs.mutable().progressKey(homeKey);
- }
- else if (!attrs.homeKey().equals(homeKey))
- {
- throw new IllegalStateException();
- }
- return attrs;
- }
-
- private static CommonAttributes updateHomeAndProgressKeys(SafeCommandStore safeStore, TxnId txnId, CommonAttributes attrs, Route<?> route, @Nullable RoutingKey progressKey, Ranges coordinateRanges)
- {
- attrs = updateHomeKey(safeStore, txnId, attrs, route.homeKey());
- if (progressKey == null || progressKey == NO_PROGRESS_KEY)
- {
- if (attrs.progressKey() == null)
- attrs = attrs.mutable().progressKey(NO_PROGRESS_KEY);
- return attrs;
- }
- if (attrs.progressKey() == null) attrs = attrs.mutable().progressKey(progressKey);
- else if (!attrs.progressKey().equals(progressKey))
- throw new AssertionError();
- return attrs;
- }
-
- private static CommonAttributes updateHomeAndProgressKeys(SafeCommandStore safeStore, TxnId txnId, CommonAttributes attrs, Route<?> route, Ranges coordinateRanges)
- {
- if (attrs.progressKey() == null)
- return attrs;
+ if (command.route() == null || !command.route().containsAll(route))
+ return command.mutable().route(Route.merge((Route)route, command.route()));
- return updateHomeAndProgressKeys(safeStore, txnId, attrs, route, attrs.progressKey(), coordinateRanges);
+ return command;
}
- private static ProgressShard progressShard(CommonAttributes attrs, @Nullable RoutingKey progressKey, Ranges coordinateRanges)
+ private static ProgressShard progressShard(Route<?> route, @Nullable RoutingKey progressKey, Ranges coordinates)
{
- if (progressKey == null || progressKey == NO_PROGRESS_KEY)
- return No;
-
- if (!coordinateRanges.contains(progressKey))
- return No;
-
- return progressKey.equals(attrs.homeKey()) ? Home : Local;
- }
-
- private static ProgressShard progressShard(CommonAttributes attrs, Ranges coordinateRanges)
- {
- if (attrs.progressKey() == null)
- return Unsure;
-
- return progressShard(attrs, attrs.progressKey(), coordinateRanges);
- }
-
- private static ProgressShard progressShard(SafeCommandStore safeStore, Command command)
- {
- RoutingKey progressKey = command.progressKey();
if (progressKey == null)
- return Unsure;
-
- if (progressKey == noProgressKey())
return No;
- Ranges coordinateRanges = safeStore.ranges().coordinates(command.txnId());
- if (!coordinateRanges.contains(progressKey))
+ if (!coordinates.contains(progressKey))
return No;
- return progressKey.equals(command.homeKey()) ? Home : Local;
+ return progressKey.equals(route.homeKey()) ? Home : Local;
}
enum EnsureAction { Ignore, Check, Add, TrySet, Set }
+ @SuppressWarnings({"unchecked", "rawtypes"})
private static CommonAttributes set(SafeCommandStore safeStore, Command command, CommonAttributes attrs,
Ranges existingRanges, Ranges additionalRanges, ProgressShard shard, Route<?> route,
@Nullable PartialTxn partialTxn, EnsureAction ensurePartialTxn,
@Nullable PartialDeps partialDeps, EnsureAction ensurePartialDeps)
{
- Invariants.checkState(attrs.progressKey() != null);
+ Invariants.checkState(shard != Unsure);
Ranges allRanges = existingRanges.with(additionalRanges);
- if (shard.isHome() || shard.isProgress())
- attrs = attrs.mutable().route(Route.merge(attrs.route(), (Route)route));
- else
- attrs = attrs.mutable().route(route.slice(allRanges));
+ attrs = attrs.mutable().route(Route.merge(attrs.route(), (Route)route));
// TODO (soon): stop round-robin hashing; partition only on ranges
switch (ensurePartialTxn)
@@ -1091,7 +1263,8 @@ public class Commands
partialTxn = partialTxn.slice(allRanges, shard.isHome());
if (!command.txnId().rw().isLocal())
{
- Routables.foldlMissing((Seekables)partialTxn.keys(), attrs.partialTxn().keys(), (keyOrRange, p, v, i) -> {
+ Invariants.checkState(attrs.partialTxn().covers(existingRanges));
+ Routables.foldl(partialTxn.keys(), additionalRanges, (keyOrRange, p, v, i) -> {
// TODO (expected, efficiency): we may register the same ranges more than once
safeStore.register(keyOrRange, allRanges, command);
return v;
@@ -1103,7 +1276,8 @@ public class Commands
case Set:
case TrySet:
- attrs = attrs.mutable().partialTxn(partialTxn = partialTxn.slice(allRanges, shard.isHome()));
+ // TODO (expected): only includeQuery if shard.isHome(); this affects state eviction and is low priority given size in C*
+ attrs = attrs.mutable().partialTxn(partialTxn = partialTxn.slice(allRanges, true));
// TODO (expected, efficiency): we may register the same ranges more than once
// TODO (desirable, efficiency): no need to register on PreAccept if already Accepted
if (!command.txnId().rw().isLocal())
@@ -1144,34 +1318,22 @@ public class Commands
return false;
// first validate route
- if (shard.isHome())
- {
- switch (ensureRoute)
- {
- default: throw new AssertionError();
- case Check:
- if (!isFullRoute(attrs.route()) && !isFullRoute(route))
- return false;
- case Ignore:
- break;
- case Add:
- case Set:
- if (!isFullRoute(route))
- throw new IllegalArgumentException("Incomplete route (" + route + ") sent to home shard");
- break;
- case TrySet:
- if (!isFullRoute(route))
- return false;
- }
- }
- else
+ switch (ensureRoute)
{
- // failing any of these tests is always an illegal state
- if (!route.covers(existingRanges))
- return false;
-
- if (existingRanges != additionalRanges && !route.covers(additionalRanges))
- throw new IllegalArgumentException("Incomplete route (" + route + ") provided; does not cover " + additionalRanges);
+ default: throw new AssertionError("Unexpected action: " + ensureRoute);
+ case Check:
+ if (!isFullRoute(attrs.route()) && !isFullRoute(route))
+ return false;
+ case Ignore:
+ break;
+ case Add:
+ case Set:
+ if (!isFullRoute(route))
+ throw new IllegalArgumentException("Incomplete route (" + route + ") sent to home shard");
+ break;
+ case TrySet:
+ if (!isFullRoute(route))
+ return false;
}
// invalid to Add deps to Accepted or AcceptedInvalidate statuses, as Committed deps are not equivalent
@@ -1183,14 +1345,8 @@ public class Commands
if (!validate(ensurePartialTxn, existingRanges, additionalRanges, covers(attrs.partialTxn()), covers(partialTxn), "txn", partialTxn))
return false;
- if (partialTxn != null && attrs.txnId().rw() != null && !attrs.txnId().rw().equals(partialTxn.kind()))
- throw new IllegalArgumentException("Transaction has different kind to its TxnId");
-
- if (shard.isHome() && ensurePartialTxn != Ignore)
- {
- if (!hasQuery(attrs.partialTxn()) && !hasQuery(partialTxn))
- throw new IllegalStateException();
- }
+ Invariants.checkState(partialTxn == null || attrs.txnId().rw() == null || attrs.txnId().rw().equals(partialTxn.kind()), "Transaction has different kind to its TxnId");
+ Invariants.checkState(!shard.isHome() || ensurePartialTxn == Ignore || hasQuery(attrs.partialTxn()) || hasQuery(partialTxn), "Home transaction should include query");
return validate(ensurePartialDeps, existingRanges, additionalRanges, covers(attrs.partialDeps()), covers(partialDeps), "deps", partialDeps);
}
@@ -1201,7 +1357,7 @@ public class Commands
{
switch (action)
{
- default: throw new IllegalStateException();
+ default: throw new IllegalStateException("Unexpected action: " + action);
case Ignore:
break;
@@ -1239,7 +1395,7 @@ public class Commands
if (action == Check)
return false;
- throw new IllegalArgumentException("Missing additional " + kind + "; existing does not cover " + additionalRanges.difference(existingRanges));
+ throw new IllegalArgumentException("Missing additional " + kind + "; existing does not cover " + additionalRanges.subtract(existingRanges));
}
}
else if (existing != null)
@@ -1251,7 +1407,7 @@ public class Commands
if (action == Check)
return false;
- throw new IllegalArgumentException("Incomplete additional " + kind + " (" + obj + ") provided; does not cover " + additionalRanges.difference(existingRanges));
+ throw new IllegalArgumentException("Incomplete additional " + kind + " (" + obj + ") provided; does not cover " + additionalRanges.subtract(existingRanges));
}
}
else
@@ -1264,7 +1420,7 @@ public class Commands
if (action == Check)
return false;
- throw new IllegalArgumentException("Incomplete additional " + kind + " (" + obj + ") provided; does not cover " + additionalRanges.difference(existingRanges));
+ throw new IllegalArgumentException("Incomplete additional " + kind + " (" + obj + ") provided; does not cover " + additionalRanges.subtract(existingRanges));
}
}
break;
@@ -1272,23 +1428,4 @@ public class Commands
return true;
}
-
- // TODO (low priority, API): this is an ugly hack, need to encode progress/homeKey/Route state combinations much more clearly
- // (perhaps introduce encapsulating class representing each possible arrangement)
- static class NoProgressKey implements RoutingKey
- {
- @Override
- public int compareTo(@Nonnull RoutableKey that)
- {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Range asRange()
- {
- throw new UnsupportedOperationException();
- }
- }
-
- private static final NoProgressKey NO_PROGRESS_KEY = new NoProgressKey();
}
diff --git a/accord-core/src/main/java/accord/local/CommonAttributes.java b/accord-core/src/main/java/accord/local/CommonAttributes.java
index cd4cfb03..7b02308b 100644
--- a/accord-core/src/main/java/accord/local/CommonAttributes.java
+++ b/accord-core/src/main/java/accord/local/CommonAttributes.java
@@ -18,7 +18,6 @@
package accord.local;
-import accord.api.RoutingKey;
import accord.api.VisibleForImplementation;
import accord.primitives.PartialDeps;
import accord.primitives.PartialTxn;
@@ -30,8 +29,6 @@ public interface CommonAttributes
{
TxnId txnId();
Status.Durability durability();
- RoutingKey homeKey();
- RoutingKey progressKey();
Route<?> route();
PartialTxn partialTxn();
PartialDeps partialDeps();
@@ -44,10 +41,11 @@ public interface CommonAttributes
class Mutable implements CommonAttributes
{
+ // TODO review these are global ones are mutable
+ public static final CommonAttributes.Mutable EMPTY_ATTRS = new CommonAttributes.Mutable((TxnId) null);
+ public static final CommonAttributes.Mutable EMPTY_ATTRS_NONE_PARTIAL_DEPS = new Mutable((TxnId) null).partialDeps(PartialDeps.NONE);
private TxnId txnId;
private Status.Durability durability;
- private RoutingKey homeKey;
- private RoutingKey progressKey;
private Route<?> route;
private PartialTxn partialTxn;
private PartialDeps partialDeps;
@@ -62,8 +60,6 @@ public interface CommonAttributes
{
this.txnId = attributes.txnId();
this.durability = attributes.durability();
- this.homeKey = attributes.homeKey();
- this.progressKey = attributes.progressKey();
this.route = attributes.route();
this.partialTxn = attributes.partialTxn();
this.partialDeps = attributes.partialDeps();
@@ -84,6 +80,7 @@ public interface CommonAttributes
public Mutable txnId(TxnId txnId)
{
+ Invariants.checkState(this != EMPTY_ATTRS && this != EMPTY_ATTRS_NONE_PARTIAL_DEPS, "Don't mutate the global empty instances");
this.txnId = txnId;
return this;
}
@@ -96,35 +93,11 @@ public interface CommonAttributes
public Mutable durability(Status.Durability durability)
{
+ Invariants.checkState(this != EMPTY_ATTRS && this != EMPTY_ATTRS_NONE_PARTIAL_DEPS, "Don't mutate the global empty instances");
this.durability = durability;
return this;
}
- @Override
- public RoutingKey homeKey()
- {
- return homeKey;
- }
-
- public Mutable homeKey(RoutingKey homeKey)
- {
- this.homeKey = homeKey;
- return this;
- }
-
- @Override
- public RoutingKey progressKey()
- {
- return progressKey;
- }
-
- public Mutable progressKey(RoutingKey progressKey)
- {
- Invariants.checkArgument(progressKey == null || progressKey.equals(progressKey));
- this.progressKey = progressKey;
- return this;
... 15446 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org