You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by be...@apache.org on 2022/06/28 09:41:39 UTC
[cassandra-accord] branch trunk updated: Liveness and Repair (#4)
This is an automated email from the ASF dual-hosted git repository.
benedict pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git
The following commit(s) were added to refs/heads/trunk by this push:
new 9671c16 Liveness and Repair (#4)
9671c16 is described below
commit 9671c16cd28ce58c1218f435273e920f4ae91ac9
Author: Benedict Elliott Smith <be...@apple.com>
AuthorDate: Tue Jun 28 10:41:35 2022 +0100
Liveness and Repair (#4)
Patch by Benedict; reviewed by Blake Eggleston for CASSANDRA-17096
Co-authored-by: Blake Eggleston <bl...@ultrablake.com>
---
accord-core/src/main/java/accord/api/Data.java | 2 +-
.../java/accord/api/{Store.java => DataStore.java} | 2 +-
accord-core/src/main/java/accord/api/Key.java | 1 -
.../src/main/java/accord/api/ProgressLog.java | 116 +++
accord-core/src/main/java/accord/api/Query.java | 2 +-
accord-core/src/main/java/accord/api/Read.java | 4 +-
.../src/main/java/accord/api/Scheduler.java | 2 +
accord-core/src/main/java/accord/api/Write.java | 2 +-
.../src/main/java/accord/coordinate/Agree.java | 40 +-
.../src/main/java/accord/coordinate/Agreed.java | 7 +-
.../java/accord/coordinate/CheckOnCommitted.java | 93 +++
.../java/accord/coordinate/CheckOnUncommitted.java | 67 ++
.../java/accord/coordinate/CheckShardStatus.java | 150 ++++
.../main/java/accord/coordinate/Coordinate.java | 14 +-
.../src/main/java/accord/coordinate/Execute.java | 33 +-
.../java/accord/coordinate/InformHomeOfTxn.java | 68 ++
.../main/java/accord/coordinate/MaybeRecover.java | 100 +++
.../src/main/java/accord/coordinate/Persist.java | 76 ++
.../coordinate/{AcceptPhase.java => Propose.java} | 23 +-
.../src/main/java/accord/coordinate/Recover.java | 88 ++-
.../main/java/accord/coordinate/StaleTopology.java | 5 +
.../coordinate/tracking/AbstractQuorumTracker.java | 24 +-
.../tracking/AbstractResponseTracker.java | 37 +-
.../coordinate/tracking/FastPathTracker.java | 2 +-
.../accord/coordinate/tracking/QuorumTracker.java | 5 +-
.../accord/coordinate/tracking/ReadTracker.java | 15 +-
.../main/java/accord/impl/SimpleProgressLog.java | 806 +++++++++++++++++++++
.../src/main/java/accord/local/Command.java | 256 ++++++-
.../src/main/java/accord/local/CommandStore.java | 149 ++--
.../src/main/java/accord/local/CommandStores.java | 521 ++++++++-----
accord-core/src/main/java/accord/local/Node.java | 213 ++++--
accord-core/src/main/java/accord/local/Status.java | 5 +
.../src/main/java/accord/messages/Accept.java | 27 +-
.../src/main/java/accord/messages/Apply.java | 55 +-
.../main/java/accord/messages/BeginRecovery.java | 90 ++-
.../src/main/java/accord/messages/Callback.java | 4 +
.../src/main/java/accord/messages/CheckStatus.java | 263 +++++++
.../src/main/java/accord/messages/Commit.java | 18 +-
.../main/java/accord/messages/EpochRequest.java | 6 +
.../java/accord/messages/InformOfPersistence.java | 56 ++
.../src/main/java/accord/messages/InformOfTxn.java | 124 ++++
.../src/main/java/accord/messages/MessageType.java | 11 +-
.../src/main/java/accord/messages/PreAccept.java | 67 +-
.../src/main/java/accord/messages/ReadData.java | 122 +---
.../src/main/java/accord/messages/TxnRequest.java | 221 ++++--
.../main/java/accord/messages/WaitOnCommit.java | 15 +-
.../java/accord/{api => topology}/KeyRange.java | 4 +-
.../src/main/java/accord/topology/KeyRanges.java | 129 ++--
.../src/main/java/accord/topology/Shard.java | 2 +-
.../src/main/java/accord/topology/Topologies.java | 70 +-
.../src/main/java/accord/topology/Topology.java | 35 +-
.../main/java/accord/topology/TopologyManager.java | 174 +++--
.../src/main/java/accord/txn/Dependencies.java | 48 +-
accord-core/src/main/java/accord/txn/Keys.java | 107 ++-
accord-core/src/main/java/accord/txn/Txn.java | 73 +-
accord-core/src/main/java/accord/txn/TxnId.java | 1 +
accord-core/src/main/java/accord/txn/Writes.java | 7 +-
.../accord/utils/DeterministicIdentitySet.java | 6 +
.../main/java/accord/utils/IndexedIntFunction.java | 6 +
.../java/accord/utils/IntrusiveLinkedList.java | 92 +++
.../java/accord/utils/IntrusiveLinkedListNode.java | 44 ++
accord-core/src/test/java/accord/KeysTest.java | 14 +-
accord-core/src/test/java/accord/Utils.java | 6 +-
.../src/test/java/accord/burn/BurnTest.java | 4 +-
.../src/test/java/accord/burn/TopologyUpdate.java | 37 +-
.../java/accord/coordinate/CoordinateTest.java | 8 +-
.../accord/coordinate/PreacceptTrackerTest.java | 9 +-
.../test/java/accord/coordinate/RecoverTest.java | 6 +-
.../java/accord/coordinate/TopologyChangeTest.java | 24 +-
.../coordinate/tracking/QuorumTrackerTest.java | 42 +-
.../src/test/java/accord/impl/IntHashKey.java | 2 +-
accord-core/src/test/java/accord/impl/IntKey.java | 2 +-
.../src/test/java/accord/impl/TopologyFactory.java | 2 +-
.../src/test/java/accord/impl/TopologyUtils.java | 2 +-
.../src/test/java/accord/impl/basic/Cluster.java | 14 +-
.../src/test/java/accord/impl/list/ListRead.java | 8 +-
.../test/java/accord/impl/list/ListRequest.java | 2 +-
.../src/test/java/accord/impl/list/ListStore.java | 4 +-
.../src/test/java/accord/impl/list/ListUpdate.java | 9 +
.../src/test/java/accord/impl/list/ListWrite.java | 6 +-
.../src/test/java/accord/impl/mock/EpochSync.java | 34 +-
.../test/java/accord/impl/mock/MockCluster.java | 4 +
.../src/test/java/accord/impl/mock/MockStore.java | 26 +-
.../accord/impl/mock/RecordingMessageSink.java | 5 +-
.../test/java/accord/local/CommandStoreTest.java | 92 +++
.../src/test/java/accord/local/CommandTest.java | 83 ++-
.../test/java/accord/messages/PreAcceptTest.java | 26 +-
.../java/accord/messages/TxnRequestScopeTest.java | 25 +-
.../java/accord/topology/TopologyManagerTest.java | 9 +-
.../java/accord/topology/TopologyRandomizer.java | 3 +-
.../test/java/accord/topology/TopologyTest.java | 1 -
.../src/test/java/accord/utils/KeyRangeTest.java | 2 +-
.../src/test/java/accord/utils/KeyRangesTest.java | 10 +-
.../verify/StrictSerializabilityVerifier.java | 4 +-
.../{burn-logback.xml => burn-logback-trace.xml} | 14 +
accord-core/src/test/resources/burn-logback.xml | 3 +
accord-maelstrom/build.gradle | 2 +-
.../src/main/java/accord/maelstrom/Cluster.java | 4 +-
.../src/main/java/accord/maelstrom/Json.java | 17 +-
.../main/java/accord/maelstrom/MaelstromKey.java | 2 +-
.../main/java/accord/maelstrom/MaelstromRead.java | 8 +-
.../java/accord/maelstrom/MaelstromRequest.java | 2 +-
.../main/java/accord/maelstrom/MaelstromStore.java | 4 +-
.../main/java/accord/maelstrom/MaelstromWrite.java | 4 +-
.../src/main/java/accord/maelstrom/Main.java | 4 +-
.../src/main/java/accord/maelstrom/Packet.java | 1 -
.../java/accord/maelstrom/TopologyFactory.java | 2 +-
build.gradle | 1 +
108 files changed, 4283 insertions(+), 1115 deletions(-)
diff --git a/accord-core/src/main/java/accord/api/Data.java b/accord-core/src/main/java/accord/api/Data.java
index 382712d..aeca36e 100644
--- a/accord-core/src/main/java/accord/api/Data.java
+++ b/accord-core/src/main/java/accord/api/Data.java
@@ -1,7 +1,7 @@
package accord.api;
/**
- * The result of some (potentially partial) {@link Read} from some {@link Store}
+ * The result of some (potentially partial) {@link Read} from some {@link DataStore}
*/
public interface Data
{
diff --git a/accord-core/src/main/java/accord/api/Store.java b/accord-core/src/main/java/accord/api/DataStore.java
similarity index 85%
rename from accord-core/src/main/java/accord/api/Store.java
rename to accord-core/src/main/java/accord/api/DataStore.java
index 898aa67..5e29db5 100644
--- a/accord-core/src/main/java/accord/api/Store.java
+++ b/accord-core/src/main/java/accord/api/DataStore.java
@@ -4,6 +4,6 @@ package accord.api;
* A marker interface for a shard instance's storage, that is passed to
* {@link Read} and {@link Write} objects for execution
*/
-public interface Store
+public interface DataStore
{
}
diff --git a/accord-core/src/main/java/accord/api/Key.java b/accord-core/src/main/java/accord/api/Key.java
index 85839ac..533d50c 100644
--- a/accord-core/src/main/java/accord/api/Key.java
+++ b/accord-core/src/main/java/accord/api/Key.java
@@ -5,7 +5,6 @@ package accord.api;
*/
public interface Key<K extends Key<K>> extends Comparable<K>
{
-
/**
* Returns a hash code of a key to support accord internal sharding. Hash values for equal keys must be equal.
*/
diff --git a/accord-core/src/main/java/accord/api/ProgressLog.java b/accord-core/src/main/java/accord/api/ProgressLog.java
new file mode 100644
index 0000000..80964c3
--- /dev/null
+++ b/accord-core/src/main/java/accord/api/ProgressLog.java
@@ -0,0 +1,116 @@
+package accord.api;
+
+import java.util.Set;
+
+import accord.coordinate.CheckOnUncommitted;
+import accord.coordinate.InformHomeOfTxn;
+import accord.local.CommandStore;
+import accord.local.Node.Id;
+import accord.txn.TxnId;
+
+/**
+ * This interface is responsible for managing incomplete transactions *and retrying them*.
+ * Each stage is fenced by two methods, one entry and one exit. The entry method notifies the implementation
+ * that it should soon be notified of the exit method, and if it is not that it should invoke some
+ * pre-specified recovery mechanism.
+ *
+ * This is a per-CommandStore structure, with transactions primarily being managed by their home shard,
+ * except during PreAccept as a transaction may not yet have been durably recorded by the home shard.
+ *
+ * The basic logical flow for ensuring a transaction is committed and applied at all replicas is as follows:
+ *
+ * - First, ensure a quorum of the home shard is aware of the transaction by invoking {@link InformHomeOfTxn}.
+ * Entry by {@link #preaccept} Non-home shards may now forget this transaction for replay purposes.
+ *
+ * - Non-home shards may also be informed of transactions that are blocking the progress of other transactions.
+ * If the {@code waitingOn} transaction that is blocking progress is uncommitted it is required that the progress
+ * log invoke {@link CheckOnUncommitted} for the transaction if no {@link #commit} is witnessed.
+ *
+ * - Members of the home shard will be informed of a transaction to monitor by the invocation of {@link #preaccept} or
+ * {@link #accept}. If this is not followed closely by {@link #commit}, {@link accord.coordinate.MaybeRecover} should
+ * be invoked.
+ *
+ * - Members of the home shard will later be informed that the transaction is {@link #readyToExecute}.
+ * If this is not followed closely by {@link #execute}, {@link accord.coordinate.MaybeRecover} should be invoked.
+ *
+ * - Finally, it is up to each shard to independently coordinate disseminating the write to every replica.
+ */
+public interface ProgressLog
+{
+ interface Factory
+ {
+ ProgressLog create(CommandStore store);
+ }
+
+ /**
+ * Has been pre-accepted.
+ *
+ * A home shard should monitor this transaction for global progress.
+ * A non-home shard should begin monitoring this transaction only to ensure it reaches the Accept phase, or is
+ * witnessed by a majority of the home shard.
+ */
+ void preaccept(TxnId txnId, boolean isProgressShard, boolean isHomeShard);
+
+ /**
+ * Has been accepted
+ *
+ * A home shard should monitor this transaction for global progress.
+ * A non-home shard can safely ignore this transaction, as it has been witnessed by a majority of the home shard.
+ */
+ void accept(TxnId txnId, boolean isProgressShard, boolean isHomeShard);
+
+ /**
+ * Has committed
+ *
+ * A home shard should monitor this transaction for global progress.
+ * A non-home shard can safely ignore this transaction, as it has been witnessed by a majority of the home shard.
+ */
+ void commit(TxnId txnId, boolean isProgressShard, boolean isHomeShard);
+
+ /**
+ * The transaction is waiting to make progress, as all local dependencies have applied.
+ *
+ * A home shard should monitor this transaction for global progress.
+ * A non-home shard can safely ignore this transaction, as it has been witnessed by a majority of the home shard.
+ */
+ void readyToExecute(TxnId txnId, boolean isProgressShard, boolean isHomeShard);
+
+ /**
+ * The transaction's outcome has been durably recorded (but not necessarily applied) locally.
+ * It will be applied once all local dependencies have been.
+ *
+ * Invoked on both home and non-home command stores, and is required to trigger per-shard processes
+ * that ensure the transaction's outcome is durably persisted on all replicas of the shard.
+ *
+ * May also permit aborting a pending waitingOn-triggered event.
+ */
+ void execute(TxnId txnId, boolean isProgressShard, boolean isHomeShard);
+
+ /**
+ * The transaction's outcome has been durably recorded (but not necessarily applied) at a quorum of all shards.
+ *
+ * If this replica has not witnessed the outcome of the transaction, it should poll a majority of the home shard
+ * for its outcome.
+ *
+ * Otherwise, this transaction no longer needs to be monitored by either home or non-home shards.
+ */
+ void executedOnAllShards(TxnId txnId, Set<Id> persistedOn);
+
+ /**
+ * The parameter is a command that some other command's execution is most proximally blocked by.
+ * This may be invoked by either the home or non-home command store.
+ *
+ * If invoked by the non-home command store for a {@code blockedBy} transaction that has not yet been committed, this
+ * must eventually trigger contact with the home shard of this {@code blockedBy} transaction in order to check on the
+ * transaction's progress (unless the transaction is committed first). This is to avoid unnecessary additional messages
+ * being exchanged in the common case, where a transaction may be committed successfully to members of its home shard,
+ * but not to all non-home shards. In such a case the transaction may be a false-dependency of another transaction that
+ * needs to perform a read, and all nodes which may do so are waiting for the commit record to arrive.
+ *
+ * If a quorum of the home shard does not know of the transaction, then we can ask the home shard to perform recovery
+ * to either complete or invalidate it, so that we may make progress.
+ *
+ * In all other scenarios, the implementation is free to choose its course of action.
+ */
+ void waiting(TxnId blockedBy, Key homeKey);
+}
diff --git a/accord-core/src/main/java/accord/api/Query.java b/accord-core/src/main/java/accord/api/Query.java
index da68d87..158ff1c 100644
--- a/accord-core/src/main/java/accord/api/Query.java
+++ b/accord-core/src/main/java/accord/api/Query.java
@@ -7,7 +7,7 @@ public interface Query
{
/**
* Perform some transformation on the complete {@link Data} result of a {@link Read}
- * from some {@link Store}, to produce a {@link Result} to return to the client.
+ * from some {@link DataStore}, to produce a {@link Result} to return to the client.
*/
Result compute(Data data);
}
diff --git a/accord-core/src/main/java/accord/api/Read.java b/accord-core/src/main/java/accord/api/Read.java
index 0823302..a14ce3a 100644
--- a/accord-core/src/main/java/accord/api/Read.java
+++ b/accord-core/src/main/java/accord/api/Read.java
@@ -1,5 +1,6 @@
package accord.api;
+import accord.txn.Keys;
import accord.txn.Timestamp;
/**
@@ -9,5 +10,6 @@ import accord.txn.Timestamp;
*/
public interface Read
{
- Data read(Key key, Timestamp executeAt, Store store);
+ Keys keys();
+ Data read(Key key, Timestamp executeAt, DataStore store);
}
diff --git a/accord-core/src/main/java/accord/api/Scheduler.java b/accord-core/src/main/java/accord/api/Scheduler.java
index 8af6bcf..64a38bb 100644
--- a/accord-core/src/main/java/accord/api/Scheduler.java
+++ b/accord-core/src/main/java/accord/api/Scheduler.java
@@ -12,7 +12,9 @@ public interface Scheduler
void cancel();
}
+ default Scheduled recurring(Runnable run) { return recurring(run, 1L, TimeUnit.SECONDS); }
Scheduled recurring(Runnable run, long delay, TimeUnit units);
+
Scheduled once(Runnable run, long delay, TimeUnit units);
void now(Runnable run);
}
diff --git a/accord-core/src/main/java/accord/api/Write.java b/accord-core/src/main/java/accord/api/Write.java
index 03d73c1..2784139 100644
--- a/accord-core/src/main/java/accord/api/Write.java
+++ b/accord-core/src/main/java/accord/api/Write.java
@@ -9,5 +9,5 @@ import accord.txn.Timestamp;
*/
public interface Write
{
- void apply(Key key, Timestamp executeAt, Store store);
+ void apply(Key key, Timestamp executeAt, DataStore store);
}
diff --git a/accord-core/src/main/java/accord/coordinate/Agree.java b/accord-core/src/main/java/accord/coordinate/Agree.java
index eb91821..3054e7a 100644
--- a/accord-core/src/main/java/accord/coordinate/Agree.java
+++ b/accord-core/src/main/java/accord/coordinate/Agree.java
@@ -5,6 +5,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import accord.api.Key;
import accord.coordinate.tracking.FastPathTracker;
import accord.topology.Shard;
import accord.topology.Topologies;
@@ -27,7 +28,7 @@ import org.apache.cassandra.utils.concurrent.Future;
* Perform initial rounds of PreAccept and Accept until we have reached agreement about when we should execute.
* If we are preempted by a recovery coordinator, we abort and let them complete (and notify us about the execution result)
*/
-class Agree extends AcceptPhase implements Callback<PreAcceptReply>
+class Agree extends Propose implements Callback<PreAcceptReply>
{
static class ShardTracker extends FastPathTracker.FastPathShardTracker
{
@@ -68,12 +69,12 @@ class Agree extends AcceptPhase implements Callback<PreAcceptReply>
}
@Override
- public void recordFailure(Id node)
+ public boolean failure(Id node)
{
if (failures == null)
failures = new HashSet<>();
failures.add(node);
- super.recordFailure(node);
+ return super.failure(node);
}
@Override
@@ -106,7 +107,7 @@ class Agree extends AcceptPhase implements Callback<PreAcceptReply>
PreacceptTracker tracker = new PreacceptTracker(topologies, false);
successes.forEach(tracker::recordSuccess);
if (failures != null)
- failures.forEach(tracker::recordFailure);
+ failures.forEach(tracker::failure);
return tracker;
}
@@ -134,14 +135,14 @@ class Agree extends AcceptPhase implements Callback<PreAcceptReply>
// TODO: hybrid fast path? or at least short-circuit accept if we gain a fast-path quorum _and_ proposed one by accept
boolean permitHybridFastPath;
- private Agree(Node node, TxnId txnId, Txn txn)
+ private Agree(Node node, TxnId txnId, Txn txn, Key homeKey)
{
- super(node, Ballot.ZERO, txnId, txn);
+ super(node, Ballot.ZERO, txnId, txn, homeKey);
this.keys = txn.keys();
- tracker = new PreacceptTracker(node.topology().forKeys(txn.keys(), txnId.epoch));
+ tracker = new PreacceptTracker(node.topology().withUnsyncedEpochs(txn.keys(), txnId.epoch, txnId.epoch));
// TODO: consider sending only to electorate of most recent topology (as only these PreAccept votes matter)
// note that we must send to all replicas of old topology, as electorate may not be reachable
- node.send(tracker.nodes(), to -> new PreAccept(to, tracker.topologies(), txnId, txn), this);
+ node.send(tracker.nodes(), to -> new PreAccept(to, tracker.topologies(), txnId, txn, homeKey), this);
}
@Override
@@ -156,8 +157,7 @@ class Agree extends AcceptPhase implements Callback<PreAcceptReply>
if (isDone() || isPreAccepted())
return;
- tracker.recordFailure(from);
- if (tracker.hasFailed())
+ if (tracker.failure(from))
tryFailure(new Timeout());
// if no other responses are expected and the slow quorum has been satisfied, proceed
@@ -169,7 +169,7 @@ class Agree extends AcceptPhase implements Callback<PreAcceptReply>
{
if (!tracker.hasSupersedingEpoch())
return;
- Topologies newTopologies = node.topology().forKeys(txn.keys(), txnId.epoch);
+ Topologies newTopologies = node.topology().withUnsyncedEpochs(txn.keys(), txnId.epoch, tracker.supersedingEpoch);
if (newTopologies.currentEpoch() < tracker.supersedingEpoch)
return;
Set<Id> previousNodes = tracker.nodes();
@@ -178,7 +178,7 @@ class Agree extends AcceptPhase implements Callback<PreAcceptReply>
// send messages to new nodes
Set<Id> needMessages = Sets.difference(tracker.nodes(), previousNodes);
if (!needMessages.isEmpty())
- node.send(needMessages, to -> new PreAccept(to, newTopologies, txnId, txn), this);
+ node.send(needMessages, to -> new PreAccept(to, newTopologies, txnId, txn, homeKey), this);
if (tracker.shouldSlowPathAccept())
onPreAccepted();
@@ -202,13 +202,11 @@ class Agree extends AcceptPhase implements Callback<PreAcceptReply>
boolean fastPath = ok.witnessedAt.compareTo(txnId) == 0;
tracker.recordSuccess(from, fastPath);
- if (!fastPath && ok.witnessedAt.epoch > txnId.epoch)
+ // TODO: we should only update epoch if we need to in order to reach quorum
+ if (!fastPath && ok.witnessedAt.epoch > txnId.epoch && tracker.recordSupersedingEpoch(ok.witnessedAt.epoch))
{
- if (tracker.recordSupersedingEpoch(ok.witnessedAt.epoch))
- {
- node.configService().fetchTopologyForEpoch(ok.witnessedAt.epoch);
- node.topology().awaitEpoch(ok.witnessedAt.epoch).addListener(this::onEpochUpdate);
- }
+ node.configService().fetchTopologyForEpoch(ok.witnessedAt.epoch);
+ node.topology().awaitEpoch(ok.witnessedAt.epoch).addListener(this::onEpochUpdate);
}
if (!tracker.hasSupersedingEpoch() && (tracker.hasMetFastPathCriteria() || tracker.shouldSlowPathAccept()))
@@ -226,7 +224,7 @@ class Agree extends AcceptPhase implements Callback<PreAcceptReply>
if (preAcceptOk.witnessedAt.equals(txnId))
deps.addAll(preAcceptOk.deps);
}
- agreed(txnId, deps, tracker.topologies());
+ agreed(txnId, deps);
}
else
{
@@ -252,8 +250,8 @@ class Agree extends AcceptPhase implements Callback<PreAcceptReply>
return preacceptOutcome != null;
}
- static Future<Agreed> agree(Node node, TxnId txnId, Txn txn)
+ static Future<Agreed> agree(Node node, TxnId txnId, Txn txn, Key homeKey)
{
- return new Agree(node, txnId, txn);
+ return new Agree(node, txnId, txn, homeKey);
}
}
diff --git a/accord-core/src/main/java/accord/coordinate/Agreed.java b/accord-core/src/main/java/accord/coordinate/Agreed.java
index c8e6d61..640f428 100644
--- a/accord-core/src/main/java/accord/coordinate/Agreed.java
+++ b/accord-core/src/main/java/accord/coordinate/Agreed.java
@@ -1,5 +1,6 @@
package accord.coordinate;
+import accord.api.Key;
import accord.api.Result;
import accord.topology.Topologies;
import accord.txn.Writes;
@@ -12,19 +13,19 @@ class Agreed
{
public final TxnId txnId;
public final Txn txn;
+ public final Key homeKey;
public final Timestamp executeAt;
public final Dependencies deps;
- public final Topologies topologies;
public final Writes applied;
public final Result result;
- public Agreed(TxnId txnId, Txn txn, Timestamp executeAt, Dependencies deps, Topologies topologies, Writes applied, Result result)
+ public Agreed(TxnId txnId, Txn txn, Key homeKey, Timestamp executeAt, Dependencies deps, Writes applied, Result result)
{
this.txnId = txnId;
this.txn = txn;
+ this.homeKey = homeKey;
this.executeAt = executeAt;
this.deps = deps;
- this.topologies = topologies;
this.applied = applied;
this.result = result;
}
diff --git a/accord-core/src/main/java/accord/coordinate/CheckOnCommitted.java b/accord-core/src/main/java/accord/coordinate/CheckOnCommitted.java
new file mode 100644
index 0000000..470a7c6
--- /dev/null
+++ b/accord-core/src/main/java/accord/coordinate/CheckOnCommitted.java
@@ -0,0 +1,93 @@
+package accord.coordinate;
+
+import accord.api.Key;
+import accord.local.Command;
+import accord.local.Node;
+import accord.messages.CheckStatus.CheckStatusOkFull;
+import accord.messages.CheckStatus.IncludeInfo;
+import accord.topology.Shard;
+import accord.txn.Txn;
+import accord.txn.TxnId;
+
+import static accord.local.Status.Executed;
+
+/**
+ * Check on the status of a locally-uncommitted transaction. Returns early if any result indicates Committed, otherwise
+ * waits only for a quorum and returns the maximum result.
+ *
+ * Updates local command stores based on the obtained information.
+ */
+public class CheckOnCommitted extends CheckShardStatus
+{
+ CheckOnCommitted(Node node, TxnId txnId, Txn txn, Key homeKey, Shard homeShard, long homeEpoch)
+ {
+ super(node, txnId, txn, homeKey, homeShard, homeEpoch, IncludeInfo.Always);
+ }
+
+ public static CheckOnCommitted checkOnCommitted(Node node, TxnId txnId, Txn txn, Key someKey, Shard someShard, long shardEpoch)
+ {
+ CheckOnCommitted checkOnCommitted = new CheckOnCommitted(node, txnId, txn, someKey, someShard, shardEpoch);
+ checkOnCommitted.start();
+ return checkOnCommitted;
+ }
+
+ @Override
+ boolean hasMetSuccessCriteria()
+ {
+ return tracker.hasReachedQuorum() || hasApplied();
+ }
+
+ public boolean hasApplied()
+ {
+ return max != null && max.status.hasBeen(Executed);
+ }
+
+ void onSuccessCriteriaOrExhaustion(CheckStatusOkFull max)
+ {
+ switch (max.status)
+ {
+ case NotWitnessed:
+ case PreAccepted:
+ case Accepted:
+ return;
+ }
+
+ Key progressKey = node.trySelectProgressKey(txnId, txn.keys, max.homeKey);
+ switch (max.status)
+ {
+ default: throw new IllegalStateException();
+ case Executed:
+ case Applied:
+ node.forEachLocalSince(txn.keys, max.executeAt.epoch, commandStore -> {
+ Command command = commandStore.command(txnId);
+ command.apply(txn, max.homeKey, progressKey, max.executeAt, max.deps, max.writes, max.result);
+ });
+ node.forEachLocal(txn.keys, txnId.epoch, max.executeAt.epoch - 1, commandStore -> {
+ Command command = commandStore.command(txnId);
+ command.commit(txn, max.homeKey, progressKey, max.executeAt, max.deps);
+ });
+ break;
+ case Committed:
+ case ReadyToExecute:
+ node.forEachLocalSince(txn.keys, txnId.epoch, commandStore -> {
+ Command command = commandStore.command(txnId);
+ command.commit(txn, max.homeKey, progressKey, max.executeAt, max.deps);
+ });
+ }
+ }
+
+ @Override
+ void onSuccessCriteriaOrExhaustion()
+ {
+ try
+ {
+ onSuccessCriteriaOrExhaustion((CheckStatusOkFull) max);
+ }
+ catch (Throwable t)
+ {
+ trySuccess(max);
+ throw t;
+ }
+ trySuccess(max);
+ }
+}
diff --git a/accord-core/src/main/java/accord/coordinate/CheckOnUncommitted.java b/accord-core/src/main/java/accord/coordinate/CheckOnUncommitted.java
new file mode 100644
index 0000000..3ddcf4b
--- /dev/null
+++ b/accord-core/src/main/java/accord/coordinate/CheckOnUncommitted.java
@@ -0,0 +1,67 @@
+package accord.coordinate;
+
+import accord.api.Key;
+import accord.local.Command;
+import accord.local.Node;
+import accord.messages.CheckStatus.CheckStatusOkFull;
+import accord.topology.Shard;
+import accord.txn.Txn;
+import accord.txn.TxnId;
+
+import static accord.local.Status.Committed;
+
+/**
+ * Check on the status of a locally-uncommitted transaction. Returns early if any result indicates Committed, otherwise
+ * waits only for a quorum and returns the maximum result.
+ *
+ * Updates local command stores based on the obtained information.
+ */
+public class CheckOnUncommitted extends CheckOnCommitted
+{
+ CheckOnUncommitted(Node node, TxnId txnId, Txn txn, Key someKey, Shard someShard, long shardEpoch)
+ {
+ super(node, txnId, txn, someKey, someShard, shardEpoch);
+ }
+
+ public static CheckOnUncommitted checkOnUncommitted(Node node, TxnId txnId, Txn txn, Key someKey, Shard someShard, long shardEpoch)
+ {
+ CheckOnUncommitted checkOnUncommitted = new CheckOnUncommitted(node, txnId, txn, someKey, someShard, shardEpoch);
+ checkOnUncommitted.start();
+ return checkOnUncommitted;
+ }
+
+ @Override
+ boolean hasMetSuccessCriteria()
+ {
+ return tracker.hasReachedQuorum() || hasCommitted();
+ }
+
+ public boolean hasCommitted()
+ {
+ return max != null && max.status.hasBeen(Committed);
+ }
+
+ @Override
+ void onSuccessCriteriaOrExhaustion(CheckStatusOkFull full)
+ {
+ switch (full.status)
+ {
+ default: throw new IllegalStateException();
+ case NotWitnessed:
+ break;
+ case PreAccepted:
+ case Accepted:
+ node.forEachLocalSince(txn.keys, txnId.epoch, commandStore -> {
+ Command command = commandStore.ifPresent(txnId);
+ if (command != null)
+ command.homeKey(full.homeKey);
+ });
+ break;
+ case Executed:
+ case Applied:
+ case Committed:
+ case ReadyToExecute:
+ super.onSuccessCriteriaOrExhaustion(full);
+ }
+ }
+}
diff --git a/accord-core/src/main/java/accord/coordinate/CheckShardStatus.java b/accord-core/src/main/java/accord/coordinate/CheckShardStatus.java
new file mode 100644
index 0000000..a5f1108
--- /dev/null
+++ b/accord-core/src/main/java/accord/coordinate/CheckShardStatus.java
@@ -0,0 +1,150 @@
+package accord.coordinate;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import com.google.common.base.Preconditions;
+
+import accord.api.Key;
+import accord.coordinate.tracking.ReadTracker.ReadShardTracker;
+import accord.local.Node;
+import accord.local.Node.Id;
+import accord.messages.Callback;
+import accord.messages.CheckStatus;
+import accord.messages.CheckStatus.CheckStatusOk;
+import accord.messages.CheckStatus.CheckStatusReply;
+import accord.messages.CheckStatus.IncludeInfo;
+import accord.topology.Shard;
+import accord.txn.Txn;
+import accord.txn.TxnId;
+import org.apache.cassandra.utils.concurrent.AsyncFuture;
+
+/**
+ * A result of null indicates the transaction is globally persistent
+ * A result of CheckStatusOk indicates the maximum status found for the transaction, which may be used to assess progress
+ */
+public abstract class CheckShardStatus extends AsyncFuture<CheckStatusOk> implements Callback<CheckStatusReply>
+{
+ static class Tracker extends ReadShardTracker
+ {
+ private int successCount;
+ public Tracker(Shard shard)
+ {
+ super(shard);
+ }
+
+ @Override
+ public boolean recordReadSuccess(Id node)
+ {
+ if (!super.recordReadSuccess(node))
+ return false;
+
+ successCount++;
+ return true;
+ }
+
+ public boolean hasReachedQuorum()
+ {
+ return successCount >= shard.slowPathQuorumSize;
+ }
+
+ public boolean hasInFlight()
+ {
+ return !inflight.isEmpty();
+ }
+ }
+
+ final Node node;
+ final TxnId txnId;
+ final Txn txn;
+ final Key key; // not necessarily homeKey
+ final Tracker tracker;
+ final List<Id> candidates;
+ final long epoch;
+ final IncludeInfo includeInfo;
+
+ CheckStatusOk max;
+ Throwable failure;
+
+ CheckShardStatus(Node node, TxnId txnId, Txn txn, Key key, Shard shard, long epoch, IncludeInfo includeInfo)
+ {
+ this.epoch = epoch;
+ Preconditions.checkNotNull(txn);
+ Preconditions.checkState(txn.keys.contains(key));
+ this.txnId = txnId;
+ this.txn = txn;
+ this.key = key;
+ this.tracker = new Tracker(shard);
+ this.candidates = new ArrayList<>(shard.nodes);
+ this.node = node;
+ this.includeInfo = includeInfo;
+ }
+
+ @Override
+ public void onSuccess(Id from, CheckStatusReply response)
+ {
+ if (response.isOk())
+ {
+ onOk((CheckStatusOk) response);
+ if (tracker.recordReadSuccess(from))
+ {
+ if (hasMetSuccessCriteria())
+ onSuccessCriteriaOrExhaustion();
+ else if (!tracker.hasInFlight() && hasMoreCandidates())
+ sendMore();
+ else
+ onSuccessCriteriaOrExhaustion();
+ }
+ }
+ else
+ {
+ onFailure(from, new StaleTopology());
+ }
+ }
+
+ @Override
+ public void onFailure(Id from, Throwable throwable)
+ {
+ if (failure == null) failure = throwable;
+ else failure.addSuppressed(throwable);
+
+ // TODO: if we fail and have an incorrect topology, trigger refresh
+ if (tracker.recordReadFailure(from))
+ {
+ if (tracker.hasFailed())
+ tryFailure(failure);
+ else if (!tracker.hasInFlight() && hasMoreCandidates())
+ sendMore();
+ else
+ onSuccessCriteriaOrExhaustion();
+ }
+ }
+
+ private void onOk(CheckStatusOk ok)
+ {
+ if (max == null) max = ok;
+ else max = max.merge(ok);
+ }
+
+ protected void start()
+ {
+ sendMore();
+ }
+
+ private void sendMore()
+ {
+ // TODO: send to local nodes first, and send in batches (local then remote)
+ Id next = candidates.get(candidates.size() - 1);
+ candidates.remove(candidates.size() - 1);
+ tracker.recordInflightRead(next);
+ node.send(next, new CheckStatus(txnId, key, epoch, includeInfo), this);
+ }
+
+ private boolean hasMoreCandidates()
+ {
+ return !candidates.isEmpty();
+ }
+
+ abstract boolean hasMetSuccessCriteria();
+ abstract void onSuccessCriteriaOrExhaustion();
+}
diff --git a/accord-core/src/main/java/accord/coordinate/Coordinate.java b/accord-core/src/main/java/accord/coordinate/Coordinate.java
index c68d5ba..73ca784 100644
--- a/accord-core/src/main/java/accord/coordinate/Coordinate.java
+++ b/accord-core/src/main/java/accord/coordinate/Coordinate.java
@@ -1,6 +1,11 @@
package accord.coordinate;
import accord.api.ConfigurationService;
+
+import com.google.common.base.Preconditions;
+
+import accord.api.Key;
+import accord.local.Node;
import accord.api.Result;
import accord.local.Node;
import accord.txn.Ballot;
@@ -28,13 +33,14 @@ public class Coordinate
return agree.flatMap(agreed -> fetchEpochOrExecute(node, agreed));
}
- public static Future<Result> execute(Node node, TxnId txnId, Txn txn)
+ public static Future<Result> execute(Node node, TxnId txnId, Txn txn, Key homeKey)
{
- return andThenExecute(node, Agree.agree(node, txnId, txn));
+ Preconditions.checkArgument(node.isReplicaOf(txnId, homeKey));
+ return andThenExecute(node, Agree.agree(node, txnId, txn, homeKey));
}
- public static Future<Result> recover(Node node, TxnId txnId, Txn txn)
+ public static Future<Result> recover(Node node, TxnId txnId, Txn txn, Key homeKey)
{
- return andThenExecute(node, new Recover(node, new Ballot(node.uniqueNow()), txnId, txn));
+ return andThenExecute(node, new Recover(node, new Ballot(node.uniqueNow()), txnId, txn, homeKey));
}
}
diff --git a/accord-core/src/main/java/accord/coordinate/Execute.java b/accord-core/src/main/java/accord/coordinate/Execute.java
index 2b6d09b..8f5767e 100644
--- a/accord-core/src/main/java/accord/coordinate/Execute.java
+++ b/accord-core/src/main/java/accord/coordinate/Execute.java
@@ -3,15 +3,15 @@ package accord.coordinate;
import java.util.Set;
import accord.api.Data;
+import accord.api.Key;
import accord.coordinate.tracking.ReadTracker;
import accord.api.Result;
import accord.messages.Callback;
import accord.local.Node;
import accord.topology.Topologies;
import accord.txn.*;
-import accord.messages.Apply;
import accord.messages.ReadData.ReadReply;
-import accord.messages.ReadData.ReadWaiting;
+import accord.txn.Dependencies;
import accord.local.Node.Id;
import accord.messages.Commit;
import accord.messages.ReadData;
@@ -24,6 +24,7 @@ class Execute extends AsyncPromise<Result> implements Callback<ReadReply>
final Node node;
final TxnId txnId;
final Txn txn;
+ final Key homeKey;
final Timestamp executeAt;
final Topologies topologies;
final Keys keys;
@@ -36,26 +37,28 @@ class Execute extends AsyncPromise<Result> implements Callback<ReadReply>
this.node = node;
this.txnId = agreed.txnId;
this.txn = agreed.txn;
+ this.homeKey = agreed.homeKey;
this.keys = txn.keys();
this.deps = agreed.deps;
this.executeAt = agreed.executeAt;
- this.topologies = node.topology().forTxn(agreed.txn).removeEpochsBefore(agreed.executeAt.epoch);
- this.readTracker = new ReadTracker(topologies.removeEpochsBefore(topologies.currentEpoch()));
+ Topologies coordinationTopologies = node.topology().unsyncForTxn(agreed.txn, agreed.executeAt.epoch);
+ Topologies readTopologies = node.topology().preciseEpochs(agreed.txn.read.keys(), agreed.executeAt.epoch);
+ this.readTracker = new ReadTracker(readTopologies);
+ this.topologies = coordinationTopologies;
// TODO: perhaps compose these different behaviours differently?
if (agreed.applied != null)
{
- node.send(topologies.nodes(),
- to -> new Apply(to, topologies, txnId, txn, executeAt, agreed.deps, agreed.applied, agreed.result));
setSuccess(agreed.result);
+ Persist.persist(node, topologies, txnId, homeKey, txn, executeAt, deps, agreed.applied, agreed.result);
}
else
{
Set<Id> readSet = readTracker.computeMinimalReadSetAndMarkInflight();
- for (Node.Id to : readTracker.nodes())
+ for (Node.Id to : coordinationTopologies.nodes())
{
boolean read = readSet.contains(to);
- Commit send = new Commit(to, topologies, txnId, txn, executeAt, agreed.deps, read);
+ Commit send = new Commit(to, topologies, txnId, txn, homeKey, executeAt, agreed.deps, read);
if (read)
{
node.send(to, send, this);
@@ -64,7 +67,6 @@ class Execute extends AsyncPromise<Result> implements Callback<ReadReply>
{
node.send(to, send);
}
-
}
}
}
@@ -76,13 +78,7 @@ class Execute extends AsyncPromise<Result> implements Callback<ReadReply>
return;
if (!reply.isFinal())
- {
- ReadWaiting waiting = (ReadWaiting) reply;
- // TODO first see if we can collect newer information (from ourselves or others), and if so send it
- // otherwise, try to complete the transaction
- node.recover(waiting.txnId, waiting.txn);
return;
- }
if (!reply.isOK())
{
@@ -98,9 +94,8 @@ class Execute extends AsyncPromise<Result> implements Callback<ReadReply>
if (readTracker.hasCompletedRead())
{
Result result = txn.result(data);
- Writes writes = txn.execute(executeAt, data);
- node.send(topologies.nodes(), to -> new Apply(to, topologies, txnId, txn, executeAt, deps, writes, result));
setSuccess(result);
+ Persist.persist(node, topologies, txnId, homeKey, txn, executeAt, deps, txn.execute(executeAt, data), result);
}
}
@@ -112,11 +107,13 @@ class Execute extends AsyncPromise<Result> implements Callback<ReadReply>
if (!(throwable instanceof Timeout))
throwable.printStackTrace();
+ // TODO: introduce two tiers of timeout, one to trigger a retry, and another to mark the original as failed
+ // TODO: if we fail, nominate another coordinator from the homeKey shard to try
readTracker.recordReadFailure(from);
Set<Id> readFrom = readTracker.computeMinimalReadSetAndMarkInflight();
if (readFrom != null)
{
- node.send(readFrom, to -> new ReadData(to, topologies, txnId, txn, executeAt), this);
+ node.send(readFrom, to -> new ReadData(to, readTracker.topologies(), txnId, txn, homeKey, executeAt), this);
}
else if (readTracker.hasFailed())
{
diff --git a/accord-core/src/main/java/accord/coordinate/InformHomeOfTxn.java b/accord-core/src/main/java/accord/coordinate/InformHomeOfTxn.java
new file mode 100644
index 0000000..fdda857
--- /dev/null
+++ b/accord-core/src/main/java/accord/coordinate/InformHomeOfTxn.java
@@ -0,0 +1,68 @@
+package accord.coordinate;
+
+import accord.api.Key;
+import accord.coordinate.tracking.AbstractQuorumTracker.QuorumShardTracker;
+import accord.local.Node;
+import accord.local.Node.Id;
+import accord.messages.Callback;
+import accord.messages.InformOfTxn;
+import accord.messages.InformOfTxn.InformOfTxnReply;
+import accord.topology.Shard;
+import accord.txn.Txn;
+import accord.txn.TxnId;
+import org.apache.cassandra.utils.concurrent.AsyncFuture;
+import org.apache.cassandra.utils.concurrent.Future;
+
+public class InformHomeOfTxn extends AsyncFuture<Void> implements Callback<InformOfTxnReply>
+{
+ final TxnId txnId;
+ final Key homeKey;
+ final QuorumShardTracker tracker;
+ Throwable failure;
+
+ InformHomeOfTxn(TxnId txnId, Key homeKey, Shard homeShard)
+ {
+ this.txnId = txnId;
+ this.homeKey = homeKey;
+ this.tracker = new QuorumShardTracker(homeShard);
+ }
+
+ public static Future<Void> inform(Node node, TxnId txnId, Txn txn, Key homeKey)
+ {
+ // TODO: we should not need to send the Txn here, but to avoid that we need to support no-ops
+ Shard homeShard = node.topology().forEpochIfKnown(homeKey, txnId.epoch);
+ if (homeShard == null)
+ {
+ node.configService().fetchTopologyForEpoch(txnId.epoch);
+ return node.topology().awaitEpoch(txnId.epoch).flatMap(ignore -> inform(node, txnId, txn, homeKey));
+ }
+ InformHomeOfTxn inform = new InformHomeOfTxn(txnId, homeKey, homeShard);
+ node.send(homeShard.nodes, new InformOfTxn(txnId, homeKey, txn), inform);
+ return inform;
+ }
+
+ @Override
+ public void onSuccess(Id from, InformOfTxnReply response)
+ {
+ if (response.isOk())
+ {
+ if (tracker.success(from))
+ trySuccess(null);
+ }
+ else
+ {
+ onFailure(from, new StaleTopology());
+ }
+ }
+
+ @Override
+ public void onFailure(Id from, Throwable throwable)
+ {
+ if (failure == null) failure = throwable;
+ else failure.addSuppressed(throwable);
+
+ // TODO: if we fail and have an incorrect topology, trigger refresh
+ if (tracker.failure(from))
+ tryFailure(failure);
+ }
+}
diff --git a/accord-core/src/main/java/accord/coordinate/MaybeRecover.java b/accord-core/src/main/java/accord/coordinate/MaybeRecover.java
new file mode 100644
index 0000000..38a5b3a
--- /dev/null
+++ b/accord-core/src/main/java/accord/coordinate/MaybeRecover.java
@@ -0,0 +1,100 @@
+package accord.coordinate;
+
+import java.util.function.BiConsumer;
+
+import accord.api.Key;
+import accord.local.Node;
+import accord.local.Status;
+import accord.messages.CheckStatus.CheckStatusOk;
+import accord.messages.CheckStatus.CheckStatusOkFull;
+import accord.messages.CheckStatus.IncludeInfo;
+import accord.topology.Shard;
+import accord.txn.Ballot;
+import accord.txn.Txn;
+import accord.txn.TxnId;
+import org.apache.cassandra.utils.concurrent.Future;
+
+import static accord.local.Status.Accepted;
+
+/**
+ * A result of null indicates the transaction is globally persistent
+ * A result of CheckStatusOk indicates the maximum status found for the transaction, which may be used to assess progress
+ */
+public class MaybeRecover extends CheckShardStatus implements BiConsumer<Object, Throwable>
+{
+ final Status knownStatus;
+ final Ballot knownPromised;
+ final boolean knownPromisedHasBeenAccepted;
+
+ MaybeRecover(Node node, TxnId txnId, Txn txn, Key homeKey, Shard homeShard, long homeEpoch, Status knownStatus, Ballot knownPromised, boolean knownPromiseHasBeenAccepted)
+ {
+ super(node, txnId, txn, homeKey, homeShard, homeEpoch, IncludeInfo.OnlyIfExecuted);
+ this.knownStatus = knownStatus;
+ this.knownPromised = knownPromised;
+ this.knownPromisedHasBeenAccepted = knownPromiseHasBeenAccepted;
+ }
+
+ @Override
+ public void accept(Object unused, Throwable fail)
+ {
+ if (fail != null) tryFailure(fail);
+ else trySuccess(null);
+ }
+
+ @Override
+ boolean hasMetSuccessCriteria()
+ {
+ return tracker.hasReachedQuorum() || hasMadeProgress();
+ }
+
+ public boolean hasMadeProgress()
+ {
+ return max != null && (max.isCoordinating
+ || max.status.compareTo(knownStatus) > 0
+ || max.promised.compareTo(knownPromised) > 0
+ || (!knownPromisedHasBeenAccepted && knownStatus == Accepted && max.accepted.equals(knownPromised)));
+ }
+
+ // TODO: invoke from {node} so we may have mutual exclusion with other attempts to recover or coordinate
+ public static Future<CheckStatusOk> maybeRecover(Node node, TxnId txnId, Txn txn, Key homeKey, Shard homeShard, long homeEpoch,
+ Status knownStatus, Ballot knownPromised, boolean knownPromiseHasBeenAccepted)
+ {
+ MaybeRecover maybeRecover = new MaybeRecover(node, txnId, txn, homeKey, homeShard, homeEpoch, knownStatus, knownPromised, knownPromiseHasBeenAccepted);
+ maybeRecover.start();
+ return maybeRecover;
+ }
+
+ void onSuccessCriteriaOrExhaustion()
+ {
+ switch (max.status)
+ {
+ default: throw new AssertionError();
+ case NotWitnessed:
+ case PreAccepted:
+ case Accepted:
+ case Committed:
+ case ReadyToExecute:
+ if (hasMadeProgress())
+ {
+ trySuccess(max);
+ }
+ else
+ {
+ node.recover(txnId, txn, key)
+ .addCallback(this);
+ }
+ break;
+
+ case Executed:
+ case Applied:
+ CheckStatusOkFull full = (CheckStatusOkFull) max;
+ if (!max.hasExecutedOnAllShards)
+ {
+ Persist.persist(node, node.topology().preciseEpochs(txn.keys, full.executeAt.epoch), txnId, key, txn, full.executeAt, full.deps, full.writes, full.result)
+ .addCallback(this);
+ }
+ // TODO: apply locally too, in case missing?
+ trySuccess(full);
+ }
+ }
+}
diff --git a/accord-core/src/main/java/accord/coordinate/Persist.java b/accord-core/src/main/java/accord/coordinate/Persist.java
new file mode 100644
index 0000000..0fd4111
--- /dev/null
+++ b/accord-core/src/main/java/accord/coordinate/Persist.java
@@ -0,0 +1,76 @@
+package accord.coordinate;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.IdentityHashMap;
+import java.util.Set;
+
+import accord.api.Key;
+import accord.api.Result;
+import accord.coordinate.tracking.QuorumTracker;
+import accord.local.Node;
+import accord.local.Node.Id;
+import accord.messages.Apply;
+import accord.messages.Apply.ApplyOk;
+import accord.messages.Callback;
+import accord.messages.InformOfPersistence;
+import accord.topology.Shard;
+import accord.topology.Topologies;
+import accord.txn.Dependencies;
+import accord.txn.Timestamp;
+import accord.txn.Txn;
+import accord.txn.TxnId;
+import accord.txn.Writes;
+import org.apache.cassandra.utils.concurrent.AsyncFuture;
+
+// TODO: do not extend AsyncFuture, just use a simple BiConsumer callback
+public class Persist extends AsyncFuture<Void> implements Callback<ApplyOk>
+{
+ final Node node;
+ final TxnId txnId;
+ final Key homeKey;
+ final Timestamp executeAt;
+ final QuorumTracker tracker;
+ Throwable failure;
+ final Set<Id> persistedOn;
+
+ public static AsyncFuture<Void> persist(Node node, Topologies topologies, TxnId txnId, Key homeKey, Txn txn, Timestamp executeAt, Dependencies deps, Writes writes, Result result)
+ {
+ Persist persist = new Persist(node, topologies, txnId, homeKey, executeAt);
+ node.send(topologies.nodes(), to -> new Apply(to, topologies, txnId, txn, homeKey, executeAt, deps, writes, result), persist);
+ return persist;
+ }
+
+ private Persist(Node node, Topologies topologies, TxnId txnId, Key homeKey, Timestamp executeAt)
+ {
+ this.node = node;
+ this.txnId = txnId;
+ this.homeKey = homeKey;
+ this.tracker = new QuorumTracker(topologies);
+ this.executeAt = executeAt;
+ this.persistedOn = new HashSet<>();
+ }
+
+ @Override
+ public void onSuccess(Id from, ApplyOk response)
+ {
+ persistedOn.add(from);
+ if (tracker.success(from) && !isDone())
+ {
+ // TODO: send to non-home replicas also, so they may clear their log more easily?
+ Shard homeShard = node.topology().forEpochIfKnown(homeKey, txnId.epoch);
+ node.send(homeShard, new InformOfPersistence(txnId, homeKey, executeAt, persistedOn));
+ trySuccess(null);
+ }
+ }
+
+ @Override
+ public void onFailure(Id from, Throwable throwable)
+ {
+ if (failure == null) failure = throwable;
+ else failure.addSuppressed(throwable);
+
+ if (tracker.failure(from))
+ tryFailure(failure);
+ }
+}
diff --git a/accord-core/src/main/java/accord/coordinate/AcceptPhase.java b/accord-core/src/main/java/accord/coordinate/Propose.java
similarity index 78%
rename from accord-core/src/main/java/accord/coordinate/AcceptPhase.java
rename to accord-core/src/main/java/accord/coordinate/Propose.java
index 95ad791..8c893dd 100644
--- a/accord-core/src/main/java/accord/coordinate/AcceptPhase.java
+++ b/accord-core/src/main/java/accord/coordinate/Propose.java
@@ -3,6 +3,7 @@ package accord.coordinate;
import java.util.ArrayList;
import java.util.List;
+import accord.api.Key;
import accord.coordinate.tracking.QuorumTracker;
import accord.topology.Topologies;
import accord.txn.Ballot;
@@ -18,23 +19,25 @@ import accord.messages.Accept.AcceptOk;
import accord.messages.Accept.AcceptReply;
import org.apache.cassandra.utils.concurrent.AsyncPromise;
-class AcceptPhase extends AsyncPromise<Agreed>
+class Propose extends AsyncPromise<Agreed>
{
final Node node;
final Ballot ballot;
final TxnId txnId;
final Txn txn;
+ final Key homeKey;
private List<AcceptOk> acceptOks;
private Timestamp proposed;
private QuorumTracker acceptTracker;
- AcceptPhase(Node node, Ballot ballot, TxnId txnId, Txn txn)
+ Propose(Node node, Ballot ballot, TxnId txnId, Txn txn, Key homeKey)
{
this.node = node;
this.ballot = ballot;
this.txnId = txnId;
this.txn = txn;
+ this.homeKey = homeKey;
}
protected void startAccept(Timestamp executeAt, Dependencies deps, Topologies topologies)
@@ -42,7 +45,8 @@ class AcceptPhase extends AsyncPromise<Agreed>
this.proposed = executeAt;
this.acceptOks = new ArrayList<>();
this.acceptTracker = new QuorumTracker(topologies);
- node.send(acceptTracker.nodes(), to -> new Accept(to, topologies, ballot, txnId, txn, executeAt, deps), new Callback<AcceptReply>()
+ // TODO: acceptTracker should be a callback itself, with a reference to us for propagating failure
+ node.send(acceptTracker.nodes(), to -> new Accept(to, topologies, ballot, txnId, txn, homeKey, executeAt, deps), new Callback<AcceptReply>()
{
@Override
public void onSuccess(Id from, AcceptReply response)
@@ -53,8 +57,7 @@ class AcceptPhase extends AsyncPromise<Agreed>
@Override
public void onFailure(Id from, Throwable throwable)
{
- acceptTracker.recordFailure(from);
- if (acceptTracker.hasFailed())
+ if (acceptTracker.failure(from))
tryFailure(new Timeout());
}
});
@@ -73,9 +76,7 @@ class AcceptPhase extends AsyncPromise<Agreed>
AcceptOk ok = (AcceptOk) reply;
acceptOks.add(ok);
- acceptTracker.recordSuccess(from);
-
- if (acceptTracker.hasReachedQuorum())
+ if (acceptTracker.success(from))
onAccepted();
}
@@ -84,11 +85,11 @@ class AcceptPhase extends AsyncPromise<Agreed>
Dependencies deps = new Dependencies();
for (AcceptOk acceptOk : acceptOks)
deps.addAll(acceptOk.deps);
- agreed(proposed, deps, acceptTracker.topologies());
+ agreed(proposed, deps);
}
- protected void agreed(Timestamp executeAt, Dependencies deps, Topologies topologies)
+ protected void agreed(Timestamp executeAt, Dependencies deps)
{
- setSuccess(new Agreed(txnId, txn, executeAt, deps, topologies, null, null));
+ setSuccess(new Agreed(txnId, txn, homeKey, executeAt, deps, null, null));
}
}
diff --git a/accord-core/src/main/java/accord/coordinate/Recover.java b/accord-core/src/main/java/accord/coordinate/Recover.java
index 1165d4d..8927f94 100644
--- a/accord-core/src/main/java/accord/coordinate/Recover.java
+++ b/accord-core/src/main/java/accord/coordinate/Recover.java
@@ -5,6 +5,7 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
+import accord.api.Key;
import accord.coordinate.tracking.FastPathTracker;
import accord.coordinate.tracking.QuorumTracker;
import accord.topology.Shard;
@@ -29,16 +30,15 @@ import org.apache.cassandra.utils.concurrent.Promise;
import static accord.local.Status.Accepted;
// TODO: rename to Recover (verb); rename Recover message to not clash
-class Recover extends AcceptPhase implements Callback<RecoverReply>
+class Recover extends Propose implements Callback<RecoverReply>
{
- private static final Object SENTINEL = new Object();
- static class AwaitCommit extends AsyncPromise<Object> implements Callback<WaitOnCommitOk>
+ class AwaitCommit extends AsyncPromise<Object> implements Callback<WaitOnCommitOk>
{
final QuorumTracker tracker;
AwaitCommit(Node node, TxnId txnId, Txn txn)
{
- Topologies topologies = node.topology().forTxn(txn, txnId.epoch);
+ Topologies topologies = node.topology().unsyncForTxn(txn, txnId.epoch);
this.tracker = new QuorumTracker(topologies);
node.send(topologies.nodes(), to -> new WaitOnCommit(to, topologies, txnId, txn.keys()), this);
}
@@ -48,9 +48,13 @@ class Recover extends AcceptPhase implements Callback<RecoverReply>
{
if (isDone()) return;
- tracker.recordSuccess(from);
- if(tracker.hasReachedQuorum())
- setSuccess(SENTINEL);
+ if (tracker.success(from))
+ {
+ new Recover(node, ballot, txnId, txn, homeKey).addCallback((success, failure) -> {
+ if (success != null) trySuccess(success);
+ else tryFailure(failure);
+ });
+ }
}
@Override
@@ -58,13 +62,12 @@ class Recover extends AcceptPhase implements Callback<RecoverReply>
{
if (isDone()) return;
- tracker.recordFailure(from);
- if (tracker.hasFailed())
+ if (tracker.failure(from))
tryFailure(new Timeout());
}
}
- static Future<Object> awaitCommits(Node node, Dependencies waitOn)
+ Future<Object> awaitCommits(Node node, Dependencies waitOn)
{
AtomicInteger remaining = new AtomicInteger(waitOn.size());
Promise<Object> future = new AsyncPromise<>();
@@ -74,7 +77,7 @@ class Recover extends AcceptPhase implements Callback<RecoverReply>
if (future.isDone())
return;
if (success != null && remaining.decrementAndGet() == 0)
- future.setSuccess(SENTINEL);
+ future.setSuccess(success);
else
future.tryFailure(failure);
});
@@ -112,16 +115,17 @@ class Recover extends AcceptPhase implements Callback<RecoverReply>
final List<RecoverOk> recoverOks = new ArrayList<>();
final FastPathTracker<ShardTracker> tracker;
- public Recover(Node node, Ballot ballot, TxnId txnId, Txn txn)
+ public Recover(Node node, Ballot ballot, TxnId txnId, Txn txn, Key homeKey)
{
- this(node, ballot, txnId, txn, node.topology().forEpoch(txn, txnId.epoch));
+ this(node, ballot, txnId, txn, homeKey, node.topology().forEpoch(txn, txnId.epoch));
}
- private Recover(Node node, Ballot ballot, TxnId txnId, Txn txn, Topologies topologies)
+ private Recover(Node node, Ballot ballot, TxnId txnId, Txn txn, Key homeKey, Topologies topologies)
{
- super(node, ballot, txnId, txn);
+ super(node, ballot, txnId, txn, homeKey);
+ assert topologies.oldestEpoch() == topologies.currentEpoch() && topologies.currentEpoch() == txnId.epoch;
this.tracker = new FastPathTracker<>(topologies, ShardTracker[]::new, ShardTracker::new);
- node.send(tracker.nodes(), to -> new BeginRecovery(to, topologies, txnId, txn, ballot), this);
+ node.send(tracker.nodes(), to -> new BeginRecovery(to, topologies, txnId, txn, homeKey, ballot), this);
}
@Override
@@ -148,30 +152,45 @@ class Recover extends AcceptPhase implements Callback<RecoverReply>
private void recover()
{
// first look for the most recent Accept; if present, go straight to proposing it again
- RecoverOk acceptOrCommit = null;
- for (RecoverOk ok : recoverOks)
- {
- if (ok.status.compareTo(Accepted) >= 0)
+ RecoverOk acceptOrCommit; {
+ RecoverOk tmp = null;
+ for (RecoverOk ok : recoverOks)
{
- if (acceptOrCommit == null) acceptOrCommit = ok;
- else if (acceptOrCommit.status.compareTo(ok.status) < 0) acceptOrCommit = ok;
- else if (acceptOrCommit.status == ok.status && acceptOrCommit.accepted.compareTo(ok.accepted) < 0) acceptOrCommit = ok;
+ if (ok.status.compareTo(Accepted) >= 0)
+ {
+ if (tmp == null) tmp = ok;
+ else if (tmp.status.compareTo(ok.status) < 0) tmp = ok;
+ else if (tmp.status == ok.status && tmp.accepted.compareTo(ok.accepted) < 0) tmp = ok;
+ }
}
+ acceptOrCommit = tmp;
}
if (acceptOrCommit != null)
{
- long minEpoch = Timestamp.min(txnId, acceptOrCommit.executeAt).epoch;
switch (acceptOrCommit.status)
{
case Accepted:
- startAccept(acceptOrCommit.executeAt, acceptOrCommit.deps, node.topology().forTxn(txn, minEpoch));
+ // TODO: this pattern needs to be given some efficient syntactic sugar
+ if (node.topology().hasEpoch(acceptOrCommit.executeAt.epoch))
+ {
+ Topologies topologies = node.topology().forTxn(txn, txnId.epoch, acceptOrCommit.executeAt.epoch);
+ startAccept(acceptOrCommit.executeAt, acceptOrCommit.deps, topologies);
+ }
+ else
+ {
+ node.configService().fetchTopologyForEpoch(acceptOrCommit.executeAt.epoch);
+ node.topology().awaitEpoch(acceptOrCommit.executeAt.epoch).addListener(() -> {
+ Topologies topologies = node.topology().forTxn(txn, txnId.epoch, acceptOrCommit.executeAt.epoch);
+ startAccept(acceptOrCommit.executeAt, acceptOrCommit.deps, topologies);
+ });
+ }
return;
case Committed:
case ReadyToExecute:
case Executed:
case Applied:
- setSuccess(new Agreed(txnId, txn, acceptOrCommit.executeAt, acceptOrCommit.deps, node.topology().forTxn(txn, minEpoch), acceptOrCommit.writes, acceptOrCommit.result));
+ setSuccess(new Agreed(txnId, txn, homeKey, acceptOrCommit.executeAt, acceptOrCommit.deps, acceptOrCommit.writes, acceptOrCommit.result));
return;
}
}
@@ -210,12 +229,22 @@ class Recover extends AcceptPhase implements Callback<RecoverReply>
executeAt = txnId;
}
- startAccept(executeAt, deps, node.topology().forTxn(txn, Timestamp.min(txnId, executeAt).epoch));
+ if (!node.topology().hasEpoch(executeAt.epoch))
+ {
+ node.configService().fetchTopologyForEpoch(executeAt.epoch);
+ node.topology().awaitEpoch(executeAt.epoch).addListener(() -> {
+ startAccept(executeAt, deps, node.topology().forTxn(txn, txnId, executeAt));
+ });
+ }
+ else
+ {
+ startAccept(executeAt, deps, node.topology().forTxn(txn, txnId, executeAt));
+ }
}
private void retry()
{
- new Recover(node, ballot, txnId, txn, node.topology().forEpoch(txn, txnId.epoch)).addCallback((success, failure) -> {
+ new Recover(node, ballot, txnId, txn, homeKey, tracker.topologies()).addCallback((success, failure) -> {
if (success != null) setSuccess(success);
else tryFailure(failure);
});
@@ -227,8 +256,7 @@ class Recover extends AcceptPhase implements Callback<RecoverReply>
if (isDone())
return;
- tracker.recordFailure(from);
- if (tracker.hasFailed())
+ if (tracker.failure(from))
tryFailure(new Timeout());
}
}
diff --git a/accord-core/src/main/java/accord/coordinate/StaleTopology.java b/accord-core/src/main/java/accord/coordinate/StaleTopology.java
new file mode 100644
index 0000000..5591013
--- /dev/null
+++ b/accord-core/src/main/java/accord/coordinate/StaleTopology.java
@@ -0,0 +1,5 @@
+package accord.coordinate;
+
+public class StaleTopology extends Exception
+{
+}
diff --git a/accord-core/src/main/java/accord/coordinate/tracking/AbstractQuorumTracker.java b/accord-core/src/main/java/accord/coordinate/tracking/AbstractQuorumTracker.java
index 1efdbca..2fb39cd 100644
--- a/accord-core/src/main/java/accord/coordinate/tracking/AbstractQuorumTracker.java
+++ b/accord-core/src/main/java/accord/coordinate/tracking/AbstractQuorumTracker.java
@@ -22,7 +22,7 @@ public class AbstractQuorumTracker<T extends AbstractQuorumTracker.QuorumShardTr
this.inflight = new HashSet<>(shard.nodes);
}
- public boolean onSuccess(Node.Id id)
+ protected boolean oneSuccess(Node.Id id)
{
if (!inflight.remove(id))
return false;
@@ -30,15 +30,23 @@ public class AbstractQuorumTracker<T extends AbstractQuorumTracker.QuorumShardTr
return true;
}
- boolean onFailure(Node.Id id)
+ // return true iff hasReachedQuorum()
+ public boolean success(Node.Id id)
+ {
+ oneSuccess(id);
+ return hasReachedQuorum();
+ }
+
+ // return true iff hasFailed()
+ public boolean failure(Node.Id id)
{
if (!inflight.remove(id))
return false;
failures++;
- return true;
+ return hasFailed();
}
- boolean hasFailed()
+ public boolean hasFailed()
{
return failures > shard.maxFailures;
}
@@ -48,7 +56,7 @@ public class AbstractQuorumTracker<T extends AbstractQuorumTracker.QuorumShardTr
return success >= shard.slowPathQuorumSize;
}
- boolean hasInFlight()
+ public boolean hasInFlight()
{
return !inflight.isEmpty();
}
@@ -59,10 +67,10 @@ public class AbstractQuorumTracker<T extends AbstractQuorumTracker.QuorumShardTr
super(topologies, arrayFactory, trackerFactory);
}
- // TODO: refactor to return true if this call caused the state change to failed
- public void recordFailure(Node.Id node)
+ // return true iff hasFailed()
+ public boolean failure(Node.Id node)
{
- forEachTrackerForNode(node, QuorumShardTracker::onFailure);
+ return anyForNode(node, QuorumShardTracker::failure);
}
public boolean hasReachedQuorum()
diff --git a/accord-core/src/main/java/accord/coordinate/tracking/AbstractResponseTracker.java b/accord-core/src/main/java/accord/coordinate/tracking/AbstractResponseTracker.java
index 942cde7..a1622df 100644
--- a/accord-core/src/main/java/accord/coordinate/tracking/AbstractResponseTracker.java
+++ b/accord-core/src/main/java/accord/coordinate/tracking/AbstractResponseTracker.java
@@ -4,12 +4,15 @@ import accord.local.Node;
import accord.topology.Shard;
import accord.topology.Topologies;
import accord.topology.Topology;
+import accord.utils.IndexedIntFunction;
+
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import java.util.*;
import java.util.function.BiConsumer;
import java.util.function.BiFunction;
+import java.util.function.BiPredicate;
import java.util.function.Function;
import java.util.function.IntFunction;
import java.util.function.Predicate;
@@ -84,16 +87,38 @@ public abstract class AbstractResponseTracker<T extends AbstractResponseTracker.
});
}
+ protected boolean anyForNode(Node.Id node, BiPredicate<T, Node.Id> consumer)
+ {
+ return matchingTrackersForNode(node, consumer, 1) == 1;
+ }
+
+ protected boolean allForNode(Node.Id node, BiPredicate<T, Node.Id> consumer)
+ {
+ return nonMatchingTrackersForNode(node, consumer, Integer.MAX_VALUE) == 0;
+ }
+
+ protected int nonMatchingTrackersForNode(Node.Id node, BiPredicate<T, Node.Id> consumer, int limit)
+ {
+ return foldlForNode(node, (shardIndex, shard, v) -> consumer.test(trackers[shardIndex], node) ? v : v + 1, 0, limit);
+ }
+
+ protected int matchingTrackersForNode(Node.Id node, BiPredicate<T, Node.Id> consumer, int limit)
+ {
+ return foldlForNode(node, (shardIndex, shard, v) -> consumer.test(trackers[shardIndex], node) ? v + 1 : v, 0, limit);
+ }
+
protected int matchingTrackersForNode(Node.Id node, Predicate<T> consumer)
{
- int matches = 0;
- for (int i=0, mi=topologies.size(); i<mi; i++)
+ return foldlForNode(node, (shardIndex, shard, v) -> consumer.test(trackers[shardIndex]) ? v + 1 : v, 0, Integer.MAX_VALUE);
+ }
+
+ protected int foldlForNode(Node.Id node, IndexedIntFunction<Shard> function, int initialValue, int terminalValue)
+ {
+ for (int i = 0 ; i < topologies.size() && initialValue != terminalValue ; ++i)
{
- Topology topology = topologies.get(i);
- int offset = topologyOffset(i);
- matches += topology.matchesOn(node, (j, shard) -> consumer.test(trackers[offset + j]));
+ initialValue = topologies.get(i).foldlIntOn(node, function, topologyOffset(i), initialValue, terminalValue);
}
- return matches;
+ return initialValue;
}
protected boolean all(Predicate<T> predicate)
diff --git a/accord-core/src/main/java/accord/coordinate/tracking/FastPathTracker.java b/accord-core/src/main/java/accord/coordinate/tracking/FastPathTracker.java
index 0da268a..18f8213 100644
--- a/accord-core/src/main/java/accord/coordinate/tracking/FastPathTracker.java
+++ b/accord-core/src/main/java/accord/coordinate/tracking/FastPathTracker.java
@@ -22,7 +22,7 @@ public class FastPathTracker<T extends FastPathTracker.FastPathShardTracker> ext
public void onSuccess(Node.Id node, boolean withFastPathTimestamp)
{
- if (onSuccess(node) && includeInFastPath(node, withFastPathTimestamp))
+ if (oneSuccess(node) && includeInFastPath(node, withFastPathTimestamp))
fastPathAccepts++;
}
diff --git a/accord-core/src/main/java/accord/coordinate/tracking/QuorumTracker.java b/accord-core/src/main/java/accord/coordinate/tracking/QuorumTracker.java
index 24539b5..3bc1815 100644
--- a/accord-core/src/main/java/accord/coordinate/tracking/QuorumTracker.java
+++ b/accord-core/src/main/java/accord/coordinate/tracking/QuorumTracker.java
@@ -11,8 +11,9 @@ public class QuorumTracker extends AbstractQuorumTracker<QuorumShardTracker>
super(topologies, QuorumShardTracker[]::new, QuorumShardTracker::new);
}
- public void recordSuccess(Node.Id node)
+ // return true iff hasReachedQuorum()
+ public boolean success(Node.Id node)
{
- forEachTrackerForNode(node, QuorumShardTracker::onSuccess);
+ return allForNode(node, QuorumShardTracker::success) && hasReachedQuorum();
}
}
diff --git a/accord-core/src/main/java/accord/coordinate/tracking/ReadTracker.java b/accord-core/src/main/java/accord/coordinate/tracking/ReadTracker.java
index 3c526a2..8f8fb61 100644
--- a/accord-core/src/main/java/accord/coordinate/tracking/ReadTracker.java
+++ b/accord-core/src/main/java/accord/coordinate/tracking/ReadTracker.java
@@ -11,9 +11,9 @@ import java.util.*;
public class ReadTracker extends AbstractResponseTracker<ReadTracker.ReadShardTracker>
{
- static class ReadShardTracker extends AbstractResponseTracker.ShardTracker
+ public static class ReadShardTracker extends AbstractResponseTracker.ShardTracker
{
- private final Set<Id> inflight = new HashSet<>();
+ protected final Set<Id> inflight = new HashSet<>();
private boolean hasData = false;
private int contacted;
@@ -29,11 +29,11 @@ public class ReadTracker extends AbstractResponseTracker<ReadTracker.ReadShardTr
++contacted;
}
- public void recordReadSuccess(Id node)
+ public boolean recordReadSuccess(Id node)
{
Preconditions.checkArgument(shard.nodes.contains(node));
- inflight.remove(node);
hasData = true;
+ return inflight.remove(node);
}
public boolean shouldRead()
@@ -41,9 +41,9 @@ public class ReadTracker extends AbstractResponseTracker<ReadTracker.ReadShardTr
return !hasData && inflight.isEmpty();
}
- public void recordReadFailure(Id node)
+ public boolean recordReadFailure(Id node)
{
- inflight.remove(node);
+ return inflight.remove(node);
}
public boolean hasCompletedRead()
@@ -57,6 +57,7 @@ public class ReadTracker extends AbstractResponseTracker<ReadTracker.ReadShardTr
}
}
+ // TODO: abstract the candidate selection process so the implementation may prioritise based on distance/health etc
private final List<Id> candidates;
public ReadTracker(Topologies topologies)
@@ -105,6 +106,8 @@ public class ReadTracker extends AbstractResponseTracker<ReadTracker.ReadShardTr
* Return the smallest set of nodes needed to satisfy required reads.
*
* Returns null if the read cannot be completed.
+ *
+ * TODO: prioritisation of nodes should be implementation-defined
*/
public Set<Id> computeMinimalReadSetAndMarkInflight()
{
diff --git a/accord-core/src/main/java/accord/impl/SimpleProgressLog.java b/accord-core/src/main/java/accord/impl/SimpleProgressLog.java
new file mode 100644
index 0000000..10d0235
--- /dev/null
+++ b/accord-core/src/main/java/accord/impl/SimpleProgressLog.java
@@ -0,0 +1,806 @@
+package accord.impl;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.TimeUnit;
+
+import javax.annotation.Nullable;
+
+import com.google.common.base.Preconditions;
+
+import accord.api.Key;
+import accord.api.ProgressLog;
+import accord.api.Result;
+import accord.coordinate.CheckShardStatus;
+import accord.impl.SimpleProgressLog.HomeState.LocalStatus;
+import accord.local.Command;
+import accord.local.CommandStore;
+import accord.local.Node;
+import accord.local.Node.Id;
+import accord.local.Status;
+import accord.messages.Apply;
+import accord.messages.Callback;
+import accord.messages.CheckStatus.CheckStatusOk;
+import accord.messages.MessageType;
+import accord.messages.Reply;
+import accord.messages.ReplyContext;
+import accord.topology.Shard;
+import accord.topology.Topologies;
+import accord.txn.Ballot;
+import accord.txn.Dependencies;
+import accord.txn.Timestamp;
+import accord.txn.Txn;
+import accord.txn.TxnId;
+import accord.txn.Writes;
+import org.apache.cassandra.utils.concurrent.AsyncFuture;
+import org.apache.cassandra.utils.concurrent.Future;
+
+import static accord.coordinate.CheckOnCommitted.checkOnCommitted;
+import static accord.coordinate.CheckOnUncommitted.checkOnUncommitted;
+import static accord.coordinate.InformHomeOfTxn.inform;
+import static accord.coordinate.MaybeRecover.maybeRecover;
+import static accord.impl.SimpleProgressLog.CoordinateApplyAndCheck.applyAndCheck;
+import static accord.impl.SimpleProgressLog.HomeState.GlobalStatus.Durable;
+import static accord.impl.SimpleProgressLog.HomeState.GlobalStatus.Disseminating;
+import static accord.impl.SimpleProgressLog.HomeState.GlobalStatus.NotExecuted;
+import static accord.impl.SimpleProgressLog.HomeState.GlobalStatus.PendingDurable;
+import static accord.impl.SimpleProgressLog.HomeState.LocalStatus.Committed;
+import static accord.impl.SimpleProgressLog.HomeState.LocalStatus.ReadyToExecute;
+import static accord.impl.SimpleProgressLog.HomeState.LocalStatus.Uncommitted;
+import static accord.impl.SimpleProgressLog.NonHomeState.Safe;
+import static accord.impl.SimpleProgressLog.NonHomeState.StillUnsafe;
+import static accord.impl.SimpleProgressLog.NonHomeState.Unsafe;
+import static accord.impl.SimpleProgressLog.Progress.Done;
+import static accord.impl.SimpleProgressLog.Progress.Expected;
+import static accord.impl.SimpleProgressLog.Progress.Investigating;
+import static accord.impl.SimpleProgressLog.Progress.NoProgress;
+import static accord.impl.SimpleProgressLog.Progress.NoneExpected;
+import static accord.impl.SimpleProgressLog.Progress.advance;
+import static accord.local.Status.Executed;
+
+public class SimpleProgressLog implements Runnable, ProgressLog.Factory
+{
+ enum Progress
+ {
+ NoneExpected, Expected, NoProgress, Investigating, Done;
+
+ static Progress advance(Progress current)
+ {
+ switch (current)
+ {
+ default: throw new IllegalStateException();
+ case NoneExpected:
+ case Investigating:
+ case Done:
+ return current;
+ case Expected:
+ case NoProgress:
+ return NoProgress;
+ }
+ }
+ }
+
+ static class GlobalPendingDurable
+ {
+ final Set<Id> persistedOn;
+
+ GlobalPendingDurable(Set<Id> persistedOn)
+ {
+ this.persistedOn = persistedOn;
+ }
+ }
+
+ static class HomeState
+ {
+ enum LocalStatus
+ {
+ NotWitnessed, Uncommitted, Committed, ReadyToExecute, Done;
+ boolean isAtMost(LocalStatus equalOrLessThan)
+ {
+ return compareTo(equalOrLessThan) <= 0;
+ }
+ boolean isAtLeast(LocalStatus equalOrGreaterThan)
+ {
+ return compareTo(equalOrGreaterThan) >= 0;
+ }
+ }
+
+ enum GlobalStatus
+ {
+ NotExecuted, Disseminating, PendingDurable, Durable, Done; // TODO: manage propagating from Durable to everyone
+ boolean isAtLeast(GlobalStatus equalOrGreaterThan)
+ {
+ return compareTo(equalOrGreaterThan) >= 0;
+ }
+ }
+
+ LocalStatus local = LocalStatus.NotWitnessed;
+ Progress localProgress = NoneExpected;
+ Status maxStatus;
+ Ballot maxPromised;
+ boolean maxPromiseHasBeenAccepted;
+
+ GlobalStatus global = NotExecuted;
+ Progress globalProgress = NoneExpected;
+ Set<Id> globalNotPersisted;
+ GlobalPendingDurable globalPendingDurable;
+
+
+ void ensureAtLeast(LocalStatus newStatus, Progress newProgress, Node node, Command command)
+ {
+ if (newStatus == Committed && global.isAtLeast(Durable) && !command.executes())
+ {
+ local = LocalStatus.Done;
+ localProgress = Done;
+ }
+ else if (newStatus.compareTo(local) > 0)
+ {
+ local = newStatus;
+ localProgress = newProgress;
+ }
+ refreshGlobal(node, command, null, null);
+ updateMax(command);
+ }
+
+ void updateMax(Command command)
+ {
+ if (maxStatus == null || maxStatus.compareTo(command.status()) < 0)
+ maxStatus = command.status();
+ if (maxPromised == null || maxPromised.compareTo(command.promised()) < 0)
+ maxPromised = command.promised();
+ maxPromiseHasBeenAccepted |= command.accepted().equals(maxPromised);
+ }
+
+ void updateMax(CheckStatusOk ok)
+ {
+ // TODO: perhaps set localProgress back to Waiting if Investigating and we update anything?
+ if (ok.status.compareTo(maxStatus) > 0) maxStatus = ok.status;
+ if (ok.promised.compareTo(maxPromised) > 0)
+ {
+ maxPromised = ok.promised;
+ maxPromiseHasBeenAccepted = ok.accepted.equals(ok.promised);
+ }
+ else if (ok.promised.equals(maxPromised))
+ {
+ maxPromiseHasBeenAccepted |= ok.accepted.equals(ok.promised);
+ }
+ }
+
+ private void refreshGlobal(@Nullable Node node, @Nullable Command command, @Nullable Id persistedOn, @Nullable Set<Id> persistedOns)
+ {
+ if (global == NotExecuted)
+ return;
+
+ if (globalPendingDurable != null)
+ {
+ if (node == null || command == null || command.is(Status.NotWitnessed))
+ return;
+
+ if (persistedOns == null) persistedOns = globalPendingDurable.persistedOn;
+ else persistedOns.addAll(globalPendingDurable.persistedOn);
+
+ global = Durable;
+ globalProgress = Expected;
+ }
+
+ if (globalNotPersisted == null)
+ {
+ assert node != null && command != null;
+ globalNotPersisted = new HashSet<>(node.topology().unsyncForTxn(command.txn(), command.executeAt().epoch).nodes());
+ if (local == LocalStatus.Done)
+ globalNotPersisted.remove(node.id());
+ }
+ if (globalNotPersisted != null)
+ {
+ if (persistedOn != null)
+ globalNotPersisted.remove(persistedOn);
+ if (persistedOns != null)
+ globalNotPersisted.removeAll(persistedOns);
+
+ if (globalNotPersisted.isEmpty())
+ {
+ global = GlobalStatus.Done;
+ globalProgress = Done;
+ }
+ }
+ }
+
+ void executedOnAllShards(Node node, Command command, Set<Id> persistedOn)
+ {
+ if (local == LocalStatus.NotWitnessed)
+ {
+ global = PendingDurable;
+ globalProgress = NoneExpected;
+ globalPendingDurable = new GlobalPendingDurable(persistedOn);
+ }
+ else if (global != GlobalStatus.Done)
+ {
+ global = Durable;
+ globalProgress = Expected;
+ refreshGlobal(node, command, null, persistedOn);
+ if (local.isAtLeast(Committed) && !command.executes())
+ {
+ local = LocalStatus.Done;
+ localProgress = Done;
+ }
+ }
+ }
+
+ void executed(Node node, Command command)
+ {
+ switch (local)
+ {
+ default: throw new IllegalStateException();
+ case NotWitnessed:
+ case Uncommitted:
+ case Committed:
+ case ReadyToExecute:
+ local = LocalStatus.Done;
+ localProgress = NoneExpected;
+ if (global == NotExecuted)
+ {
+ global = Disseminating;
+ globalProgress = Expected;
+ }
+ refreshGlobal(node, command, node.id(), null);
+ case Done:
+ }
+ }
+
+ void updateLocal(Node node, TxnId txnId, Command command)
+ {
+ if (localProgress != NoProgress)
+ {
+ localProgress = advance(localProgress);
+ return;
+ }
+
+ localProgress = Investigating;
+ switch (local)
+ {
+ default: throw new IllegalStateException();
+ case NotWitnessed:
+ case Committed:
+ throw new IllegalStateException(); // NoProgressExpected
+
+ case Uncommitted:
+ case ReadyToExecute:
+ {
+ Key homeKey = command.homeKey();
+ long homeEpoch = (local.isAtMost(Uncommitted) ? txnId : command.executeAt()).epoch;
+ Shard homeShard = node.topology().forEpochIfKnown(homeKey, homeEpoch);
+ if (homeShard == null)
+ {
+ node.configService().fetchTopologyForEpoch(homeEpoch);
+ localProgress = Expected;
+ break;
+ }
+ if (global.isAtLeast(PendingDurable))
+ {
+ checkOnCommitted(node, txnId, command.txn(), homeKey, homeShard, homeEpoch)
+ .addCallback((success, fail) -> {
+ // should have found enough information to apply the result, but in case we did not reset progress
+ if (localProgress != Done && localProgress == Investigating)
+ localProgress = Expected;
+ });
+ }
+ else
+ {
+ Future<CheckStatusOk> recover = maybeRecover(node, txnId, command.txn(),
+ homeKey, homeShard, homeEpoch,
+ maxStatus, maxPromised, maxPromiseHasBeenAccepted);
+
+ recover.addCallback((success, fail) -> {
+ if (local.isAtMost(ReadyToExecute) && localProgress == Investigating)
+ {
+ localProgress = Expected;
+ if (success != null)
+ {
+ if (success.hasExecutedOnAllShards)
+ executedOnAllShards(node, command, null);
+ else
+ updateMax(success);
+ }
+ }
+ });
+ }
+ }
+ case Done:
+ }
+ }
+
+ void updateGlobal(Node node, TxnId txnId, Command command)
+ {
+ refreshGlobal(node, command, null, null);
+
+ if (global != Disseminating)
+ return;
+
+ if (!command.hasBeen(Executed))
+ return;
+
+ if (globalProgress != NoProgress)
+ {
+ globalProgress = advance(globalProgress);
+ return;
+ }
+
+ globalProgress = Investigating;
+ applyAndCheck(node, txnId, command, this).addCallback((success, fail) -> {
+ if (globalProgress != Done)
+ globalProgress = Expected;
+ });
+ }
+
+ void update(Node node, TxnId txnId, Command command)
+ {
+ updateLocal(node, txnId, command);
+ updateGlobal(node, txnId, command);
+ }
+
+ @Override
+ public String toString()
+ {
+ return "{" + local + ',' + localProgress + ',' + global + ',' + globalProgress + '}';
+ }
+ }
+
+ static class BlockingState
+ {
+ Status blockedOn = Status.NotWitnessed;
+ Progress progress = NoneExpected;
+ Key homeKey;
+
+ void recordBlocking(Command blocking, Key homeKey)
+ {
+ Preconditions.checkState(homeKey != null);
+ this.homeKey = homeKey;
+ switch (blocking.status())
+ {
+ default: throw new IllegalStateException();
+ case NotWitnessed:
+ case PreAccepted:
+ case Accepted:
+ blockedOn = Status.Committed;
+ progress = Expected;
+ break;
+ case Committed:
+ case ReadyToExecute:
+ blockedOn = Executed;
+ progress = Expected;
+ break;
+ case Executed:
+ case Applied:
+ throw new IllegalStateException("Should not be recorded as blocked if result already recorded locally");
+ }
+ }
+
+ void recordCommit()
+ {
+ if (blockedOn == Status.Committed)
+ progress = NoneExpected;
+ }
+
+ void recordExecute()
+ {
+ progress = Done;
+ }
+
+ void update(Node node, TxnId txnId, Command command)
+ {
+ if (progress != NoProgress)
+ {
+ progress = advance(progress);
+ return;
+ }
+
+ progress = Investigating;
+ // check status with the only keys we know, if any, then:
+ // 1. if we cannot find any primary record of the transaction, then it cannot be a dependency so record this fact
+ // 2. otherwise record the homeKey for future reference and set the status based on whether progress has been made
+ Key someKey = command.someKey();
+ Shard someShard = node.topology().forEpochIfKnown(someKey, txnId.epoch);
+ if (someShard == null)
+ {
+ node.configService().fetchTopologyForEpoch(txnId.epoch);
+ progress = Expected;
+ return;
+ }
+ CheckShardStatus check = blockedOn == Executed ? checkOnCommitted(node, txnId, command.txn(), someKey, someShard, txnId.epoch)
+ : checkOnUncommitted(node, txnId, command.txn(), someKey, someShard, txnId.epoch);
+
+ check.addCallback((success, fail) -> {
+ if (fail != null)
+ {
+ if (progress == Investigating)
+ progress = Expected;
+ return;
+ }
+ switch (success.status)
+ {
+ default: throw new IllegalStateException();
+ case NotWitnessed:
+ // TODO: this should instead invalidate the transaction on this shard, which invalidates it for all shards,
+ // but we need to first support invalidation
+ inform(node, txnId, command.txn(), homeKey).addListener(() -> progress = Expected);
+ break;
+ case PreAccepted:
+ case Accepted:
+ progress = Expected;
+ break;
+ case Committed:
+ case ReadyToExecute:
+ if (blockedOn == Status.Committed)
+ progress = NoneExpected;
+ break;
+ case Executed:
+ case Applied:
+ progress = Done;
+ }
+ });
+ }
+
+ public String toString()
+ {
+ return progress.toString();
+ }
+ }
+
+ enum NonHomeState
+ {
+ Unsafe, StillUnsafe, Investigating, Safe
+ }
+
+ static class State
+ {
+ final TxnId txnId;
+ final Command command;
+
+ HomeState homeState;
+ NonHomeState nonHomeState;
+ BlockingState blockingState;
+
+ State(TxnId txnId, Command command)
+ {
+ this.txnId = txnId;
+ this.command = command;
+ }
+
+ void recordBlocking(Command blockedByCommand, Key homeKey)
+ {
+ Preconditions.checkArgument(blockedByCommand.txnId().equals(txnId));
+ if (blockingState == null)
+ blockingState = new BlockingState();
+ blockingState.recordBlocking(blockedByCommand, homeKey);
+ }
+
+ void ensureAtLeast(NonHomeState ensureAtLeast)
+ {
+ if (nonHomeState == null || nonHomeState.compareTo(ensureAtLeast) < 0)
+ nonHomeState = ensureAtLeast;
+ }
+
+ HomeState home()
+ {
+ if (homeState == null)
+ homeState = new HomeState();
+ return homeState;
+ }
+
+ void ensureAtLeast(LocalStatus newStatus, Progress newProgress, Node node)
+ {
+ home().ensureAtLeast(newStatus, newProgress, node, command);
+ }
+
+ void updateNonHome(Node node)
+ {
+ switch (nonHomeState)
+ {
+ default: throw new IllegalStateException();
+ case Safe:
+ case Investigating:
+ break;
+ case Unsafe:
+ nonHomeState = StillUnsafe;
+ break;
+ case StillUnsafe:
+ // make sure a quorum of the home shard is aware of the transaction, so we can rely on it to ensure progress
+ Future<Void> inform = inform(node, txnId, command.txn(), command.homeKey());
+ inform.addCallback((success, fail) -> {
+ if (nonHomeState == Safe)
+ return;
+
+ if (fail != null) nonHomeState = Unsafe;
+ else nonHomeState = Safe;
+ });
+ break;
+ }
+ }
+
+ void update(Node node)
+ {
+ if (blockingState != null)
+ blockingState.update(node, txnId, command);
+
+ if (homeState != null)
+ homeState.update(node, txnId, command);
+
+ if (nonHomeState != null)
+ updateNonHome(node);
+ }
+
+ @Override
+ public String toString()
+ {
+ return homeState != null ? homeState.toString()
+ : nonHomeState != null
+ ? nonHomeState.toString()
+ : blockingState.toString();
+ }
+ }
+
+ final Node node;
+ final List<Instance> instances = new CopyOnWriteArrayList<>();
+
+ public SimpleProgressLog(Node node)
+ {
+ this.node = node;
+ node.scheduler().recurring(this, 200L, TimeUnit.MILLISECONDS);
+ }
+
+ class Instance implements ProgressLog
+ {
+ final CommandStore commandStore;
+ final Map<TxnId, State> stateMap = new HashMap<>();
+
+ Instance(CommandStore commandStore)
+ {
+ this.commandStore = commandStore;
+ instances.add(this);
+ }
+
+ State ensure(TxnId txnId)
+ {
+ return stateMap.computeIfAbsent(txnId, id -> new State(id, commandStore.command(id)));
+ }
+
+ State ensure(TxnId txnId, State state)
+ {
+ return state != null ? state : ensure(txnId);
+ }
+
+ @Override
+ public void preaccept(TxnId txnId, boolean isProgressShard, boolean isHomeShard)
+ {
+ if (isProgressShard)
+ {
+ State state = ensure(txnId);
+ if (isHomeShard) state.ensureAtLeast(Uncommitted, Expected, node);
+ else state.ensureAtLeast(NonHomeState.Unsafe);
+ }
+ }
+
+ State recordCommit(TxnId txnId)
+ {
+ State state = stateMap.get(txnId);
+ if (state != null && state.blockingState != null)
+ state.blockingState.recordCommit();
+ return state;
+ }
+
+ State recordExecute(TxnId txnId)
+ {
+ State state = stateMap.get(txnId);
+ if (state != null && state.blockingState != null)
+ state.blockingState.recordExecute();
+ return state;
+ }
+
+ private void ensureSafeOrAtLeast(TxnId txnId, boolean isProgressShard, boolean isHomeShard, LocalStatus newStatus, Progress newProgress)
+ {
+ State state = null;
+ assert newStatus.isAtMost(ReadyToExecute);
+ if (newStatus.isAtLeast(LocalStatus.Committed))
+ state = recordCommit(txnId);
+
+ if (isProgressShard)
+ {
+ state = ensure(txnId, state);
+
+ if (isHomeShard) state.ensureAtLeast(newStatus, newProgress, node);
+ else ensure(txnId).ensureAtLeast(Safe);
+ }
+ }
+
+ @Override
+ public void accept(TxnId txnId, boolean isProgressShard, boolean isHomeShard)
+ {
+ ensureSafeOrAtLeast(txnId, isProgressShard, isHomeShard, Uncommitted, Expected);
+ }
+
+ @Override
+ public void commit(TxnId txnId, boolean isProgressShard, boolean isHomeShard)
+ {
+ ensureSafeOrAtLeast(txnId, isProgressShard, isHomeShard, LocalStatus.Committed, NoneExpected);
+ }
+
+ @Override
+ public void readyToExecute(TxnId txnId, boolean isProgressShard, boolean isHomeShard)
+ {
+ ensureSafeOrAtLeast(txnId, isProgressShard, isHomeShard, LocalStatus.ReadyToExecute, Expected);
+ }
+
+ @Override
+ public void execute(TxnId txnId, boolean isProgressShard, boolean isHomeShard)
+ {
+ State state = recordExecute(txnId);
+
+ if (isProgressShard)
+ {
+ state = ensure(txnId, state);
+
+ if (isHomeShard) state.home().executed(node, state.command);
+ else ensure(txnId).ensureAtLeast(Safe);
+ }
+ }
+
+ @Override
+ public void executedOnAllShards(TxnId txnId, Set<Id> persistedOn)
+ {
+ State state = ensure(txnId);
+ state.home().executedOnAllShards(node, state.command, persistedOn);
+ }
+
+ @Override
+ public void waiting(TxnId blockedBy, Key homeKey)
+ {
+ Command blockedByCommand = commandStore.command(blockedBy);
+ if (!blockedByCommand.hasBeen(Executed))
+ ensure(blockedBy).recordBlocking(blockedByCommand, homeKey);
+ }
+ }
+
+ @Override
+ public void run()
+ {
+ for (Instance instance : instances)
+ {
+ // TODO: we want to be able to poll others about pending dependencies to check forward progress,
+ // as we don't know all dependencies locally (or perhaps any, at execution time) so we may
+ // begin expecting forward progress too early
+ instance.stateMap.values().forEach(state -> state.update(node));
+ }
+ }
+
+ static class CoordinateApplyAndCheck extends AsyncFuture<Void> implements Callback<ApplyAndCheckOk>
+ {
+ final TxnId txnId;
+ final Command command;
+ final HomeState state;
+ final Set<Id> waitingOnResponses;
+
+ static Future<Void> applyAndCheck(Node node, TxnId txnId, Command command, HomeState state)
+ {
+ CoordinateApplyAndCheck coordinate = new CoordinateApplyAndCheck(txnId, command, state);
+ Topologies topologies = node.topology().unsyncForTxn(command.txn(), command.executeAt().epoch);
+ state.globalNotPersisted.retainAll(topologies.nodes()); // we might have had some nodes from older shards that are now redundant
+ node.send(state.globalNotPersisted, id -> new ApplyAndCheck(id, topologies,
+ command.txnId(), command.txn(), command.homeKey(),
+ command.savedDeps(), command.executeAt(),
+ command.writes(), command.result(),
+ state.globalNotPersisted),
+ coordinate);
+ return coordinate;
+ }
+
+ CoordinateApplyAndCheck(TxnId txnId, Command command, HomeState state)
+ {
+ this.txnId = txnId;
+ this.command = command;
+ this.state = state;
+ this.waitingOnResponses = new HashSet<>(state.globalNotPersisted);
+ }
+
+ @Override
+ public void onSuccess(Id from, ApplyAndCheckOk response)
+ {
+ state.globalNotPersisted.retainAll(response.notPersisted);
+ state.refreshGlobal(null, null, null, null);
+ }
+
+ @Override
+ public void onFailure(Id from, Throwable throwable)
+ {
+ if (waitingOnResponses.remove(from) && waitingOnResponses.isEmpty())
+ trySuccess(null);
+ }
+ }
+
+ static class ApplyAndCheck extends Apply
+ {
+ final Set<Id> notPersisted;
+ ApplyAndCheck(Id id, Topologies topologies, TxnId txnId, Txn txn, Key homeKey, Dependencies deps, Timestamp executeAt, Writes writes, Result result, Set<Id> notPersisted)
+ {
+ super(id, topologies, txnId, txn, homeKey, executeAt, deps, writes, result);
+ this.notPersisted = notPersisted;
+ }
+
+ @Override
+ public void process(Node node, Id from, ReplyContext replyContext)
+ {
+ Key progressKey = node.trySelectProgressKey(txnId, txn.keys, homeKey);
+ node.reply(from, replyContext, node.mapReduceLocalSince(scope(), executeAt, instance -> {
+ Command command = instance.command(txnId);
+ command.apply(txn, homeKey, progressKey, executeAt, deps, writes, result);
+ if (homeKey.equals(progressKey) && command.handles(txnId.epoch, progressKey))
+ {
+ SimpleProgressLog.Instance log = ((SimpleProgressLog.Instance)instance.progressLog());
+ State state = log.stateMap.get(txnId);
+ state.homeState.globalNotPersisted.retainAll(notPersisted);
+ return new ApplyAndCheckOk(state.homeState.globalNotPersisted);
+ }
+ notPersisted.remove(node.id());
+ return new ApplyAndCheckOk(notPersisted);
+ }, (a, b) -> {
+ if (a == null) return b;
+ if (b == null) return a;
+ a.notPersisted.retainAll(b.notPersisted);
+ return a;
+ }));
+ }
+
+ @Override
+ public MessageType type()
+ {
+ return MessageType.APPLY_AND_CHECK_REQ;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "SendAndCheck{" +
+ "txnId:" + txnId +
+ ", txn:" + txn +
+ ", deps:" + deps +
+ ", executeAt:" + executeAt +
+ ", writes:" + writes +
+ ", result:" + result +
+ ", waitingOn:" + notPersisted +
+ '}';
+ }
+ }
+
+ static class ApplyAndCheckOk implements Reply
+ {
+ final Set<Id> notPersisted;
+
+ ApplyAndCheckOk(Set<Id> notPersisted)
+ {
+ this.notPersisted = notPersisted;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "SendAndCheckOk{" +
+ "waitingOn:" + notPersisted +
+ '}';
+ }
+
+ @Override
+ public MessageType type()
+ {
+ return MessageType.APPLY_AND_CHECK_RSP;
+ }
+ }
+
+ @Override
+ public ProgressLog create(CommandStore commandStore)
+ {
+ return new Instance(commandStore);
+ }
+}
diff --git a/accord-core/src/main/java/accord/local/Command.java b/accord-core/src/main/java/accord/local/Command.java
index 1e3e1f8..27ccc13 100644
--- a/accord-core/src/main/java/accord/local/Command.java
+++ b/accord-core/src/main/java/accord/local/Command.java
@@ -4,9 +4,13 @@ import java.util.NavigableMap;
import java.util.TreeMap;
import java.util.function.Consumer;
+import accord.api.Key;
import accord.api.Result;
+import accord.local.Node.Id;
+import accord.topology.KeyRanges;
import accord.txn.Ballot;
import accord.txn.Dependencies;
+import accord.txn.Keys.KeyFold;
import accord.txn.Timestamp;
import accord.txn.Txn;
import accord.txn.TxnId;
@@ -20,18 +24,21 @@ import static accord.local.Status.NotWitnessed;
import static accord.local.Status.PreAccepted;
import static accord.local.Status.ReadyToExecute;
+// TODO: this needs to be backed by persistent storage
public class Command implements Listener, Consumer<Listener>
{
public final CommandStore commandStore;
private final TxnId txnId;
- private Txn txn;
+ private Key homeKey, progressKey;
+ private Txn txn; // TODO: only store this on the home shard, or split to each shard independently
private Ballot promised = Ballot.ZERO, accepted = Ballot.ZERO;
- private Timestamp executeAt;
+ private Timestamp executeAt; // TODO: compress these states together
private Dependencies deps = new Dependencies();
private Writes writes;
private Result result;
private Status status = NotWitnessed;
+ private boolean isGloballyPersistent; // only set on home shard
private NavigableMap<TxnId, Command> waitingOnCommit;
private NavigableMap<Timestamp, Command> waitingOnApply;
@@ -99,32 +106,65 @@ public class Command implements Listener, Consumer<Listener>
return this.status == status;
}
+ public boolean isGloballyPersistent()
+ {
+ return isGloballyPersistent;
+ }
+
+ public void setGloballyPersistent(Key homeKey, Timestamp executeAt)
+ {
+ homeKey(homeKey);
+ if (!hasBeen(Committed))
+ this.executeAt = executeAt;
+ else if (!this.executeAt.equals(executeAt))
+ commandStore.agent().onInconsistentTimestamp(this, this.executeAt, executeAt);
+ isGloballyPersistent = true;
+ }
+
// requires that command != null
// relies on mutual exclusion for each key
- public boolean witness(Txn txn)
+ // note: we do not set status = newStatus, we only use it to decide how we register with the retryLog
+ private void witness(Txn txn, Key homeKey, Key progressKey)
{
- if (promised.compareTo(Ballot.ZERO) > 0)
- return false;
-
if (hasBeen(PreAccepted))
- return true;
+ return;
- Timestamp max = txn.maxConflict(commandStore);
+ Timestamp max = commandStore.maxConflict(txn.keys);
// unlike in the Accord paper, we partition shards within a node, so that to ensure a total order we must either:
// - use a global logical clock to issue new timestamps; or
// - assign each shard _and_ process a unique id, and use both as components of the timestamp
- Timestamp witnessed = txnId.compareTo(max) > 0 && txnId.epoch >= commandStore.epoch() ? txnId : commandStore.uniqueNow(max);
+ Timestamp witnessed = txnId.compareTo(max) > 0 && txnId.epoch >= commandStore.latestEpoch() ? txnId : commandStore.uniqueNow(max);
- this.txn = txn;
+ txn(txn);
+ homeKey(homeKey);
+ progressKey(progressKey);
this.executeAt = witnessed;
this.status = PreAccepted;
- txn.register(commandStore, this);
+ txn.keys().foldl(commandStore.ranges().since(txnId.epoch), (key, param) -> {
+ if (commandStore.hashIntersects(key))
+ commandStore.commandsForKey(key).register(this);
+ return null;
+ }, null);
+ }
+
+ public boolean preaccept(Txn txn, Key homeKey, Key progressKey)
+ {
+ if (promised.compareTo(Ballot.ZERO) > 0)
+ return false;
+
+ if (hasBeen(PreAccepted))
+ return true;
+
+ witness(txn, homeKey, progressKey);
+ boolean isProgressShard = progressKey != null && handles(txnId.epoch, progressKey);
+ commandStore.progressLog().preaccept(txnId, isProgressShard, isProgressShard && progressKey.equals(homeKey));
+
listeners.forEach(this);
return true;
}
- public boolean accept(Ballot ballot, Txn txn, Timestamp executeAt, Dependencies deps)
+ public boolean accept(Ballot ballot, Txn txn, Key homeKey, Key progressKey, Timestamp executeAt, Dependencies deps)
{
if (this.promised.compareTo(ballot) > 0)
return false;
@@ -132,17 +172,21 @@ public class Command implements Listener, Consumer<Listener>
if (hasBeen(Committed))
return false;
- witness(txn);
+ witness(txn, homeKey, progressKey);
this.deps = deps;
this.executeAt = executeAt;
promised = accepted = ballot;
status = Accepted;
+
+ boolean isProgressShard = progressKey != null && handles(txnId.epoch, progressKey);
+ commandStore.progressLog().accept(txnId, isProgressShard, isProgressShard && progressKey.equals(homeKey));
+
listeners.forEach(this);
return true;
}
// relies on mutual exclusion for each key
- public boolean commit(Txn txn, Dependencies deps, Timestamp executeAt)
+ public boolean commit(Txn txn, Key homeKey, Key progressKey, Timestamp executeAt, Dependencies deps)
{
if (hasBeen(Committed))
{
@@ -152,14 +196,14 @@ public class Command implements Listener, Consumer<Listener>
commandStore.agent().onInconsistentTimestamp(this, this.executeAt, executeAt);
}
- witness(txn);
+ witness(txn, homeKey, progressKey);
this.status = Committed;
this.deps = deps;
this.executeAt = executeAt;
this.waitingOnCommit = new TreeMap<>();
this.waitingOnApply = new TreeMap<>();
- for (TxnId id : savedDeps().on(commandStore))
+ for (TxnId id : savedDeps().on(commandStore, executeAt))
{
Command command = commandStore.command(id);
switch (command.status)
@@ -167,7 +211,8 @@ public class Command implements Listener, Consumer<Listener>
default:
throw new IllegalStateException();
case NotWitnessed:
- command.witness(deps.get(command.txnId));
+ Txn depTxn = savedDeps().get(id);
+ command.txn(depTxn);
case PreAccepted:
case Accepted:
// we don't know when these dependencies will execute, and cannot execute until we do
@@ -192,17 +237,23 @@ public class Command implements Listener, Consumer<Listener>
if (waitingOnApply.isEmpty())
waitingOnApply = null;
}
+
+ // TODO: we might not be the homeShard for later phases if we are no longer replicas of the range at executeAt;
+ // this should be fine, but it might be helpful to provide this info to the progressLog here?
+ boolean isProgressShard = progressKey != null && handles(txnId.epoch, progressKey);
+ commandStore.progressLog().commit(txnId, isProgressShard, isProgressShard && progressKey.equals(homeKey));
+
+ maybeExecute(false);
listeners.forEach(this);
- maybeExecute();
return true;
}
- public boolean apply(Txn txn, Dependencies deps, Timestamp executeAt, Writes writes, Result result)
+ public boolean apply(Txn txn, Key homeKey, Key progressKey, Timestamp executeAt, Dependencies deps, Writes writes, Result result)
{
if (hasBeen(Executed) && executeAt.equals(this.executeAt))
return false;
else if (!hasBeen(Committed))
- commit(txn, deps, executeAt);
+ commit(txn, homeKey, progressKey, executeAt, deps);
else if (!executeAt.equals(this.executeAt))
commandStore.agent().onInconsistentTimestamp(this, this.executeAt, executeAt);
@@ -210,18 +261,24 @@ public class Command implements Listener, Consumer<Listener>
this.writes = writes;
this.result = result;
this.status = Executed;
+
+ boolean isProgressShard = progressKey != null && handles(txnId.epoch, progressKey);
+ commandStore.progressLog().execute(txnId, isProgressShard, isProgressShard && progressKey.equals(homeKey));
+
+ maybeExecute(false);
this.listeners.forEach(this);
- maybeExecute();
return true;
}
- public boolean recover(Txn txn, Ballot ballot)
+ public boolean recover(Txn txn, Key homeKey, Key progressKey, Ballot ballot)
{
if (this.promised.compareTo(ballot) > 0)
return false;
- witness(txn);
+ witness(txn, homeKey, progressKey);
this.promised = ballot;
+ boolean isProgressShard = progressKey != null && handles(txnId.epoch, progressKey);
+ commandStore.progressLog().preaccept(txnId, isProgressShard, isProgressShard && progressKey.equals(homeKey));
return true;
}
@@ -241,6 +298,12 @@ public class Command implements Listener, Consumer<Listener>
{
switch (command.status)
{
+ default: throw new IllegalStateException();
+ case NotWitnessed:
+ case PreAccepted:
+ case Accepted:
+ break;
+
case Committed:
case ReadyToExecute:
case Executed:
@@ -260,61 +323,180 @@ public class Command implements Listener, Consumer<Listener>
{
command.removeListener(this);
}
- maybeExecute();
+ maybeExecute(true);
break;
}
}
- private void maybeExecute()
+ private void maybeExecute(boolean notifyListeners)
{
if (status != Committed && status != Executed)
return;
if (waitingOnApply != null)
- return;
+ {
+ BlockedBy blockedBy = blockedBy();
+ if (blockedBy != null)
+ {
+ Key homeKey = blockedBy.directlyBlockedBy.homeKey();
+ if (homeKey == null)
+ homeKey = blockedBy.directlyBlocked.deps.homeKey(blockedBy.directlyBlockedBy.txnId());
+ commandStore.progressLog().waiting(blockedBy.directlyBlockedBy.txnId, homeKey);
+ return;
+ }
+ assert waitingOnApply == null;
+ }
switch (status)
{
case Committed:
// TODO: maintain distinct ReadyToRead and ReadyToWrite states
status = ReadyToExecute;
- listeners.forEach(this);
+ boolean isProgressShard = progressKey != null && handles(txnId.epoch, progressKey);
+ commandStore.progressLog().readyToExecute(txnId, isProgressShard, isProgressShard && progressKey.equals(homeKey));
+ if (notifyListeners)
+ listeners.forEach(this);
break;
case Executed:
writes.apply(commandStore);
status = Applied;
- listeners.forEach(this);
+ if (notifyListeners)
+ listeners.forEach(this);
}
}
- private void updatePredecessor(Command committed)
+ /**
+ * @param dependency is either committed or has an executeAtExclusiveLowerBound >= our executeAt
+ */
+ private void updatePredecessor(Command dependency)
{
- if (committed.executeAt.compareTo(executeAt) > 0)
+ if (!dependency.hasBeen(Committed))
+ {
+ dependency.removeListener(this);
+ }
+ else if (dependency.executeAt.compareTo(executeAt) > 0)
{
// cannot be a predecessor if we execute later
- committed.removeListener(this);
+ dependency.removeListener(this);
}
- else if (committed.hasBeen(Applied))
+ else if (dependency.hasBeen(Applied))
{
- waitingOnApply.remove(committed.executeAt);
- committed.removeListener(this);
+ waitingOnApply.remove(dependency.executeAt);
+ dependency.removeListener(this);
}
else
{
- waitingOnApply.putIfAbsent(committed.executeAt, committed);
+ waitingOnApply.putIfAbsent(dependency.executeAt, dependency);
+ }
+ }
+
+ // TEMPORARY: once we can invalidate commands that have not been witnessed on any shard, we do not need to know the home shard
+ static class BlockedBy
+ {
+ final Command directlyBlocked;
+ final Command directlyBlockedBy;
+
+ BlockedBy(Command directlyBlocked, Command directlyBlockedBy)
+ {
+ this.directlyBlocked = directlyBlocked;
+ this.directlyBlockedBy = directlyBlockedBy;
}
}
- public Command blockedBy()
+ public BlockedBy blockedBy()
{
+ Command prev = this;
Command cur = directlyBlockedBy();
if (cur == null)
return null;
Command next;
while (null != (next = cur.directlyBlockedBy()))
+ {
+ prev = cur;
cur = next;
- return cur;
+ }
+
+ return new BlockedBy(prev, cur);
+ }
+
+ /**
+ * A key nominated to represent the "home" shard - only members of the home shard may be nominated to recover
+ * a transaction, to reduce the cluster-wide overhead of ensuring progress. A transaction that has only been
+ * witnessed at PreAccept may however trigger a process of ensuring the home shard is durably informed of
+ * the transaction.
+ *
+ * Note that for ProgressLog purposes the "home shard" is the shard as of txnId.epoch.
+ * For recovery purposes the "home shard" is as of txnId.epoch until Committed, and executeAt.epoch once Executed
+ *
+ * TODO: Markdown documentation explaining the home shard and local shard concepts
+ */
+ public Key homeKey()
+ {
+ return homeKey;
+ }
+
+ public void homeKey(Key homeKey)
+ {
+ if (this.homeKey == null) this.homeKey = homeKey;
+ else if (!this.homeKey.equals(homeKey)) throw new AssertionError();
+ }
+
+ public Key someKey()
+ {
+ if (homeKey != null)
+ return homeKey;
+
+ return txn.keys.get(0);
+ }
+
+ /**
+ * A key nominated to be the primary shard within this node for managing progress of the command.
+ * It is nominated only as of txnId.epoch, and may be null (indicating that this node does not monitor
+ * the progress of this command).
+ *
+ * Preferentially, this is homeKey on nodes that replicate it, and otherwise any key that is replicated, as of txnId.epoch
+ */
+ public Key progressKey()
+ {
+ return progressKey;
+ }
+
+ public void progressKey(Key progressKey)
+ {
+ if (this.progressKey == null) this.progressKey = progressKey;
+ else if (!this.progressKey.equals(progressKey)) throw new AssertionError();
+ }
+
+ // does this specific Command instance execute (i.e. does it lose ownership post Commit)
+ public boolean executes()
+ {
+ KeyRanges ranges = commandStore.ranges().at(executeAt.epoch);
+ return ranges != null && txn.keys.any(ranges, commandStore::hashIntersects);
+ }
+
+ public void txn(Txn txn)
+ {
+ if (this.txn == null) this.txn = txn;
+ else if (!this.txn.equals(txn)) throw new AssertionError();
+ }
+
+ public boolean handles(long epoch, Key someKey)
+ {
+ if (!commandStore.hashIntersects(someKey))
+ return false;
+
+ KeyRanges ranges = commandStore.ranges().at(epoch);
+ if (ranges == null)
+ return false;
+ return ranges.contains(someKey);
+ }
+
+ private Id coordinator()
+ {
+ if (promised.equals(Ballot.ZERO))
+ return txnId.node;
+ return promised.node;
}
private Command directlyBlockedBy()
diff --git a/accord-core/src/main/java/accord/local/CommandStore.java b/accord-core/src/main/java/accord/local/CommandStore.java
index ae1c153..ec538c7 100644
--- a/accord-core/src/main/java/accord/local/CommandStore.java
+++ b/accord-core/src/main/java/accord/local/CommandStore.java
@@ -2,12 +2,11 @@ package accord.local;
import accord.api.Agent;
import accord.api.Key;
-import accord.api.KeyRange;
-import accord.api.Store;
-import accord.local.CommandStores.StoreGroup;
-import accord.local.Node.Id;
+import accord.local.CommandStores.ShardedRanges;
+import accord.api.ProgressLog;
+import accord.topology.KeyRange;
+import accord.api.DataStore;
import accord.topology.KeyRanges;
-import accord.topology.Topology;
import accord.txn.Keys;
import accord.txn.Timestamp;
import accord.txn.TxnId;
@@ -15,14 +14,15 @@ import org.apache.cassandra.utils.concurrent.AsyncPromise;
import org.apache.cassandra.utils.concurrent.Future;
import org.apache.cassandra.utils.concurrent.Promise;
+import com.google.common.base.Preconditions;
+
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
import java.util.function.Function;
-import java.util.function.Supplier;
+import java.util.function.LongFunction;
-import com.google.common.base.Preconditions;
import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
/**
@@ -33,48 +33,56 @@ public abstract class CommandStore
public interface Factory
{
CommandStore create(int generation,
- int index,
+ int shardIndex,
int numShards,
- Node.Id nodeId,
- Function<Timestamp, Timestamp> uniqueNow,
+ Node node,
Agent agent,
- Store store,
- KeyRanges ranges,
- Supplier<Topology> localTopologySupplier);
+ DataStore store,
+ ProgressLog.Factory progressLogFactory,
+ RangesForEpoch rangesForEpoch);
+ }
+
+ public interface RangesForEpoch
+ {
+ KeyRanges at(long epoch);
+ KeyRanges since(long epoch);
}
private final int generation;
- private final int index;
+ private final int shardIndex;
private final int numShards;
- private final Node.Id nodeId;
- private final Function<Timestamp, Timestamp> uniqueNow;
+ private final Node node;
private final Agent agent;
- private final Store store;
- private final KeyRanges ranges;
- private final Supplier<Topology> localTopologySupplier;
+ private final DataStore store;
+ private final ProgressLog progressLog;
+ private final RangesForEpoch rangesForEpoch;
private final NavigableMap<TxnId, Command> commands = new TreeMap<>();
private final NavigableMap<Key, CommandsForKey> commandsForKey = new TreeMap<>();
public CommandStore(int generation,
- int index,
+ int shardIndex,
int numShards,
- Node.Id nodeId,
- Function<Timestamp, Timestamp> uniqueNow,
+ Node node,
Agent agent,
- Store store,
- KeyRanges ranges,
- Supplier<Topology> localTopologySupplier)
+ DataStore store,
+ ProgressLog.Factory progressLogFactory,
+ RangesForEpoch rangesForEpoch)
{
+ Preconditions.checkArgument(shardIndex < numShards);
this.generation = generation;
- this.index = index;
+ this.shardIndex = shardIndex;
this.numShards = numShards;
- this.nodeId = nodeId;
- this.uniqueNow = uniqueNow;
+ this.node = node;
this.agent = agent;
this.store = store;
- this.ranges = ranges;
- this.localTopologySupplier = localTopologySupplier;
+ this.progressLog = progressLogFactory.create(this);
+ this.rangesForEpoch = rangesForEpoch;
+ }
+
+ public Command ifPresent(TxnId txnId)
+ {
+ return commands.get(txnId);
}
public Command command(TxnId txnId)
@@ -92,19 +100,24 @@ public abstract class CommandStore
return commandsForKey.computeIfAbsent(key, ignore -> new CommandsForKey());
}
+ public CommandsForKey maybeCommandsForKey(Key key)
+ {
+ return commandsForKey.get(key);
+ }
+
public boolean hasCommandsForKey(Key key)
{
return commandsForKey.containsKey(key);
}
- public Store store()
+ public DataStore store()
{
return store;
}
public Timestamp uniqueNow(Timestamp atLeast)
{
- return uniqueNow.apply(atLeast);
+ return node.uniqueNow(atLeast);
}
public Agent agent()
@@ -112,19 +125,35 @@ public abstract class CommandStore
return agent;
}
- public Node.Id nodeId()
+ public ProgressLog progressLog()
{
- return nodeId;
+ return progressLog;
}
- public long epoch()
+ public Node node()
{
- return localTopologySupplier.get().epoch();
+ return node;
}
- public KeyRanges ranges()
+ public RangesForEpoch ranges()
{
- return ranges;
+ return rangesForEpoch;
+ }
+
+ public long latestEpoch()
+ {
+ // TODO: why not inject the epoch to each command store?
+ return node.epoch();
+ }
+
+ protected Timestamp maxConflict(Keys keys)
+ {
+ return keys.stream()
+ .map(this::maybeCommandsForKey)
+ .filter(Objects::nonNull)
+ .map(CommandsForKey::max)
+ .max(Comparator.naturalOrder())
+ .orElse(Timestamp.NONE);
}
public void forEpochCommands(KeyRanges ranges, long epoch, Consumer<Command> consumer)
@@ -166,26 +195,16 @@ public abstract class CommandStore
}
}
- public int index()
- {
- return index;
- }
-
public boolean hashIntersects(Key key)
{
- return StoreGroup.keyIndex(key, numShards) == index;
+ return ShardedRanges.keyIndex(key, numShards) == shardIndex;
}
- public boolean intersects(Keys keys)
+ public boolean intersects(Keys keys, KeyRanges ranges)
{
return keys.any(ranges, this::hashIntersects);
}
- public boolean contains(Key key)
- {
- return ranges.contains(key);
- }
-
public static void onEach(Collection<CommandStore> stores, Consumer<? super CommandStore> consumer)
{
for (CommandStore store : stores)
@@ -241,12 +260,12 @@ public abstract class CommandStore
public static class Synchronized extends CommandStore
{
- public Synchronized(int generation, int index, int numShards, Node.Id nodeId,
- Function<Timestamp, Timestamp> uniqueNow,
- Agent agent, Store store,
- KeyRanges ranges, Supplier<Topology> localTopologySupplier)
+ public Synchronized(int generation, int index, int numShards, Node node,
+ Agent agent, DataStore store,
+ ProgressLog.Factory progressLogFactory,
+ RangesForEpoch rangesForEpoch)
{
- super(generation, index, numShards, nodeId, uniqueNow, agent, store, ranges, localTopologySupplier);
+ super(generation, index, numShards, node, agent, store, progressLogFactory, rangesForEpoch);
}
@Override
@@ -305,15 +324,14 @@ public abstract class CommandStore
}
}
- public SingleThread(int generation, int index, int numShards, Node.Id nodeId,
- Function<Timestamp, Timestamp> uniqueNow,
- Agent agent, Store store,
- KeyRanges ranges, Supplier<Topology> localTopologySupplier)
+ public SingleThread(int generation, int index, int numShards, Node node,
+ Agent agent, DataStore store, ProgressLog.Factory progressLogFactory,
+ RangesForEpoch rangesForEpoch)
{
- super(generation, index, numShards, nodeId, uniqueNow, agent, store, ranges, localTopologySupplier);
+ super(generation, index, numShards, node, agent, store, progressLogFactory, rangesForEpoch);
executor = Executors.newSingleThreadExecutor(r -> {
Thread thread = new Thread(r);
- thread.setName(CommandStore.class.getSimpleName() + '[' + nodeId + ':' + index + ']');
+ thread.setName(CommandStore.class.getSimpleName() + '[' + node.id() + ':' + index + ']');
return thread;
});
}
@@ -345,12 +363,11 @@ public abstract class CommandStore
{
private final AtomicReference<Thread> expectedThread = new AtomicReference<>();
- public Debug(int generation, int index, int numShards, Node.Id nodeId,
- Function<Timestamp, Timestamp> uniqueNow,
- Agent agent, Store store,
- KeyRanges ranges, Supplier<Topology> localTopologySupplier)
+ public Debug(int generation, int index, int numShards, Node node,
+ Agent agent, DataStore store, ProgressLog.Factory progressLogFactory,
+ RangesForEpoch rangesForEpoch)
{
- super(generation, index, numShards, nodeId, uniqueNow, agent, store, ranges, localTopologySupplier);
+ super(generation, index, numShards, node, agent, store, progressLogFactory, rangesForEpoch);
}
private void assertThread()
diff --git a/accord-core/src/main/java/accord/local/CommandStores.java b/accord-core/src/main/java/accord/local/CommandStores.java
index 68efb68..0b195c7 100644
--- a/accord-core/src/main/java/accord/local/CommandStores.java
+++ b/accord-core/src/main/java/accord/local/CommandStores.java
@@ -2,15 +2,13 @@ package accord.local;
import accord.api.Agent;
import accord.api.Key;
-import accord.api.Store;
-import accord.local.CommandStore.SingleThread;
-import accord.local.CommandStores.StoreGroups.Fold;
-import accord.local.Node.Id;
+import accord.api.DataStore;
+import accord.local.CommandStore.RangesForEpoch;
import accord.messages.TxnRequest;
+import accord.api.ProgressLog;
import accord.topology.KeyRanges;
import accord.topology.Topology;
import accord.txn.Keys;
-import accord.txn.Timestamp;
import org.apache.cassandra.utils.concurrent.Future;
import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
@@ -30,212 +28,357 @@ public abstract class CommandStores
{
public interface Factory
{
- CommandStores create(int num, Node.Id node, Function<Timestamp, Timestamp> uniqueNow, Agent agent, Store store);
+ CommandStores create(int num,
+ Node node,
+ Agent agent,
+ DataStore store,
+ ProgressLog.Factory progressLogFactory);
}
- static class StoreGroup
+ interface Fold<I1, I2, O>
{
- final CommandStore[] stores;
- final KeyRanges ranges;
+ O fold(CommandStore store, I1 i1, I2 i2, O accumulator);
+ }
+
+ interface Select<Scope>
+ {
+ long select(ShardedRanges ranges, Scope scope, long minEpoch, long maxEpoch);
+ }
- public StoreGroup(CommandStore[] stores, KeyRanges ranges)
+ static class KeysAndEpoch
+ {
+ final Keys keys;
+ final long epoch;
+
+ KeysAndEpoch(Keys keys, long epoch)
{
- Preconditions.checkArgument(stores.length <= 64);
- this.stores = stores;
- this.ranges = ranges;
+ this.keys = keys;
+ this.epoch = epoch;
}
+ }
- long all()
+ static class KeysAndEpochRange
+ {
+ final Keys keys;
+ final long minEpoch;
+ final long maxEpoch;
+
+ KeysAndEpochRange(Keys keys, long minEpoch, long maxEpoch)
{
- return -1L >>> (64 - stores.length);
+ this.keys = keys;
+ this.minEpoch = minEpoch;
+ this.maxEpoch = maxEpoch;
}
+ }
- long matches(Keys keys)
+ static class KeyAndEpoch
+ {
+ final Key key;
+ final long epoch;
+
+ KeyAndEpoch(Key key, long epoch)
{
- return keys.foldl(ranges, StoreGroup::addKeyIndex, stores.length, 0L, -1L);
+ this.key = key;
+ this.epoch = epoch;
}
+ }
- long matches(TxnRequest.Scope scope)
+ private static class Supplier
+ {
+ private final Node node;
+ private final Agent agent;
+ private final DataStore store;
+ private final ProgressLog.Factory progressLogFactory;
+ private final CommandStore.Factory shardFactory;
+ private final int numShards;
+
+ Supplier(Node node, Agent agent, DataStore store, ProgressLog.Factory progressLogFactory, CommandStore.Factory shardFactory, int numShards)
{
- return matches(scope.keys());
+ this.node = node;
+ this.agent = agent;
+ this.store = store;
+ this.progressLogFactory = progressLogFactory;
+ this.shardFactory = shardFactory;
+ this.numShards = numShards;
}
- static long keyIndex(Key key, long numShards)
+ CommandStore create(int generation, int shardIndex, RangesForEpoch rangesForEpoch)
{
- return Integer.toUnsignedLong(key.keyHash()) % numShards;
+ return shardFactory.create(generation, shardIndex, numShards, node, agent, store, progressLogFactory, rangesForEpoch);
}
- private static long addKeyIndex(Key key, long numShards, long accumulate)
+ ShardedRanges createShardedRanges(int generation, long epoch, KeyRanges ranges, RangesForEpoch rangesForEpoch)
{
- return accumulate | (1L << keyIndex(key, numShards));
+ CommandStore[] newStores = new CommandStore[numShards];
+ for (int i=0; i<numShards; i++)
+ newStores[i] = create(generation, i, rangesForEpoch);
+
+ return new ShardedRanges(newStores, epoch, ranges);
}
}
- static class StoreGroups
+ static class ShardedRanges
{
- final StoreGroup[] groups;
- final Topology global;
- final Topology local;
- final int size;
+ final CommandStore[] shards;
+ final long[] epochs;
+ final KeyRanges[] ranges;
- public StoreGroups(StoreGroup[] groups, Topology global, Topology local)
+ public ShardedRanges(CommandStore[] shards, long epoch, KeyRanges ranges)
{
- this.groups = groups;
- this.global = global;
- this.local = local;
- int size = 0;
- for (StoreGroup group : groups)
- size += group.stores.length;
- this.size = size;
+ Preconditions.checkArgument(shards.length <= 64);
+ this.shards = shards;
+ this.epochs = new long[] { epoch };
+ this.ranges = new KeyRanges[] { ranges };
}
- StoreGroups withNewTopology(Topology global, Topology local)
+ private ShardedRanges(CommandStore[] shards, long[] epochs, KeyRanges[] ranges)
{
- return new StoreGroups(groups, global, local);
+ Preconditions.checkArgument(shards.length <= 64);
+ this.shards = shards;
+ this.epochs = epochs;
+ this.ranges = ranges;
}
- public int size()
+ ShardedRanges withRanges(long epoch, KeyRanges ranges)
{
- return size;
+ long[] newEpochs = Arrays.copyOf(this.epochs, this.epochs.length + 1);
+ KeyRanges[] newRanges = Arrays.copyOf(this.ranges, this.ranges.length + 1);
+ newEpochs[this.epochs.length] = epoch;
+ newRanges[this.ranges.length] = ranges;
+ return new ShardedRanges(shards, newEpochs, newRanges);
}
- private <I1, I2, O> O foldl(int startGroup, long bitset, Fold<? super I1, ? super I2, O> fold, I1 param1, I2 param2, O accumulator)
+ KeyRanges rangesForEpoch(long epoch)
{
- int groupIndex = startGroup;
- StoreGroup group = groups[groupIndex];
- int offset = 0;
- while (true)
- {
- int i = Long.numberOfTrailingZeros(bitset) - offset;
- while (i < group.stores.length)
- {
- accumulator = fold.fold(group.stores[i], param1, param2, accumulator);
- bitset ^= Long.lowestOneBit(bitset);
- i = Long.numberOfTrailingZeros(bitset) - offset;
- }
+ int i = Arrays.binarySearch(epochs, epoch);
+ if (i < 0) i = -2 -i;
+ if (i < 0) return null;
+ return ranges[i];
+ }
- if (++groupIndex == groups.length)
- break;
+ KeyRanges rangesSinceEpoch(long epoch)
+ {
+ int i = Arrays.binarySearch(epochs, epoch);
+ if (i < 0) i = Math.max(0, -2 -i);
+ KeyRanges result = ranges[i++];
+ while (i < ranges.length)
+ result = ranges[i++].union(result);
+ return result;
+ }
- if (bitset == 0)
- break;
+ int indexForEpoch(long epoch)
+ {
+ int i = Arrays.binarySearch(epochs, epoch);
+ if (i < 0) i = -2 -i;
+ return i;
+ }
- offset += group.stores.length;
- group = groups[groupIndex];
- if (offset + group.stores.length > 64)
- break;
- }
- return accumulator;
+ long all()
+ {
+ return -1L >>> (64 - shards.length);
}
- interface Fold<I1, I2, O>
+ long shards(Keys keys, long minEpoch, long maxEpoch)
{
- O fold(CommandStore store, I1 i1, I2 i2, O accumulator);
+ long accumulate = 0L;
+ for (int i = Math.max(0, indexForEpoch(minEpoch)), maxi = indexForEpoch(maxEpoch); i <= maxi ; ++i)
+ {
+ accumulate = keys.foldl(ranges[i], ShardedRanges::addKeyIndex, shards.length, accumulate, -1L);
+ }
+ return accumulate;
}
- <S, I1, I2, O> O foldl(ToLongBiFunction<StoreGroup, S> select, S scope, Fold<? super I1, ? super I2, O> fold, I1 param1, I2 param2, IntFunction<? extends O> factory)
+ long shard(Key scope, long minEpoch, long maxEpoch)
{
- O accumulator = null;
- int startGroup = 0;
- while (startGroup < groups.length)
+ long result = 0L;
+ for (int i = Math.max(0, indexForEpoch(minEpoch)), maxi = indexForEpoch(maxEpoch); i <= maxi ; ++i)
{
- long bits = select.applyAsLong(groups[startGroup], scope);
- if (bits == 0)
- {
- ++startGroup;
+ int index = ranges[i].rangeIndexForKey(scope);
+ if (index < 0)
continue;
- }
+ result = addKeyIndex(scope, shards.length, result);
+ }
+ return result;
+ }
- int offset = groups[startGroup].stores.length;
- int endGroup = startGroup + 1;
- while (endGroup < groups.length)
- {
- StoreGroup group = groups[endGroup];
- if (offset + group.stores.length > 64)
- break;
+ KeyRanges currentRanges()
+ {
+ return ranges[ranges.length - 1];
+ }
- bits += select.applyAsLong(group, scope) << offset;
- offset += group.stores.length;
- ++endGroup;
- }
+ static long keyIndex(Key key, long numShards)
+ {
+ return Integer.toUnsignedLong(key.keyHash()) % numShards;
+ }
- if (accumulator == null)
- accumulator = factory.apply(Long.bitCount(bits));
+ private static long addKeyIndex(Key key, long numShards, long accumulate)
+ {
+ return accumulate | (1L << keyIndex(key, numShards));
+ }
+ }
- accumulator = foldl(startGroup, bits, fold, param1, param2, accumulator);
- startGroup = endGroup;
- }
+ static class Snapshot
+ {
+ final ShardedRanges[] ranges;
+ final Topology global;
+ final Topology local;
+ final int size;
- return accumulator;
+ Snapshot(ShardedRanges[] ranges, Topology global, Topology local)
+ {
+ this.ranges = ranges;
+ this.global = global;
+ this.local = local;
+ int size = 0;
+ for (ShardedRanges group : ranges)
+ size += group.shards.length;
+ this.size = size;
}
}
- private final Node.Id node;
- private final Function<Timestamp, Timestamp> uniqueNow;
- private final Agent agent;
- private final Store store;
- private final CommandStore.Factory shardFactory;
- private final int numShards;
- protected volatile StoreGroups groups = new StoreGroups(new StoreGroup[0], Topology.EMPTY, Topology.EMPTY);
+ final Supplier supplier;
+ volatile Snapshot current;
- public CommandStores(int num, Node.Id node, Function<Timestamp, Timestamp> uniqueNow, Agent agent, Store store, CommandStore.Factory shardFactory)
+ private CommandStores(Supplier supplier)
{
- this.node = node;
- this.numShards = num;
- this.uniqueNow = uniqueNow;
- this.agent = agent;
- this.store = store;
- this.shardFactory = shardFactory;
+ this.supplier = supplier;
+ this.current = new Snapshot(new ShardedRanges[0], Topology.EMPTY, Topology.EMPTY);
}
- private CommandStore createCommandStore(int generation, int index, KeyRanges ranges)
+ public CommandStores(int num, Node node, Agent agent, DataStore store,
+ ProgressLog.Factory progressLogFactory, CommandStore.Factory shardFactory)
{
- return shardFactory.create(generation, index, numShards, node, uniqueNow, agent, store, ranges, this::getLocalTopology);
+ this(new Supplier(node, agent, store, progressLogFactory, shardFactory, num));
}
- private Topology getLocalTopology()
+ public Topology local()
{
- return groups.local;
+ return current.local;
+ }
+
+ public Topology global()
+ {
+ return current.global;
+ }
+
+ private Snapshot updateTopology(Snapshot prev, Topology newTopology)
+ {
+ Preconditions.checkArgument(!newTopology.isSubset(), "Use full topology for CommandStores.updateTopology");
+
+ long epoch = newTopology.epoch();
+ if (epoch <= prev.global.epoch())
+ return prev;
+
+ Topology newLocalTopology = newTopology.forNode(supplier.node.id());
+ KeyRanges added = newLocalTopology.ranges().difference(prev.local.ranges());
+ KeyRanges subtracted = prev.local.ranges().difference(newLocalTopology.ranges());
+// for (ShardedRanges range : stores.ranges)
+// {
+// // FIXME: remove this (and the corresponding check in TopologyRandomizer) once lower bounds are implemented.
+// // In the meantime, the logic needed to support acquiring ranges that we previously replicated is pretty
+// // convoluted without the ability to jettison epochs.
+// Preconditions.checkState(!range.ranges.intersects(added));
+// }
+
+ if (added.isEmpty() && subtracted.isEmpty())
+ return new Snapshot(prev.ranges, newTopology, newLocalTopology);
+
+ ShardedRanges[] result = new ShardedRanges[prev.ranges.length + (added.isEmpty() ? 0 : 1)];
+ if (subtracted.isEmpty())
+ {
+ int newGeneration = prev.ranges.length;
+ System.arraycopy(prev.ranges, 0, result, 0, newGeneration);
+ result[newGeneration] = supplier.createShardedRanges(newGeneration, epoch, added, rangesForEpochFunction(newGeneration));
+ }
+ else
+ {
+ int i = 0;
+ while (i < prev.ranges.length)
+ {
+ ShardedRanges ranges = prev.ranges[i];
+ if (ranges.currentRanges().intersects(subtracted))
+ ranges = ranges.withRanges(newTopology.epoch(), ranges.currentRanges().difference(subtracted));
+ result[i++] = ranges;
+ }
+ if (i < result.length)
+ result[i] = supplier.createShardedRanges(i, epoch, added, rangesForEpochFunction(i));
+ }
+
+ return new Snapshot(result, newTopology, newLocalTopology);
+ }
+
+ private RangesForEpoch rangesForEpochFunction(int generation)
+ {
+ return new RangesForEpoch()
+ {
+
+ @Override
+ public KeyRanges at(long epoch)
+ {
+ return current.ranges[generation].rangesForEpoch(epoch);
+ }
+
+ @Override
+ public KeyRanges since(long epoch)
+ {
+ return current.ranges[generation].rangesSinceEpoch(epoch);
+ }
+ };
}
public synchronized void shutdown()
{
- for (StoreGroup group : groups.groups)
- for (CommandStore commandStore : group.stores)
+ for (ShardedRanges group : current.ranges)
+ for (CommandStore commandStore : group.shards)
commandStore.shutdown();
}
- protected abstract <S> void forEach(ToLongBiFunction<StoreGroup, S> select, S scope, Consumer<? super CommandStore> forEach);
- protected abstract <S, T> T mapReduce(ToLongBiFunction<StoreGroup, S> select, S scope, Function<? super CommandStore, T> map, BiFunction<T, T, T> reduce);
+ protected abstract <S> void forEach(Select<S> select, S scope, long minEpoch, long maxEpoch, Consumer<? super CommandStore> forEach);
+ protected abstract <S, T> T mapReduce(Select<S> select, S scope, long minEpoch, long maxEpoch, Function<? super CommandStore, T> map, BiFunction<T, T, T> reduce);
public void forEach(Consumer<CommandStore> forEach)
{
- forEach((s, i) -> s.all(), null, forEach);
+ forEach((s, i, min, max) -> s.all(), null, 0, 0, forEach);
+ }
+
+ public void forEach(Keys keys, long epoch, Consumer<CommandStore> forEach)
+ {
+ forEach(keys, epoch, epoch, forEach);
+ }
+
+ public void forEach(Keys keys, long minEpoch, long maxEpoch, Consumer<CommandStore> forEach)
+ {
+ forEach(ShardedRanges::shards, keys, minEpoch, maxEpoch, forEach);
+ }
+
+ public <T> T mapReduce(Keys keys, long epoch, Function<CommandStore, T> map, BiFunction<T, T, T> reduce)
+ {
+ return mapReduce(keys, epoch, epoch, map, reduce);
}
- public void forEach(Keys keys, Consumer<CommandStore> forEach)
+ public <T> T mapReduce(Keys keys, long minEpoch, long maxEpoch, Function<CommandStore, T> map, BiFunction<T, T, T> reduce)
{
- forEach(StoreGroup::matches, keys, forEach);
+ return mapReduce(ShardedRanges::shards, keys, minEpoch, maxEpoch, map, reduce);
}
- public void forEach(TxnRequest.Scope scope, Consumer<CommandStore> forEach)
+ public <T> T mapReduce(Key key, long epoch, Function<CommandStore, T> map, BiFunction<T, T, T> reduce)
{
- forEach(StoreGroup::matches, scope, forEach);
+ return mapReduce(ShardedRanges::shard, key, epoch, epoch, map, reduce);
}
- public <T> T mapReduce(TxnRequest.Scope scope, Function<CommandStore, T> map, BiFunction<T, T, T> reduce)
+ public <T extends Collection<CommandStore>> T collect(Keys keys, long epoch, IntFunction<T> factory)
{
- return mapReduce(StoreGroup::matches, scope, map, reduce);
+ return foldl(ShardedRanges::shards, keys, epoch, epoch, CommandStores::append, null, null, factory);
}
- public <T extends Collection<CommandStore>> T collect(Keys keys, IntFunction<T> factory)
+ public <T extends Collection<CommandStore>> T collect(Keys keys, long minEpoch, long maxEpoch, IntFunction<T> factory)
{
- return groups.foldl(StoreGroup::matches, keys, CommandStores::append, null, null, factory);
+ return foldl(ShardedRanges::shards, keys, minEpoch, maxEpoch, CommandStores::append, null, null, factory);
}
- public <T extends Collection<CommandStore>> T collect(TxnRequest.Scope scope, IntFunction<T> factory)
+ public synchronized void updateTopology(Topology newTopology)
{
- return groups.foldl(StoreGroup::matches, scope, CommandStores::append, null, null, factory);
+ current = updateTopology(current, newTopology);
}
private static <T extends Collection<CommandStore>> T append(CommandStore store, Object ignore1, Object ignore2, T to)
@@ -244,52 +387,82 @@ public abstract class CommandStores
return to;
}
- public synchronized void updateTopology(Topology cluster)
+ private <I1, I2, O> O foldl(int startGroup, long bitset, Fold<? super I1, ? super I2, O> fold, I1 param1, I2 param2, O accumulator)
{
- Preconditions.checkArgument(!cluster.isSubset(), "Use full topology for CommandStores.updateTopology");
+ ShardedRanges[] ranges = current.ranges;
+ int groupIndex = startGroup;
+ ShardedRanges group = ranges[groupIndex];
+ int offset = 0;
+ while (true)
+ {
+ int i = Long.numberOfTrailingZeros(bitset) - offset;
+ while (i < group.shards.length)
+ {
+ accumulator = fold.fold(group.shards[i], param1, param2, accumulator);
+ bitset ^= Long.lowestOneBit(bitset);
+ i = Long.numberOfTrailingZeros(bitset) - offset;
+ }
- StoreGroups current = groups;
- if (cluster.epoch() <= current.global.epoch())
- return;
+ if (++groupIndex == ranges.length)
+ break;
- Topology local = cluster.forNode(node);
- KeyRanges added = local.ranges().difference(current.local.ranges());
+ if (bitset == 0)
+ break;
- for (StoreGroup group : groups.groups)
- {
- // FIXME: remove this (and the corresponding check in TopologyRandomizer) once lower bounds are implemented.
- // In the meantime, the logic needed to support acquiring ranges that we previously replicated is pretty
- // convoluted without the ability to jettison epochs.
- Preconditions.checkState(!group.ranges.intersects(added));
+ offset += group.shards.length;
+ group = ranges[groupIndex];
+ if (offset + group.shards.length > 64)
+ break;
}
+ return accumulator;
+ }
- if (added.isEmpty())
+ <S, I1, I2, O> O foldl(Select<S> select, S scope, long minEpoch, long maxEpoch, Fold<? super I1, ? super I2, O> fold, I1 param1, I2 param2, IntFunction<? extends O> factory)
+ {
+ ShardedRanges[] ranges = current.ranges;
+ O accumulator = null;
+ int startGroup = 0;
+ while (startGroup < ranges.length)
{
- groups = groups.withNewTopology(cluster, local);
- return;
- }
+ long bits = select.select(ranges[startGroup], scope, minEpoch, maxEpoch);
+ if (bits == 0)
+ {
+ ++startGroup;
+ continue;
+ }
- int newGeneration = current.groups.length;
- StoreGroup[] newGroups = new StoreGroup[current.groups.length + 1];
- CommandStore[] newStores = new CommandStore[numShards];
- System.arraycopy(current.groups, 0, newGroups, 0, current.groups.length);
+ int offset = ranges[startGroup].shards.length;
+ int endGroup = startGroup + 1;
+ while (endGroup < ranges.length)
+ {
+ ShardedRanges group = ranges[endGroup];
+ if (offset + group.shards.length > 64)
+ break;
- for (int i=0; i<numShards; i++)
- newStores[i] = createCommandStore(newGeneration, i, added);
+ bits += select.select(group, scope, minEpoch, maxEpoch) << offset;
+ offset += group.shards.length;
+ ++endGroup;
+ }
+
+ if (accumulator == null)
+ accumulator = factory.apply(Long.bitCount(bits));
- newGroups[current.groups.length] = new StoreGroup(newStores, added);
+ accumulator = foldl(startGroup, bits, fold, param1, param2, accumulator);
+ startGroup = endGroup;
+ }
- groups = new StoreGroups(newGroups, cluster, local);
+ return accumulator;
}
@VisibleForTesting
public CommandStore unsafeForKey(Key key)
{
- for (StoreGroup group : groups.groups)
+ ShardedRanges[] ranges = current.ranges;
+ for (ShardedRanges group : ranges)
{
- if (group.ranges.contains(key))
+ if (group.currentRanges().contains(key))
{
- for (CommandStore store : group.stores)
+ for (CommandStore store : group.shards)
{
if (store.hashIntersects(key))
return store;
@@ -301,39 +474,47 @@ public abstract class CommandStores
public static class Synchronized extends CommandStores
{
- public Synchronized(int num, Id node, Function<Timestamp, Timestamp> uniqueNow, Agent agent, Store store)
+ public Synchronized(int num, Node node, Agent agent, DataStore store, ProgressLog.Factory progressLogFactory)
{
- super(num, node, uniqueNow, agent, store, CommandStore.Synchronized::new);
+ super(num, node, agent, store, progressLogFactory, CommandStore.Synchronized::new);
+ }
+
+ public Synchronized(Supplier supplier)
+ {
+ super(supplier);
}
@Override
- protected <S, T> T mapReduce(ToLongBiFunction<StoreGroup, S> select, S scope, Function<? super CommandStore, T> map, BiFunction<T, T, T> reduce)
+ protected <S, T> T mapReduce(Select<S> select, S scope, long minEpoch, long maxEpoch, Function<? super CommandStore, T> map, BiFunction<T, T, T> reduce)
{
- return groups.foldl(select, scope, (store, f, r, t) -> t == null ? f.apply(store) : r.apply(t, f.apply(store)), map, reduce, ignore -> null);
+ return foldl(select, scope, minEpoch, maxEpoch, (store, f, r, t) -> t == null ? f.apply(store) : r.apply(t, f.apply(store)), map, reduce, ignore -> null);
}
@Override
- protected <S> void forEach(ToLongBiFunction<StoreGroup, S> select, S scope, Consumer<? super CommandStore> forEach)
+ protected <S> void forEach(Select<S> select, S scope, long minEpoch, long maxEpoch, Consumer<? super CommandStore> forEach)
{
- groups.foldl(select, scope, (store, f, r, t) -> { f.accept(store); return null; }, forEach, null, ignore -> FALSE);
+ foldl(select, scope, minEpoch, maxEpoch, (store, f, r, t) -> { f.accept(store); return null; }, forEach, null, ignore -> FALSE);
}
}
public static class SingleThread extends CommandStores
{
- public SingleThread(int num, Id node, Function<Timestamp, Timestamp> uniqueNow, Agent agent, Store store)
+ public SingleThread(int num, Node node, Agent agent, DataStore store, ProgressLog.Factory progressLogFactory)
{
- this(num, node, uniqueNow, agent, store, CommandStore.SingleThread::new);
+ this(num, node, agent, store, progressLogFactory, CommandStore.SingleThread::new);
}
- public SingleThread(int num, Node.Id node, Function<Timestamp, Timestamp> uniqueNow, Agent agent, Store store, CommandStore.Factory shardFactory)
+ public SingleThread(int num, Node node, Agent agent, DataStore store, ProgressLog.Factory progressLogFactory, CommandStore.Factory shardFactory)
{
- super(num, node, uniqueNow, agent, store, shardFactory);
+ super(num, node, agent, store, progressLogFactory, shardFactory);
}
- private <S, F, T> T mapReduce(ToLongBiFunction<StoreGroup, S> select, S scope, F f, Fold<F, ?, List<Future<T>>> fold, BiFunction<T, T, T> reduce)
+ private <S, F, T> T mapReduce(Select<S> select, S scope, long minEpoch, long maxEpoch, F f, Fold<F, ?, List<Future<T>>> fold, BiFunction<T, T, T> reduce)
{
- List<Future<T>> futures = groups.foldl(select, scope, fold, f, null, ArrayList::new);
+ List<Future<T>> futures = foldl(select, scope, minEpoch, maxEpoch, fold, f, null, ArrayList::new);
+ if (futures == null)
+ return null;
+
T result = null;
for (Future<T> future : futures)
{
@@ -356,22 +537,22 @@ public abstract class CommandStores
}
@Override
- protected <S, T> T mapReduce(ToLongBiFunction<StoreGroup, S> select, S scope, Function<? super CommandStore, T> map, BiFunction<T, T, T> reduce)
+ protected <S, T> T mapReduce(Select<S> select, S scope, long minEpoch, long maxEpoch, Function<? super CommandStore, T> map, BiFunction<T, T, T> reduce)
{
- return mapReduce(select, scope, map, (store, f, i, t) -> { t.add(store.process(f)); return t; }, reduce);
+ return mapReduce(select, scope, minEpoch, maxEpoch, map, (store, f, i, t) -> { t.add(store.process(f)); return t; }, reduce);
}
- protected <S> void forEach(ToLongBiFunction<StoreGroup, S> select, S scope, Consumer<? super CommandStore> forEach)
+ protected <S> void forEach(Select<S> select, S scope, long minEpoch, long maxEpoch, Consumer<? super CommandStore> forEach)
{
- mapReduce(select, scope, forEach, (store, f, i, t) -> { t.add(store.process(f)); return t; }, (Void i1, Void i2) -> null);
+ mapReduce(select, scope, minEpoch, maxEpoch, forEach, (store, f, i, t) -> { t.add(store.process(f)); return t; }, (Void i1, Void i2) -> null);
}
}
public static class Debug extends SingleThread
{
- public Debug(int num, Id node, Function<Timestamp, Timestamp> uniqueNow, Agent agent, Store store)
+ public Debug(int num, Node node, Agent agent, DataStore store, ProgressLog.Factory progressLogFactory)
{
- super(num, node, uniqueNow, agent, store, CommandStore.Debug::new);
+ super(num, node, agent, store, progressLogFactory, CommandStore.Debug::new);
}
}
diff --git a/accord-core/src/main/java/accord/local/Node.java b/accord-core/src/main/java/accord/local/Node.java
index e70e9f1..e36b070 100644
--- a/accord-core/src/main/java/accord/local/Node.java
+++ b/accord-core/src/main/java/accord/local/Node.java
@@ -2,7 +2,6 @@ package accord.local;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.BiFunction;
import java.util.function.Consumer;
@@ -16,13 +15,26 @@ import com.google.common.annotations.VisibleForTesting;
import accord.api.*;
import accord.coordinate.Coordinate;
import accord.messages.*;
+
+import javax.annotation.Nullable;
+
+import accord.api.Agent;
+import accord.api.Key;
+import accord.api.MessageSink;
+import accord.api.Result;
+import accord.api.ProgressLog;
+import accord.api.Scheduler;
+import accord.api.DataStore;
import accord.messages.Callback;
import accord.messages.ReplyContext;
import accord.messages.Request;
import accord.messages.Reply;
+import accord.topology.KeyRange;
+import accord.topology.KeyRanges;
import accord.topology.Shard;
import accord.topology.Topology;
import accord.topology.TopologyManager;
+import accord.txn.Ballot;
import accord.txn.Keys;
import accord.txn.Timestamp;
import accord.txn.Txn;
@@ -72,40 +84,47 @@ public class Node implements ConfigurationService.Listener
}
}
+ public boolean isCoordinating(TxnId txnId, Ballot promised)
+ {
+ return promised.node.equals(id) && coordinating.containsKey(txnId);
+ }
+
public static int numCommandShards()
{
return 8; // TODO: make configurable
}
- private final CommandStores commandStores;
private final Id id;
private final MessageSink messageSink;
private final ConfigurationService configService;
private final TopologyManager topology;
+ private final CommandStores commandStores;
private final LongSupplier nowSupplier;
private final AtomicReference<Timestamp> now;
private final Agent agent;
+ private final Random random;
// TODO: this really needs to be thought through some more, as it needs to be per-instance in some cases, and per-node in others
private final Scheduler scheduler;
private final Map<TxnId, Future<Result>> coordinating = new ConcurrentHashMap<>();
- private final Set<TxnId> pendingRecovery = Collections.newSetFromMap(new ConcurrentHashMap<>());
public Node(Id id, MessageSink messageSink, ConfigurationService configService, LongSupplier nowSupplier,
- Supplier<Store> dataSupplier, Agent agent, Scheduler scheduler, CommandStores.Factory factory)
+ Supplier<DataStore> dataSupplier, Agent agent, Random random, Scheduler scheduler,
+ Function<Node, ProgressLog.Factory> progressLogFactory, CommandStores.Factory factory)
{
this.id = id;
- this.agent = agent;
this.messageSink = messageSink;
this.configService = configService;
this.topology = new TopologyManager(id, configService::reportEpoch);
+ this.nowSupplier = nowSupplier;
Topology topology = configService.currentTopology();
this.now = new AtomicReference<>(new Timestamp(topology.epoch(), nowSupplier.getAsLong(), 0, id));
- this.nowSupplier = nowSupplier;
+ this.agent = agent;
+ this.random = random;
this.scheduler = scheduler;
- this.commandStores = factory.create(numCommandShards(), id, this::uniqueNow, agent, dataSupplier.get());
+ this.commandStores = factory.create(numCommandShards(), this, agent, dataSupplier.get(), progressLogFactory.apply(this));
configService.registerListener(this);
onTopologyUpdate(topology, false);
@@ -192,34 +211,49 @@ public class Node implements ConfigurationService.Listener
commandStores.forEach(forEach);
}
- public void forEachLocal(Keys keys, Consumer<CommandStore> forEach)
+ public void forEachLocal(Keys keys, long epoch, Consumer<CommandStore> forEach)
+ {
+ commandStores.forEach(keys, epoch, forEach);
+ }
+
+ public void forEachLocal(Keys keys, long minEpoch, long maxEpoch, Consumer<CommandStore> forEach)
+ {
+ commandStores.forEach(keys, minEpoch, maxEpoch, forEach);
+ }
+
+ public void forEachLocalSince(Keys keys, Timestamp since, Consumer<CommandStore> forEach)
+ {
+ commandStores.forEach(keys, since.epoch, Long.MAX_VALUE, forEach);
+ }
+
+ public void forEachLocalSince(Keys keys, long sinceEpoch, Consumer<CommandStore> forEach)
{
- commandStores.forEach(keys, forEach);
+ commandStores.forEach(keys, sinceEpoch, Long.MAX_VALUE, forEach);
}
- public void forEachLocal(Txn txn, Consumer<CommandStore> forEach)
+ public <T> T mapReduceLocal(Keys keys, long minEpoch, long maxEpoch, Function<CommandStore, T> map, BiFunction<T, T, T> reduce)
{
- forEachLocal(txn.keys, forEach);
+ return commandStores.mapReduce(keys, minEpoch, maxEpoch, map, reduce);
}
- public void forEachLocal(TxnRequest.Scope scope, Consumer<CommandStore> forEach)
+ public <T> T mapReduceLocalSince(Keys keys, Timestamp since, Function<CommandStore, T> map, BiFunction<T, T, T> reduce)
{
- commandStores.forEach(scope, forEach);
+ return commandStores.mapReduce(keys, since.epoch, Long.MAX_VALUE, map, reduce);
}
- public <T> T mapReduceLocal(TxnRequest.Scope scope, Function<CommandStore, T> map, BiFunction<T, T, T> reduce)
+ public <T> T ifLocal(Key key, Timestamp at, Function<CommandStore, T> ifLocal)
{
- return commandStores.mapReduce(scope, map, reduce);
+ return ifLocal(key, at.epoch, ifLocal);
}
- public <T extends Collection<CommandStore>> T collectLocal(Keys keys, IntFunction<T> factory)
+ public <T> T ifLocal(Key key, long epoch, Function<CommandStore, T> ifLocal)
{
- return commandStores.collect(keys, factory);
+ return commandStores.mapReduce(key, epoch, ifLocal, (a, b) -> { throw new IllegalStateException();} );
}
- public <T extends Collection<CommandStore>> T collectLocal(TxnRequest.Scope scope, IntFunction<T> factory)
+ public <T extends Collection<CommandStore>> T collectLocal(Keys keys, Timestamp at, IntFunction<T> factory)
{
- return commandStores.collect(scope, factory);
+ return commandStores.collect(keys, at.epoch, factory);
}
// send to every node besides ourselves
@@ -234,6 +268,11 @@ public class Node implements ConfigurationService.Listener
shard.nodes.forEach(node -> messageSink.send(node, send));
}
+ public void send(Shard shard, Request send, Callback callback)
+ {
+ shard.nodes.forEach(node -> messageSink.send(node, send, callback));
+ }
+
private <T> void send(Shard shard, Request send, Set<Id> alreadyContacted)
{
shard.nodes.forEach(node -> {
@@ -278,6 +317,8 @@ public class Node implements ConfigurationService.Listener
public void reply(Id replyingToNode, ReplyContext replyContext, Reply send)
{
+ if (send == null)
+ throw new NullPointerException();
messageSink.reply(replyingToNode, replyContext, send);
}
@@ -286,61 +327,118 @@ public class Node implements ConfigurationService.Listener
return new TxnId(uniqueNow());
}
+ public Future<Result> coordinate(Txn txn)
+ {
+ return coordinate(nextTxnId(), txn);
+ }
+
public Future<Result> coordinate(TxnId txnId, Txn txn)
{
// TODO: The combination of updating the epoch of the next timestamp with epochs we don’t have topologies for,
// and requiring preaccept to talk to its topology epoch means that learning of a new epoch via timestamp
// (ie not via config service) will halt any new txns from a node until it receives this topology
+ Future<Result> result;
if (txnId.epoch > topology().epoch())
{
configService.fetchTopologyForEpoch(txnId.epoch);
- return topology().awaitEpoch(txnId.epoch).flatMap(v -> coordinate(txnId, txn));
+ result = topology().awaitEpoch(txnId.epoch)
+ .flatMap(ignore -> initiateCoordination(txnId, txn));
+ }
+ else
+ {
+ result = initiateCoordination(txnId, txn);
}
- Future<Result> result = Coordinate.execute(this, txnId, txn);
- coordinating.put(txnId, result);
- // TODO (now): error handling
- result.addCallback((success, fail) ->
- {
- coordinating.remove(txnId);
- // if we don't succeed, try again in 30s to make sure somebody finishes it
- // TODO: this is an ugly liveness mechanism
- if (fail != null && pendingRecovery.add(txnId))
- scheduler.once(() -> { pendingRecovery.remove(txnId); recover(txnId, txn); } , 30L, TimeUnit.SECONDS);
- });
+ coordinating.putIfAbsent(txnId, result);
+ // TODO: if we fail, nominate another node to try instead
+ result.addCallback((success, fail) -> coordinating.remove(txnId, result));
return result;
}
- public Future<Result> coordinate(Txn txn)
+ private Future<Result> initiateCoordination(TxnId txnId, Txn txn)
{
- return coordinate(nextTxnId(), txn);
+ Key homeKey = trySelectHomeKey(txnId, txn.keys);
+ if (homeKey == null)
+ {
+ homeKey = selectRandomHomeKey(txnId);
+ txn = new Txn(txn.keys.with(homeKey), txn.read, txn.query, txn.update);
+ }
+ return Coordinate.execute(this, txnId, txn, homeKey);
+ }
+
+ @VisibleForTesting
+ public @Nullable Key trySelectHomeKey(TxnId txnId, Keys keys)
+ {
+ int i = topology().localForEpoch(txnId.epoch).ranges().findFirstIntersecting(keys);
+ return i >= 0 ? keys.get(i) : null;
+ }
+
+ public Key selectHomeKey(TxnId txnId, Keys keys)
+ {
+ Key key = trySelectHomeKey(txnId, keys);
+ return key != null ? key : selectRandomHomeKey(txnId);
+ }
+
+ public Key selectProgressKey(TxnId txnId, Keys keys, Key homeKey)
+ {
+ Key progressKey = trySelectProgressKey(txnId, keys, homeKey);
+ if (progressKey == null)
+ throw new IllegalStateException();
+ return progressKey;
+ }
+
+ public Key trySelectProgressKey(TxnId txnId, Keys keys, Key homeKey)
+ {
+ return trySelectProgressKey(txnId.epoch, keys, homeKey);
+ }
+
+ public Key trySelectProgressKey(long epoch, Keys keys, Key homeKey)
+ {
+ Topology topology = this.topology.localForEpoch(epoch);
+ if (topology.ranges().contains(homeKey))
+ return homeKey;
+
+ int i = topology.ranges().findFirstIntersecting(keys);
+ if (i < 0)
+ return null;
+ return keys.get(i);
+ }
+
+ public Key selectRandomHomeKey(TxnId txnId)
+ {
+ KeyRanges ranges = topology().localForEpoch(txnId.epoch).ranges();
+ KeyRange range = ranges.get(ranges.size() == 1 ? 0 : random.nextInt(ranges.size()));
+ return range.endInclusive() ? range.end() : range.start();
}
// TODO: encapsulate in Coordinate, so we can request that e.g. commits be re-sent?
- public void recover(TxnId txnId, Txn txn)
+ public Future<Result> recover(TxnId txnId, Txn txn, Key homeKey)
{
+ {
+ Future<Result> result = coordinating.get(txnId);
+ if (result != null)
+ return result;
+ }
+
+ Future<Result> result;
if (txnId.epoch > topology.epoch())
{
configService.fetchTopologyForEpoch(txnId.epoch);
- topology().awaitEpoch(txnId.epoch).addListener(() -> recover(txnId, txn));
- return;
+ result = topology().awaitEpoch(txnId.epoch)
+ .flatMap(ignore -> Coordinate.recover(this, txnId, txn, homeKey));
+ }
+ else
+ {
+ result = Coordinate.recover(this, txnId, txn, homeKey);
}
- Future<Result> result = coordinating.get(txnId);
- if (result != null)
- return;
-
- result = Coordinate.recover(this, txnId, txn);
coordinating.putIfAbsent(txnId, result);
- // TODO (now): error handling
result.addCallback((success, fail) -> {
- coordinating.remove(txnId);
- agent.onRecover(this, success, fail);
- // if we don't succeed, try again in 30s to make sure somebody finishes it
- // TODO: this is an ugly liveness mechanism
- if (fail != null && pendingRecovery.add(txnId))
- scheduler.once(() -> { pendingRecovery.remove(txnId); recover(txnId, txn); } , 30L, TimeUnit.SECONDS);
+ coordinating.remove(txnId, result);
+ agent.onRecover(this, success, fail);
+ // TODO: if we fail, nominate another node to try instead
});
+ return result;
}
public void receive(Request request, Id from, ReplyContext replyContext)
@@ -355,6 +453,16 @@ public class Node implements ConfigurationService.Listener
scheduler.now(() -> request.process(this, from, replyContext));
}
+ public boolean isReplicaOf(Timestamp at, Key key)
+ {
+ return topology().localForEpoch(at.epoch).ranges().contains(key);
+ }
+
+ public boolean isReplicaOf(long epoch, Key key)
+ {
+ return topology().localForEpoch(epoch).ranges().contains(key);
+ }
+
public Scheduler scheduler()
{
return scheduler;
@@ -382,4 +490,13 @@ public class Node implements ConfigurationService.Listener
return commandStores.unsafeForKey(key);
}
+ public CommandStore unsafeByIndex(int index)
+ {
+ return commandStores.current.ranges[0].shards[index];
+ }
+
+ public LongSupplier unsafeGetNowSupplier()
+ {
+ return nowSupplier;
+ }
}
diff --git a/accord-core/src/main/java/accord/local/Status.java b/accord-core/src/main/java/accord/local/Status.java
index f19478d..a29a2b4 100644
--- a/accord-core/src/main/java/accord/local/Status.java
+++ b/accord-core/src/main/java/accord/local/Status.java
@@ -8,4 +8,9 @@ public enum Status
{
return a.compareTo(b) >= 0 ? a : b;
}
+
+ public boolean hasBeen(Status equalOrGreaterThan)
+ {
+ return compareTo(equalOrGreaterThan) >= 0;
+ }
}
diff --git a/accord-core/src/main/java/accord/messages/Accept.java b/accord-core/src/main/java/accord/messages/Accept.java
index b9dc1c2..c18472d 100644
--- a/accord-core/src/main/java/accord/messages/Accept.java
+++ b/accord-core/src/main/java/accord/messages/Accept.java
@@ -1,6 +1,8 @@
package accord.messages;
+import accord.messages.TxnRequest.WithUnsynced;
import accord.topology.Topologies;
+import accord.api.Key;
import accord.txn.Ballot;
import accord.local.Node;
import accord.txn.Timestamp;
@@ -11,34 +13,33 @@ import accord.txn.TxnId;
import static accord.messages.PreAccept.calculateDeps;
-public class Accept extends TxnRequest
+public class Accept extends WithUnsynced
{
public final Ballot ballot;
- public final TxnId txnId;
public final Txn txn;
+ public final long minEpoch;
public final Timestamp executeAt;
public final Dependencies deps;
- public Accept(Scope scope, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Dependencies deps)
+ public Accept(Node.Id to, Topologies topologies, Ballot ballot, TxnId txnId, Txn txn, Key homeKey, Timestamp executeAt, Dependencies deps)
{
- super(scope);
+ super(to, topologies, txn.keys, txnId, homeKey);
this.ballot = ballot;
- this.txnId = txnId;
this.txn = txn;
+ this.minEpoch = topologies.oldestEpoch();
this.executeAt = executeAt;
this.deps = deps;
}
- public Accept(Node.Id dst, Topologies topologies, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Dependencies deps)
+ public void process(Node node, Node.Id replyToNode, ReplyContext replyContext)
{
- this(Scope.forTopologies(dst, topologies, txn), ballot, txnId, txn, executeAt, deps);
- }
-
- public void process(Node on, Node.Id replyToNode, ReplyContext replyContext)
- {
- on.reply(replyToNode, replyContext, on.mapReduceLocal(scope(), instance -> {
+ Key progressKey = progressKey(node);
+ // TODO: when we begin expunging old epochs we need to ensure we handle the case where we do not fully handle the keys;
+ // since this will likely imply the transaction has been applied or aborted we can indicate the coordinator
+ // should enquire as to the result
+ node.reply(replyToNode, replyContext, node.mapReduceLocal(scope(), minEpoch, executeAt.epoch, instance -> {
Command command = instance.command(txnId);
- if (!command.accept(ballot, txn, executeAt, deps))
+ if (!command.accept(ballot, txn, homeKey, progressKey, executeAt, deps))
return new AcceptNack(txnId, command.promised());
return new AcceptOk(txnId, calculateDeps(instance, txnId, txn, executeAt));
}, (r1, r2) -> {
diff --git a/accord-core/src/main/java/accord/messages/Apply.java b/accord-core/src/main/java/accord/messages/Apply.java
index a917359..3f9f92c 100644
--- a/accord-core/src/main/java/accord/messages/Apply.java
+++ b/accord-core/src/main/java/accord/messages/Apply.java
@@ -1,5 +1,6 @@
package accord.messages;
+import accord.api.Key;
import accord.local.Node;
import accord.local.Node.Id;
import accord.api.Result;
@@ -10,53 +11,73 @@ import accord.txn.Writes;
import accord.txn.Txn;
import accord.txn.TxnId;
+import static accord.messages.MessageType.APPLY_REQ;
+import static accord.messages.MessageType.APPLY_RSP;
+
public class Apply extends TxnRequest
{
public final TxnId txnId;
public final Txn txn;
- // TODO: these only need to be sent if we don't know if this node has witnessed a Commit
- public final Dependencies deps;
+ protected final Key homeKey;
public final Timestamp executeAt;
+ public final Dependencies deps;
public final Writes writes;
public final Result result;
- public Apply(Scope scope, TxnId txnId, Txn txn, Timestamp executeAt, Dependencies deps, Writes writes, Result result)
+ public Apply(Node.Id to, Topologies topologies, TxnId txnId, Txn txn, Key homeKey, Timestamp executeAt, Dependencies deps, Writes writes, Result result)
{
- super(scope);
+ super(to, topologies, txn.keys);
this.txnId = txnId;
this.txn = txn;
+ this.homeKey = homeKey;
this.deps = deps;
this.executeAt = executeAt;
this.writes = writes;
this.result = result;
}
- public Apply(Node.Id to, Topologies topologies, TxnId txnId, Txn txn, Timestamp executeAt, Dependencies deps, Writes writes, Result result)
- {
- this(Scope.forTopologies(to, topologies, txn), txnId, txn, executeAt, deps, writes, result);
- }
-
public void process(Node node, Id replyToNode, ReplyContext replyContext)
{
- node.forEachLocal(scope(), instance -> instance.command(txnId).apply(txn, deps, executeAt, writes, result));
+ Key progressKey = node.trySelectProgressKey(txnId, txn.keys, homeKey);
+ node.forEachLocalSince(scope(), executeAt,
+ instance -> instance.command(txnId).apply(txn, homeKey, progressKey, executeAt, deps, writes, result));
+ node.reply(replyToNode, replyContext, ApplyOk.INSTANCE);
}
@Override
public MessageType type()
{
- return MessageType.APPLY_REQ;
+ return APPLY_REQ;
+ }
+
+ public static class ApplyOk implements Reply
+ {
+ public static final ApplyOk INSTANCE = new ApplyOk();
+ public ApplyOk() {}
+
+ @Override
+ public String toString()
+ {
+ return "ApplyOk";
+ }
+
+ @Override
+ public MessageType type()
+ {
+ return APPLY_RSP;
+ }
}
@Override
public String toString()
{
return "Apply{" +
- "txnId: " + txnId +
- ", txn: " + txn +
- ", deps: " + deps +
- ", executeAt: " + executeAt +
- ", writes: " + writes +
- ", result: " + result +
+ "txnId:" + txnId +
+ ", txn:" + txn +
+ ", deps:" + deps +
+ ", executeAt:" + executeAt +
+ ", writes:" + writes +
+ ", result:" + result +
'}';
}
}
diff --git a/accord-core/src/main/java/accord/messages/BeginRecovery.java b/accord-core/src/main/java/accord/messages/BeginRecovery.java
index bf2d6b8..9cd1e3f 100644
--- a/accord-core/src/main/java/accord/messages/BeginRecovery.java
+++ b/accord-core/src/main/java/accord/messages/BeginRecovery.java
@@ -3,6 +3,12 @@ package accord.messages;
import accord.api.ConfigurationService;
import accord.api.Result;
import accord.topology.Topologies;
+import java.util.stream.Stream;
+
+import accord.api.Key;
+import accord.local.CommandStore;
+import accord.local.CommandsForKey;
+import accord.txn.Keys;
import accord.txn.Writes;
import accord.txn.Ballot;
import accord.local.Node;
@@ -24,29 +30,27 @@ import static accord.messages.PreAccept.calculateDeps;
public class BeginRecovery extends TxnRequest
{
- public final TxnId txnId;
- public final Txn txn;
- public final Ballot ballot;
+ final TxnId txnId;
+ final Txn txn;
+ final Key homeKey;
+ final Ballot ballot;
- public BeginRecovery(Scope scope, TxnId txnId, Txn txn, Ballot ballot)
+ public BeginRecovery(Id to, Topologies topologies, TxnId txnId, Txn txn, Key homeKey, Ballot ballot)
{
- super(scope);
+ super(to, topologies, txn.keys);
this.txnId = txnId;
this.txn = txn;
+ this.homeKey = homeKey;
this.ballot = ballot;
}
- public BeginRecovery(Id to, Topologies topologies, TxnId txnId, Txn txn, Ballot ballot)
- {
- this(Scope.forTopologies(to, topologies, txn), txnId, txn, ballot);
- }
-
public void process(Node node, Id replyToNode, ReplyContext replyContext)
{
- RecoverReply reply = node.mapReduceLocal(scope(), instance -> {
+ Key progressKey = node.selectProgressKey(txnId, txn.keys, homeKey);
+ RecoverReply reply = node.mapReduceLocal(scope(), txnId.epoch, txnId.epoch, instance -> {
Command command = instance.command(txnId);
- if (!command.recover(txn, ballot))
+ if (!command.recover(txn, homeKey, progressKey, ballot))
return new RecoverNack(command.promised());
Dependencies deps = command.status() == PreAccepted ? calculateDeps(instance, txnId, txn, txnId)
@@ -62,25 +66,23 @@ public class BeginRecovery extends TxnRequest
}
else
{
- // if accepted or committed txns with a later txnid exist and they do not contain
- // our txnid as a dependency, it could not have been witnessed by a quorum
- rejectsFastPath = txn.uncommittedStartedAfter(instance, txnId)
+ rejectsFastPath = uncommittedStartedAfter(instance, txnId, txn.keys)
.filter(c -> c.hasBeen(Accepted))
.anyMatch(c -> !c.savedDeps().contains(txnId));
if (!rejectsFastPath)
- rejectsFastPath = txn.committedExecutesAfter(instance, txnId)
+ rejectsFastPath = committedExecutesAfter(instance, txnId, txn.keys)
.anyMatch(c -> !c.savedDeps().contains(txnId));
// committed txns with an earlier txnid and have our txnid as a dependency
- earlierCommittedWitness = txn.committedStartedBefore(instance, txnId)
- .filter(c -> c.savedDeps().contains(txnId))
- .collect(Dependencies::new, Dependencies::add, Dependencies::addAll);
+ earlierCommittedWitness = committedStartedBefore(instance, txnId, txn.keys)
+ .filter(c -> c.savedDeps().contains(txnId))
+ .collect(Dependencies::new, Dependencies::add, Dependencies::addAll);
// accepted txns with an earlier txnid that don't have our txnid as a dependency
- earlierAcceptedNoWitness = txn.uncommittedStartedBefore(instance, txnId)
+ earlierAcceptedNoWitness = uncommittedStartedBefore(instance, txnId, txn.keys)
.filter(c -> c.is(Accepted)
- && !c.savedDeps().contains(txnId)
- && c.executeAt().compareTo(txnId) > 0)
+ && !c.savedDeps().contains(txnId)
+ && c.executeAt().compareTo(txnId) > 0)
.collect(Dependencies::new, Dependencies::add, Dependencies::addAll);
}
return new RecoverOk(txnId, command.status(), command.accepted(), command.executeAt(), deps, earlierCommittedWitness, earlierAcceptedNoWitness, rejectsFastPath, command.writes(), command.result());
@@ -167,8 +169,8 @@ public class BeginRecovery extends TxnRequest
node.topology().awaitEpoch(ok.executeAt.epoch).addListener(() -> disseminateApply(node, ok));
return;
}
- Topologies topologies = node.topology().forKeys(txn.keys, ok.executeAt.epoch);
- node.send(topologies.nodes(), to -> new Apply(to, topologies, txnId, txn, ok.executeAt, ok.deps, ok.writes, ok.result));
+ Topologies topologies = node.topology().withUnsyncedEpochs(txn.keys, ok.executeAt.epoch);
+ node.send(topologies.nodes(), to -> new Apply(to, topologies, txnId, txn, homeKey, ok.executeAt, ok.deps, ok.writes, ok.result));
}
@Override
@@ -271,4 +273,44 @@ public class BeginRecovery extends TxnRequest
", ballot:" + ballot +
'}';
}
+
+ private static Stream<Command> uncommittedStartedBefore(CommandStore commandStore, TxnId startedBefore, Keys keys)
+ {
+ return keys.stream().flatMap(key -> {
+ CommandsForKey forKey = commandStore.maybeCommandsForKey(key);
+ if (forKey == null)
+ return Stream.of();
+ return forKey.uncommitted.headMap(startedBefore, false).values().stream();
+ });
+ }
+
+ private static Stream<Command> committedStartedBefore(CommandStore commandStore, TxnId startedBefore, Keys keys)
+ {
+ return keys.stream().flatMap(key -> {
+ CommandsForKey forKey = commandStore.maybeCommandsForKey(key);
+ if (forKey == null)
+ return Stream.of();
+ return forKey.committedById.headMap(startedBefore, false).values().stream();
+ });
+ }
+
+ private static Stream<Command> uncommittedStartedAfter(CommandStore commandStore, TxnId startedAfter, Keys keys)
+ {
+ return keys.stream().flatMap(key -> {
+ CommandsForKey forKey = commandStore.maybeCommandsForKey(key);
+ if (forKey == null)
+ return Stream.of();
+ return forKey.uncommitted.tailMap(startedAfter, false).values().stream();
+ });
+ }
+
+ private static Stream<Command> committedExecutesAfter(CommandStore commandStore, TxnId startedAfter, Keys keys)
+ {
+ return keys.stream().flatMap(key -> {
+ CommandsForKey forKey = commandStore.maybeCommandsForKey(key);
+ if (forKey == null)
+ return Stream.of();
+ return forKey.committedByExecuteAt.tailMap(startedAfter, false).values().stream();
+ });
+ }
}
diff --git a/accord-core/src/main/java/accord/messages/Callback.java b/accord-core/src/main/java/accord/messages/Callback.java
index 4c1d830..6688b66 100644
--- a/accord-core/src/main/java/accord/messages/Callback.java
+++ b/accord-core/src/main/java/accord/messages/Callback.java
@@ -2,6 +2,10 @@ package accord.messages;
import accord.local.Node.Id;
+/**
+ * Represents some execution for handling responses from messages a node has sent.
+ * TODO: associate a Callback with a CommandShard or other context for execution (for coordination, usually its home shard)
+ */
public interface Callback<T>
{
void onSuccess(Id from, T response);
diff --git a/accord-core/src/main/java/accord/messages/CheckStatus.java b/accord-core/src/main/java/accord/messages/CheckStatus.java
new file mode 100644
index 0000000..b3f6722
--- /dev/null
+++ b/accord-core/src/main/java/accord/messages/CheckStatus.java
@@ -0,0 +1,263 @@
+package accord.messages;
+
+import accord.api.Key;
+import accord.api.Result;
+import accord.local.Command;
+import accord.local.Node;
+import accord.local.Node.Id;
+import accord.local.Status;
+import accord.txn.Ballot;
+import accord.txn.Dependencies;
+import accord.txn.Timestamp;
+import accord.txn.TxnId;
+import accord.txn.Writes;
+
+public class CheckStatus implements Request
+{
+ // order is important
+ public enum IncludeInfo
+ {
+ No, OnlyIfExecuted, Always;
+ boolean include(Status status)
+ {
+ switch (this)
+ {
+ default: throw new IllegalStateException();
+ case No: return false;
+ case Always: return true;
+ case OnlyIfExecuted: return status.hasBeen(Status.Executed);
+ }
+ }
+ }
+
+ final TxnId txnId;
+ final Key key; // the key's commandStore to consult - not necessarily the homeKey
+ final long epoch;
+ final IncludeInfo includeInfo;
+
+ public CheckStatus(TxnId txnId, Key key, long epoch, IncludeInfo includeInfo)
+ {
+ this.txnId = txnId;
+ this.key = key;
+ this.epoch = epoch;
+ this.includeInfo = includeInfo;
+ }
+
+ public void process(Node node, Id replyToNode, ReplyContext replyContext)
+ {
+
+ Reply reply = node.ifLocal(key, epoch, instance -> {
+ Command command = instance.command(txnId);
+ boolean includeInfo = this.includeInfo.include(command.status());
+ if (includeInfo)
+ {
+ return (CheckStatusReply) new CheckStatusOkFull(command.status(), command.promised(), command.accepted(),
+ node.isCoordinating(txnId, command.promised()),
+ command.isGloballyPersistent(),
+ command.homeKey(),
+ command.executeAt(),
+ command.savedDeps(),
+ command.writes(),
+ command.result());
+ }
+
+ return new CheckStatusOk(command.status(), command.promised(), command.accepted(),
+ node.isCoordinating(txnId, command.promised()),
+ command.isGloballyPersistent());
+ });
+
+ if (reply == null)
+ reply = CheckStatusNack.nack();
+
+ node.reply(replyToNode, replyContext, reply);
+ }
+
+ public interface CheckStatusReply extends Reply
+ {
+ boolean isOk();
+ }
+
+ public static class CheckStatusOk implements CheckStatusReply, Comparable<CheckStatusOk>
+ {
+ public final Status status;
+ public final Ballot promised;
+ public final Ballot accepted;
+ public final boolean isCoordinating;
+ public final boolean hasExecutedOnAllShards;
+
+ CheckStatusOk(Status status, Ballot promised, Ballot accepted, boolean isCoordinating, boolean hasExecutedOnAllShards)
+ {
+ this.status = status;
+ this.promised = promised;
+ this.accepted = accepted;
+ this.isCoordinating = isCoordinating;
+ this.hasExecutedOnAllShards = hasExecutedOnAllShards;
+ }
+
+ @Override
+ public boolean isOk()
+ {
+ return true;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "CheckStatusOk{" +
+ "status:" + status +
+ ", promised:" + promised +
+ ", accepted:" + accepted +
+ ", hasExecutedOnAllShards:" + hasExecutedOnAllShards +
+ ", isCoordinating:" + isCoordinating +
+ '}';
+ }
+
+ @Override
+ public int compareTo(CheckStatusOk that)
+ {
+ int c = this.promised.compareTo(that.promised);
+ if (c == 0) c = this.status.compareTo(that.status);
+ if (c == 0) c = this.accepted.compareTo(that.accepted);
+ if (c == 0 && this.hasExecutedOnAllShards != that.hasExecutedOnAllShards)
+ return this.hasExecutedOnAllShards ? 1 : -1;
+ return c;
+ }
+
+ public CheckStatusOk merge(CheckStatusOk that)
+ {
+ if (that.status.compareTo(this.status) > 0)
+ return that.merge(this);
+
+ // preferentially select the one that is coordinating, if any
+ CheckStatusOk prefer = this.isCoordinating ? this : that;
+ CheckStatusOk defer = prefer == this ? that : this;
+
+ // then select the max along each criteria, preferring the coordinator
+ CheckStatusOk maxStatus = prefer.status.compareTo(defer.status) >= 0 ? prefer : defer;
+ CheckStatusOk maxPromised = prefer.promised.compareTo(defer.promised) >= 0 ? prefer : defer;
+ CheckStatusOk maxAccepted = prefer.accepted.compareTo(defer.accepted) >= 0 ? prefer : defer;
+ CheckStatusOk maxHasExecuted = !defer.hasExecutedOnAllShards || prefer.hasExecutedOnAllShards ? prefer : defer;
+
+ // if the maximum (or preferred equal) is the same on all dimensions, return it
+ if (maxStatus == maxPromised && maxStatus == maxAccepted && maxStatus == maxHasExecuted)
+ return maxStatus;
+
+ // otherwise assemble the maximum of each, and propagate isCoordinating from the origin we selected the promise from
+
+ boolean isCoordinating = maxPromised == prefer ? prefer.isCoordinating : defer.isCoordinating;
+ return new CheckStatusOk(maxStatus.status, maxPromised.promised, maxAccepted.accepted,
+ isCoordinating, maxHasExecuted.hasExecutedOnAllShards);
+ }
+
+ @Override
+ public MessageType type()
+ {
+ return MessageType.CHECK_STATUS_RSP;
+ }
+ }
+
+ public static class CheckStatusOkFull extends CheckStatusOk
+ {
+ public final Key homeKey;
+ public final Timestamp executeAt;
+ public final Dependencies deps;
+ public final Writes writes;
+ public final Result result;
+
+ CheckStatusOkFull(Status status, Ballot promised, Ballot accepted, boolean isCoordinating, boolean hasExecutedOnAllShards,
+ Key homeKey, Timestamp executeAt, Dependencies deps, Writes writes, Result result)
+ {
+ super(status, promised, accepted, isCoordinating, hasExecutedOnAllShards);
+ this.homeKey = homeKey;
+ this.executeAt = executeAt;
+ this.deps = deps;
+ this.writes = writes;
+ this.result = result;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "CheckStatusOk{" +
+ "status:" + status +
+ ", promised:" + promised +
+ ", accepted:" + accepted +
+ ", executeAt:" + executeAt +
+ ", hasExecutedOnAllShards:" + hasExecutedOnAllShards +
+ ", isCoordinating:" + isCoordinating +
+ ", deps:" + deps +
+ ", writes:" + writes +
+ ", result:" + result +
+ '}';
+ }
+
+ /**
+ * This method assumes parameter is of the same type and has the same additional info.
+ * If parameters have different info, it is undefined which properties will be returned - no effort
+ * is made to merge different info.
+ *
+ * This method is NOT guaranteed to return CheckStatusOkFull unless the parameter is also CheckStatusOkFull.
+ * This method is NOT guaranteed to return either parameter: it may merge the two to represent the maximum
+ * combined info, (and in this case if the parameter were not CheckStatusOkFull, and were the higher status
+ * reply, the info would potentially be unsafe to act upon when given a higher status
+ * (e.g. Accepted executeAt is very different to Committed executeAt))
+ */
+ public CheckStatusOk merge(CheckStatusOk that)
+ {
+ CheckStatusOk max = super.merge(that);
+ if (this == max || that == max) return max;
+
+ CheckStatusOk maxSrc = this.status.compareTo(that.status) >= 0 ? this : that;
+ if (!(maxSrc instanceof CheckStatusOkFull))
+ return max;
+
+ CheckStatusOkFull src = (CheckStatusOkFull) maxSrc;
+ return new CheckStatusOkFull(max.status, max.promised, max.accepted, max.isCoordinating,
+ max.hasExecutedOnAllShards, src.homeKey, src.executeAt, src.deps, src.writes, src.result);
+ }
+ }
+
+ public static class CheckStatusNack implements CheckStatusReply
+ {
+ private static final CheckStatusNack instance = new CheckStatusNack();
+
+ private CheckStatusNack() { }
+
+ @Override
+ public MessageType type()
+ {
+ return MessageType.CHECK_STATUS_RSP;
+ }
+
+ static CheckStatusNack nack()
+ {
+ return instance;
+ }
+
+ @Override
+ public boolean isOk()
+ {
+ return false;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "CheckStatusNack";
+ }
+ }
+
+ @Override
+ public String toString()
+ {
+ return "CheckStatus{" +
+ "txnId:" + txnId +
+ '}';
+ }
+
+ @Override
+ public MessageType type()
+ {
+ return MessageType.CHECK_STATUS_REQ;
+ }
+}
diff --git a/accord-core/src/main/java/accord/messages/Commit.java b/accord-core/src/main/java/accord/messages/Commit.java
index 8bf512d..7fb83f3 100644
--- a/accord-core/src/main/java/accord/messages/Commit.java
+++ b/accord-core/src/main/java/accord/messages/Commit.java
@@ -1,5 +1,6 @@
package accord.messages;
+import accord.api.Key;
import accord.local.Node;
import accord.local.Node.Id;
import accord.topology.Topologies;
@@ -14,22 +15,21 @@ public class Commit extends ReadData
public final Dependencies deps;
public final boolean read;
- public Commit(Scope scope, TxnId txnId, Txn txn, Timestamp executeAt, Dependencies deps, boolean read)
+ public Commit(Id to, Topologies topologies, TxnId txnId, Txn txn, Key homeKey, Timestamp executeAt, Dependencies deps, boolean read)
{
- super(scope, txnId, txn, executeAt);
+ super(to, topologies, txnId, txn, homeKey, executeAt);
this.deps = deps;
this.read = read;
}
- public Commit(Id to, Topologies topologies, TxnId txnId, Txn txn, Timestamp executeAt, Dependencies deps, boolean read)
- {
- this(Scope.forTopologies(to, topologies, txn), txnId, txn, executeAt, deps, read);
- }
-
public void process(Node node, Id from, ReplyContext replyContext)
{
- node.forEachLocal(scope(), instance -> instance.command(txnId).commit(txn, deps, executeAt));
- if (read) super.process(node, from, replyContext);
+ Key progressKey = node.trySelectProgressKey(txnId, txn.keys, homeKey);
+ node.forEachLocal(scope(), txnId.epoch,
+ instance -> instance.command(txnId).commit(txn, homeKey, progressKey, executeAt, deps));
+
+ if (read)
+ super.process(node, from, replyContext);
}
@Override
diff --git a/accord-core/src/main/java/accord/messages/EpochRequest.java b/accord-core/src/main/java/accord/messages/EpochRequest.java
new file mode 100644
index 0000000..9e2855a
--- /dev/null
+++ b/accord-core/src/main/java/accord/messages/EpochRequest.java
@@ -0,0 +1,6 @@
+package accord.messages;
+
+public interface EpochRequest extends Request
+{
+ long waitForEpoch();
+}
diff --git a/accord-core/src/main/java/accord/messages/InformOfPersistence.java b/accord-core/src/main/java/accord/messages/InformOfPersistence.java
new file mode 100644
index 0000000..1ea3ffd
--- /dev/null
+++ b/accord-core/src/main/java/accord/messages/InformOfPersistence.java
@@ -0,0 +1,56 @@
+package accord.messages;
+
+import java.util.Set;
+
+import accord.api.Key;
+import accord.local.Node;
+import accord.local.Node.Id;
+import accord.txn.Timestamp;
+import accord.txn.TxnId;
+
+import static accord.messages.InformOfTxn.InformOfTxnNack.nack;
+import static accord.messages.InformOfTxn.InformOfTxnOk.ok;
+
+public class InformOfPersistence implements Request
+{
+ final TxnId txnId;
+ final Key homeKey;
+ final Timestamp executeAt;
+ final Set<Id> persistedOn;
+
+ public InformOfPersistence(TxnId txnId, Key homeKey, Timestamp executeAt, Set<Id> persistedOn)
+ {
+ this.txnId = txnId;
+ this.homeKey = homeKey;
+ this.executeAt = executeAt;
+ this.persistedOn = persistedOn;
+ }
+
+ public void process(Node node, Id replyToNode, ReplyContext replyContext)
+ {
+ Reply reply = node.ifLocal(homeKey, txnId, instance -> {
+ instance.command(txnId).setGloballyPersistent(homeKey, executeAt);
+ instance.progressLog().executedOnAllShards(txnId, persistedOn);
+ return ok();
+ });
+
+ if (reply == null)
+ reply = nack();
+
+ node.reply(replyToNode, replyContext, reply);
+ }
+
+ @Override
+ public String toString()
+ {
+ return "InformOfPersistence{" +
+ "txnId:" + txnId +
+ '}';
+ }
+
+ @Override
+ public MessageType type()
+ {
+ return MessageType.INFORM_PERSISTED_REQ;
+ }
+}
diff --git a/accord-core/src/main/java/accord/messages/InformOfTxn.java b/accord-core/src/main/java/accord/messages/InformOfTxn.java
new file mode 100644
index 0000000..3c7bbf9
--- /dev/null
+++ b/accord-core/src/main/java/accord/messages/InformOfTxn.java
@@ -0,0 +1,124 @@
+package accord.messages;
+
+import accord.api.Key;
+import accord.local.Node;
+import accord.local.Node.Id;
+import accord.txn.Txn;
+import accord.txn.TxnId;
+
+import static accord.messages.InformOfTxn.InformOfTxnNack.nack;
+import static accord.messages.InformOfTxn.InformOfTxnOk.ok;
+
+public class InformOfTxn implements EpochRequest
+{
+ final TxnId txnId;
+ final Key homeKey;
+ final Txn txn;
+
+ public InformOfTxn(TxnId txnId, Key homeKey, Txn txn)
+ {
+ this.txnId = txnId;
+ this.homeKey = homeKey;
+ this.txn = txn;
+ }
+
+ public void process(Node node, Id replyToNode, ReplyContext replyContext)
+ {
+ Key progressKey = node.selectProgressKey(txnId, txn.keys, homeKey);
+ Reply reply = node.ifLocal(homeKey, txnId, instance -> {
+ instance.command(txnId).preaccept(txn, homeKey, progressKey);
+ return ok();
+ });
+
+ if (reply == null)
+ reply = nack();
+
+ node.reply(replyToNode, replyContext, reply);
+ }
+
+ @Override
+ public String toString()
+ {
+ return "InformOfTxn{" +
+ "txnId:" + txnId +
+ ", txn:" + txn +
+ '}';
+ }
+
+ public interface InformOfTxnReply extends Reply
+ {
+ boolean isOk();
+ }
+
+ public static class InformOfTxnOk implements InformOfTxnReply
+ {
+ private static final InformOfTxnOk instance = new InformOfTxnOk();
+
+ @Override
+ public MessageType type()
+ {
+ return MessageType.INFORM_RSP;
+ }
+
+ static InformOfTxnReply ok()
+ {
+ return instance;
+ }
+
+ private InformOfTxnOk() { }
+
+ @Override
+ public boolean isOk()
+ {
+ return true;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "InformOfTxnOk";
+ }
+ }
+
+ public static class InformOfTxnNack implements InformOfTxnReply
+ {
+ private static final InformOfTxnNack instance = new InformOfTxnNack();
+
+ @Override
+ public MessageType type()
+ {
+ return MessageType.INFORM_RSP;
+ }
+
+ static InformOfTxnReply nack()
+ {
+ return instance;
+ }
+
+ private InformOfTxnNack() { }
+
+ @Override
+ public boolean isOk()
+ {
+ return false;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "InformOfTxnNack";
+ }
+ }
+
+ @Override
+ public MessageType type()
+ {
+ return MessageType.INFORM_REQ;
+ }
+
+ @Override
+ public long waitForEpoch()
+ {
+ return txnId.epoch;
+ }
+}
diff --git a/accord-core/src/main/java/accord/messages/MessageType.java b/accord-core/src/main/java/accord/messages/MessageType.java
index c60b35d..4a7c06c 100644
--- a/accord-core/src/main/java/accord/messages/MessageType.java
+++ b/accord-core/src/main/java/accord/messages/MessageType.java
@@ -11,10 +11,19 @@ public enum MessageType
ACCEPT_RSP,
COMMIT_REQ,
APPLY_REQ,
+ APPLY_RSP,
+ APPLY_AND_CHECK_REQ,
+ APPLY_AND_CHECK_RSP,
READ_REQ,
READ_RSP,
RECOVER_REQ,
RECOVER_RSP,
WAIT_ON_COMMIT_REQ,
- WAIT_ON_COMMIT_RSP
+ WAIT_ON_COMMIT_RSP,
+ INFORM_REQ,
+ INFORM_RSP,
+ INFORM_PERSISTED_REQ,
+ INFORM_PERSISTED_RSP,
+ CHECK_STATUS_REQ,
+ CHECK_STATUS_RSP
}
diff --git a/accord-core/src/main/java/accord/messages/PreAccept.java b/accord-core/src/main/java/accord/messages/PreAccept.java
index cee9cff..223dd5e 100644
--- a/accord-core/src/main/java/accord/messages/PreAccept.java
+++ b/accord-core/src/main/java/accord/messages/PreAccept.java
@@ -1,44 +1,54 @@
package accord.messages;
-import java.util.NavigableMap;
import java.util.Objects;
-import java.util.TreeMap;
+import java.util.stream.Stream;
+import com.google.common.annotations.VisibleForTesting;
+
+import accord.api.Key;
import accord.local.CommandStore;
+import accord.local.CommandsForKey;
import accord.local.Node;
import accord.local.Node.Id;
+import accord.messages.TxnRequest.WithUnsynced;
import accord.topology.Topologies;
+import accord.txn.Keys;
import accord.txn.Timestamp;
import accord.local.Command;
import accord.txn.Dependencies;
import accord.txn.Txn;
import accord.txn.TxnId;
-public class PreAccept extends TxnRequest
+public class PreAccept extends WithUnsynced
{
- public final TxnId txnId;
public final Txn txn;
+ public final long maxEpoch;
- public PreAccept(Scope scope, TxnId txnId, Txn txn)
+ public PreAccept(Id to, Topologies topologies, TxnId txnId, Txn txn, Key homeKey)
{
- super(scope);
- this.txnId = txnId;
+ super(to, topologies, txn.keys, txnId, homeKey);
this.txn = txn;
+ this.maxEpoch = topologies.currentEpoch();
}
- public PreAccept(Id to, Topologies topologies, TxnId txnId, Txn txn)
+ @VisibleForTesting
+ public PreAccept(Keys scope, long epoch, TxnId txnId, Txn txn, Key homeKey)
{
- this(Scope.forTopologies(to, topologies, txn), txnId, txn);
+ super(scope, epoch, txnId, homeKey);
+ this.txn = txn;
+ this.maxEpoch = epoch;
}
public void process(Node node, Id from, ReplyContext replyContext)
{
- node.reply(from, replyContext, node.mapReduceLocal(scope(), instance -> {
+ // TODO: verify we handle all of the scope() keys
+ Key progressKey = doNotComputeProgressKey ? null : node.trySelectProgressKey(waitForEpoch(), txn.keys, homeKey);
+ node.reply(from, replyContext, node.mapReduceLocal(scope(), minEpoch, maxEpoch, instance -> {
// note: this diverges from the paper, in that instead of waiting for JoinShard,
// we PreAccept to both old and new topologies and require quorums in both.
// This necessitates sending to ALL replicas of old topology, not only electorate (as fast path may be unreachable).
Command command = instance.command(txnId);
- if (!command.witness(txn))
+ if (!command.preaccept(txn, homeKey, progressKey))
return PreAcceptNack.INSTANCE;
return new PreAcceptOk(txnId, command.executeAt(), calculateDeps(instance, txnId, txn, txnId));
}, (r1, r2) -> {
@@ -108,9 +118,9 @@ public class PreAccept extends TxnRequest
public String toString()
{
return "PreAcceptOk{" +
- "txnId=" + txnId +
- ", witnessedAt=" + witnessedAt +
- ", deps=" + deps +
+ "txnId:" + txnId +
+ ", witnessedAt:" + witnessedAt +
+ ", deps:" + deps +
'}';
}
}
@@ -136,23 +146,40 @@ public class PreAccept extends TxnRequest
static Dependencies calculateDeps(CommandStore commandStore, TxnId txnId, Txn txn, Timestamp executeAt)
{
- NavigableMap<TxnId, Txn> deps = new TreeMap<>();
- txn.conflictsMayExecuteBefore(commandStore, executeAt).forEach(conflict -> {
+ Dependencies deps = new Dependencies();
+ conflictsMayExecuteBefore(commandStore, executeAt, txn.keys).forEach(conflict -> {
if (conflict.txnId().equals(txnId))
return;
if (txn.isWrite() || conflict.txn().isWrite())
- deps.put(conflict.txnId(), conflict.txn());
+ deps.add(conflict);
});
- return new Dependencies(deps);
+ return deps;
}
@Override
public String toString()
{
return "PreAccept{" +
- "txnId: " + txnId +
- ", txn: " + txn +
+ "txnId:" + txnId +
+ ", txn:" + txn +
+ ", homeKey:" + homeKey +
'}';
}
+
+ private static Stream<Command> conflictsMayExecuteBefore(CommandStore commandStore, Timestamp mayExecuteBefore, Keys keys)
+ {
+ return keys.stream().flatMap(key -> {
+ CommandsForKey forKey = commandStore.maybeCommandsForKey(key);
+ if (forKey == null)
+ return Stream.of();
+
+ return Stream.concat(
+ forKey.uncommitted.headMap(mayExecuteBefore, false).values().stream(),
+ // TODO: only return latest of Committed?
+ forKey.committedByExecuteAt.headMap(mayExecuteBefore, false).values().stream()
+ );
+ });
+ }
+
}
diff --git a/accord-core/src/main/java/accord/messages/ReadData.java b/accord-core/src/main/java/accord/messages/ReadData.java
index 6c08b97..bc32a77 100644
--- a/accord-core/src/main/java/accord/messages/ReadData.java
+++ b/accord-core/src/main/java/accord/messages/ReadData.java
@@ -1,21 +1,18 @@
package accord.messages;
-import java.util.HashSet;
-import java.util.Iterator;
import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
+import com.google.common.base.Preconditions;
+
+import accord.api.Key;
import accord.local.*;
import accord.local.Node.Id;
import accord.api.Data;
-import accord.topology.KeyRanges;
import accord.topology.Topologies;
import accord.txn.Keys;
+import accord.txn.Timestamp;
import accord.txn.Txn;
import accord.txn.TxnId;
-import accord.txn.Timestamp;
-import accord.api.Scheduler.Scheduled;
import accord.utils.DeterministicIdentitySet;
public class ReadData extends TxnRequest
@@ -25,45 +22,21 @@ public class ReadData extends TxnRequest
final TxnId txnId;
final Node node;
final Node.Id replyToNode;
- final Keys keyScope;
+ final Keys readKeys;
final ReplyContext replyContext;
Data data;
boolean isObsolete; // TODO: respond with the Executed result we have stored?
Set<CommandStore> waitingOn;
- Scheduled waitingOnReporter;
- LocalRead(TxnId txnId, Node node, Id replyToNode, Keys keyScope, ReplyContext replyContext)
+ LocalRead(TxnId txnId, Node node, Id replyToNode, Keys readKeys, ReplyContext replyContext)
{
+ Preconditions.checkArgument(!readKeys.isEmpty());
this.txnId = txnId;
this.node = node;
this.replyToNode = replyToNode;
- this.keyScope = keyScope;
+ this.readKeys = readKeys;
this.replyContext = replyContext;
- // TODO: this is messy, we want a complete separate liveness mechanism that ensures progress for all transactions
- this.waitingOnReporter = node.scheduler().once(new ReportWaiting(), 1L, TimeUnit.SECONDS);
- }
-
- class ReportWaiting implements Listener, Runnable
- {
- @Override
- public void onChange(Command command)
- {
- command.removeListener(this);
- run();
- }
-
- @Override
- public void run()
- {
- Iterator<CommandStore> i = waitingOn.iterator();
- Command blockedBy = null;
- while (i.hasNext() && null == (blockedBy = i.next().command(txnId).blockedBy()));
- if (blockedBy == null) return;
- blockedBy.addListener(this);
- assert blockedBy.status().compareTo(Status.NotWitnessed) > 0;
- node.reply(replyToNode, replyContext, new ReadWaiting(txnId, blockedBy.txnId(), blockedBy.txn(), blockedBy.executeAt(), blockedBy.status()));
- }
}
@Override
@@ -79,7 +52,7 @@ public class ReadData extends TxnRequest
case Executed:
case Applied:
- obsolete(command);
+ obsolete();
case ReadyToExecute:
}
@@ -91,44 +64,31 @@ public class ReadData extends TxnRequest
private void read(Command command)
{
// TODO: threading/futures (don't want to perform expensive reads within this mutually exclusive context)
- Data next = command.txn().read(command, keyScope);
+ Data next = command.txn().read(command, readKeys);
data = data == null ? next : data.merge(next);
waitingOn.remove(command.commandStore);
if (waitingOn.isEmpty())
- {
- waitingOnReporter.cancel();
node.reply(replyToNode, replyContext, new ReadOk(data));
- }
}
- void obsolete(Command command)
+ void obsolete()
{
if (!isObsolete)
{
isObsolete = true;
- waitingOnReporter.cancel();
- Set<Node.Id> nodes = new HashSet<>();
- Topologies topologies = node.topology().forKeys(command.txn().keys());
- KeyRanges ranges = command.commandStore.ranges();
- topologies.forEachShard(shard -> {
- if (ranges.intersects(shard.range))
- nodes.addAll(shard.nodes);
- });
- // FIXME: this may result in redundant messages being sent when a shard is split across several command shards
- node.send(nodes, to -> new Apply(to, topologies, command.txnId(), command.txn(), command.executeAt(), command.savedDeps(), command.writes(), command.result()));
node.reply(replyToNode, replyContext, new ReadNack());
}
}
- synchronized void setup(TxnId txnId, Txn txn, Scope scope)
+ synchronized void setup(TxnId txnId, Txn txn, Key homeKey, Keys keys, Timestamp executeAt)
{
- // TODO: simple hash set supporting concurrent modification, or else avoid concurrent modification
- waitingOn = node.collectLocal(scope, DeterministicIdentitySet::new);
+ Key progressKey = node.trySelectProgressKey(txnId, txn.keys, homeKey);
+ waitingOn = node.collectLocal(keys, executeAt, DeterministicIdentitySet::new);
// FIXME: fix/check thread safety
CommandStore.onEach(waitingOn, instance -> {
Command command = instance.command(txnId);
- command.witness(txn);
+ command.preaccept(txn, homeKey, progressKey); // ensure pre-accepted
switch (command.status())
{
case NotWitnessed:
@@ -141,7 +101,7 @@ public class ReadData extends TxnRequest
case Executed:
case Applied:
- obsolete(command);
+ obsolete();
break;
case ReadyToExecute:
@@ -154,24 +114,22 @@ public class ReadData extends TxnRequest
public final TxnId txnId;
public final Txn txn;
+ final Key homeKey;
public final Timestamp executeAt;
- public ReadData(Scope scope, TxnId txnId, Txn txn, Timestamp executeAt)
+ public ReadData(Node.Id to, Topologies topologies, TxnId txnId, Txn txn, Key homeKey, Timestamp executeAt)
{
- super(scope);
+ super(to, topologies, txn.keys);
this.txnId = txnId;
this.txn = txn;
+ this.homeKey = homeKey;
this.executeAt = executeAt;
}
- public ReadData(Node.Id to, Topologies topologies, TxnId txnId, Txn txn, Timestamp executeAt)
- {
- this(Scope.forTopologies(to, topologies, txn), txnId, txn, executeAt);
- }
-
public void process(Node node, Node.Id from, ReplyContext replyContext)
{
- new LocalRead(txnId, node, from, scope().keys(), replyContext).setup(txnId, txn, scope());
+ new LocalRead(txnId, node, from, txn.read.keys().intersect(scope()), replyContext)
+ .setup(txnId, txn, homeKey, scope(), executeAt);
}
@Override
@@ -218,42 +176,6 @@ public class ReadData extends TxnRequest
}
}
- public static class ReadWaiting extends ReadReply
- {
- public final TxnId forTxn;
- public final TxnId txnId;
- public final Txn txn;
- public final Timestamp executeAt;
- public final Status status;
-
- public ReadWaiting(TxnId forTxn, TxnId txnId, Txn txn, Timestamp executeAt, Status status)
- {
- this.forTxn = forTxn;
- this.txnId = txnId;
- this.txn = txn;
- this.executeAt = executeAt;
- this.status = status;
- }
-
- @Override
- public boolean isFinal()
- {
- return false;
- }
-
- @Override
- public String toString()
- {
- return "ReadWaiting{" +
- "forTxn:" + forTxn +
- ", txnId:" + txnId +
- ", txn:" + txn +
- ", executeAt:" + executeAt +
- ", status:" + status +
- '}';
- }
- }
-
@Override
public String toString()
{
diff --git a/accord-core/src/main/java/accord/messages/TxnRequest.java b/accord-core/src/main/java/accord/messages/TxnRequest.java
index a41b56e..7af2aac 100644
--- a/accord-core/src/main/java/accord/messages/TxnRequest.java
+++ b/accord-core/src/main/java/accord/messages/TxnRequest.java
@@ -1,104 +1,197 @@
package accord.messages;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+
+import accord.api.Key;
import accord.local.Node;
+import accord.local.Node.Id;
import accord.topology.KeyRanges;
import accord.topology.Topologies;
import accord.topology.Topology;
import accord.txn.Keys;
-import accord.txn.Txn;
+import accord.txn.TxnId;
-import java.util.Objects;
+import static java.lang.Long.min;
-public abstract class TxnRequest implements Request
+public abstract class TxnRequest implements EpochRequest
{
- private final Scope scope;
+ public static abstract class WithUnsynced extends TxnRequest
+ {
+ public final TxnId txnId;
+ public final Key homeKey;
+ public final long minEpoch;
+ protected final boolean doNotComputeProgressKey;
+
+ public WithUnsynced(Id to, Topologies topologies, Keys keys, TxnId txnId, Key homeKey)
+ {
+ this(to, topologies, keys, txnId, homeKey, latestRelevantEpochIndex(to, topologies, keys));
+ }
+
+ private WithUnsynced(Id to, Topologies topologies, Keys keys, TxnId txnId, Key homeKey, int startIndex)
+ {
+ super(to, topologies, keys, startIndex);
+ this.txnId = txnId;
+ this.homeKey = homeKey;
+ this.minEpoch = topologies.oldestEpoch();
+ // to understand this calculation we must bear in mind the following:
+ // - startIndex is the "latest relevant" which means we skip over recent epochs where we are not owners at all,
+ // i.e. if this node does not participate in the most recent epoch, startIndex > 0
+ // - waitForEpoch gives us the most recent epoch with differing ownership information, starting from startIndex
+ // So, we can have some surprising situations arise where a *prior* owner must be contacted for its vote,
+ // and does not need to wait for the latest ring information because from the point of view of its contribution
+ // the stale ring information is sufficient, however we do not want it to compute a progress key with this stale
+ // ring information and mistakenly believe that it is a home shard for the transaction, as it will not receive
+ // updates for the transaction going forward.
+ // So in these cases we send a special flag indicating that the progress key should not be computed
+ // (as it might be done so with stale ring information)
+ this.doNotComputeProgressKey = waitForEpoch() < txnId.epoch && startIndex > 0
+ && topologies.get(startIndex).epoch() < txnId.epoch;
+
+ KeyRanges ranges = topologies.forEpoch(txnId.epoch).rangesForNode(to);
+ if (doNotComputeProgressKey)
+ {
+ Preconditions.checkState(ranges == null || !ranges.intersects(keys)); // confirm dest is not a replica on txnId.epoch
+ }
+ else
+ {
+ boolean intersects = ranges != null && ranges.intersects(keys);
+ long progressEpoch = Math.min(waitForEpoch(), txnId.epoch);
+ KeyRanges computesRangesOn = topologies.forEpoch(progressEpoch).rangesForNode(to);
+ boolean check = computesRangesOn != null && computesRangesOn.intersects(keys);
+ if (check != intersects)
+ throw new IllegalStateException();
+ }
+ }
- public TxnRequest(Scope scope)
+ Key progressKey(Node node)
+ {
+ // if waitForEpoch < txnId.epoch, then this replica's ownership is unchanged
+ long progressEpoch = min(waitForEpoch(), txnId.epoch);
+ return doNotComputeProgressKey ? null : node.trySelectProgressKey(progressEpoch, scope(), homeKey);
+ }
+
+ @VisibleForTesting
+ public WithUnsynced(Keys scope, long epoch, TxnId txnId, Key homeKey)
+ {
+ super(scope, epoch);
+ this.txnId = txnId;
+ this.homeKey = homeKey;
+ this.minEpoch = epoch;
+ this.doNotComputeProgressKey = false;
+ }
+ }
+
+ private final Keys scope;
+ private final long waitForEpoch;
+
+ public TxnRequest(Node.Id to, Topologies topologies, Keys keys)
+ {
+ this(to, topologies, keys, 0);
+ }
+
+ public TxnRequest(Node.Id to, Topologies topologies, Keys keys, int startIndex)
+ {
+ this(computeScope(to, topologies, keys, startIndex),
+ computeWaitForEpoch(to, topologies, startIndex));
+ }
+
+ public TxnRequest(Keys scope, long waitForEpoch)
{
this.scope = scope;
+ this.waitForEpoch = waitForEpoch;
}
- public Scope scope()
+ public Keys scope()
{
return scope;
}
- /**
- * Indicates the keys the coordinator expects the recipient to service for a request, and
- * the minimum epochs the recipient will need to be aware of for each set of keys
- */
- public static class Scope
+ public long waitForEpoch()
+ {
+ return waitForEpoch;
+ }
+
+ protected static int latestRelevantEpochIndex(Node.Id node, Topologies topologies, Keys keys)
{
- private final long minRequiredEpoch;
- private final Keys keys;
+ KeyRanges latest = topologies.get(0).rangesForNode(node);
- public Scope(long minRequiredEpoch, Keys keys)
+ if (latest != null && latest.intersects(keys))
+ return 0;
+
+ int i = 0;
+ int mi = topologies.size();
+
+ // find first non-null for node
+ while (latest == null)
{
- this.minRequiredEpoch = minRequiredEpoch;
- this.keys = keys;
+ if (++i == mi)
+ return mi;
+
+ latest = topologies.get(i).rangesForNode(node);
}
- public static Scope forTopologies(Node.Id node, Topologies topologies, Keys txnKeys)
+ if (latest.intersects(keys))
+ return i;
+
+ // find first non-empty intersection for node
+ while (++i < mi)
{
- long minEpoch = 0;
- Keys scopeKeys = Keys.EMPTY;
- Keys lastKeys = null;
- for (int i=topologies.size() - 1; i>=0; i--)
+ KeyRanges next = topologies.get(i).rangesForNode(node);
+ if (!next.equals(latest))
{
- Topology topology = topologies.get(i);
- KeyRanges topologyRanges = topology.rangesForNode(node);
- if (topologyRanges == null)
- continue;
- topologyRanges = topologyRanges.intersection(txnKeys);
- Keys epochKeys = txnKeys.intersection(topologyRanges);
- if (lastKeys == null || !lastKeys.containsAll(epochKeys))
- {
- minEpoch = topology.epoch();
- scopeKeys = scopeKeys.merge(epochKeys);
- }
- lastKeys = epochKeys;
+ if (next.intersects(keys))
+ return i;
+ latest = next;
}
-
- return new Scope(minEpoch, scopeKeys);
}
+ return mi;
+ }
- public static Scope forTopologies(Node.Id node, Topologies topologies, Txn txn)
- {
- return forTopologies(node, topologies, txn.keys());
- }
+ // for now use a simple heuristic of whether the node's ownership ranges have changed,
+ // on the assumption that this might also mean some local shard rearrangement
+ // except if the latest epoch is empty for the keys
+ public static long computeWaitForEpoch(Node.Id node, Topologies topologies, Keys keys)
+ {
+ return computeWaitForEpoch(node, topologies, latestRelevantEpochIndex(node, topologies, keys));
+ }
- public long minRequiredEpoch()
- {
- return minRequiredEpoch;
- }
+ public static long computeWaitForEpoch(Node.Id node, Topologies topologies, int startIndex)
+ {
+ int i = startIndex;
+ int mi = topologies.size();
+ if (i == mi)
+ return topologies.oldestEpoch();
- public Keys keys()
+ KeyRanges latest = topologies.get(i).rangesForNode(node);
+ while (++i < mi)
{
- return keys;
+ Topology topology = topologies.get(i);
+ KeyRanges ranges = topology.rangesForNode(node);
+ if (ranges == null || !ranges.equals(latest))
+ break;
}
+ return topologies.get(i - 1).epoch();
+ }
- @Override
- public boolean equals(Object o)
- {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
- Scope scope = (Scope) o;
- return minRequiredEpoch == scope.minRequiredEpoch && keys.equals(scope.keys);
- }
+ public static Keys computeScope(Node.Id node, Topologies topologies, Keys keys)
+ {
+ return computeScope(node, topologies, keys, latestRelevantEpochIndex(node, topologies, keys));
+ }
- @Override
- public int hashCode()
+ public static Keys computeScope(Node.Id node, Topologies topologies, Keys keys, int startIndex)
+ {
+ KeyRanges last = null;
+ Keys scopeKeys = Keys.EMPTY;
+ for (int i = startIndex, mi = topologies.size() ; i < mi ; ++i)
{
- return Objects.hash(minRequiredEpoch, keys);
- }
+ Topology topology = topologies.get(i);
+ KeyRanges ranges = topology.rangesForNode(node);
+ if (ranges != last && ranges != null && !ranges.equals(last))
+ scopeKeys = scopeKeys.union(keys.intersect(ranges));
- @Override
- public String toString()
- {
- return "Scope{" +
- "maxEpoch=" + minRequiredEpoch +
- ", keys=" + keys +
- '}';
+ last = ranges;
}
+ return scopeKeys;
}
}
diff --git a/accord-core/src/main/java/accord/messages/WaitOnCommit.java b/accord-core/src/main/java/accord/messages/WaitOnCommit.java
index 63aafb9..d3e54c8 100644
--- a/accord-core/src/main/java/accord/messages/WaitOnCommit.java
+++ b/accord-core/src/main/java/accord/messages/WaitOnCommit.java
@@ -78,30 +78,23 @@ public class WaitOnCommit extends TxnRequest
synchronized void setup(Keys keys)
{
- List<CommandStore> instances = node.collectLocal(keys, ArrayList::new);
+ List<CommandStore> instances = node.collectLocal(keys, txnId, ArrayList::new);
waitingOn.set(instances.size());
instances.forEach(instance -> instance.processBlocking(this::setup));
}
}
public final TxnId txnId;
- public final Keys keys;
-
- public WaitOnCommit(Scope scope, TxnId txnId, Keys keys)
- {
- super(scope);
- this.txnId = txnId;
- this.keys = keys;
- }
public WaitOnCommit(Id to, Topologies topologies, TxnId txnId, Keys keys)
{
- this(Scope.forTopologies(to, topologies, keys), txnId, keys);
+ super(to, topologies, keys);
+ this.txnId = txnId;
}
public void process(Node node, Id replyToNode, ReplyContext replyContext)
{
- new LocalWait(node, replyToNode, txnId, replyContext).setup(keys);
+ new LocalWait(node, replyToNode, txnId, replyContext).setup(scope());
}
@Override
diff --git a/accord-core/src/main/java/accord/api/KeyRange.java b/accord-core/src/main/java/accord/topology/KeyRange.java
similarity index 99%
rename from accord-core/src/main/java/accord/api/KeyRange.java
rename to accord-core/src/main/java/accord/topology/KeyRange.java
index b045a0c..a3aabef 100644
--- a/accord-core/src/main/java/accord/api/KeyRange.java
+++ b/accord-core/src/main/java/accord/topology/KeyRange.java
@@ -1,6 +1,6 @@
-package accord.api;
+package accord.topology;
-import accord.topology.KeyRanges;
+import accord.api.Key;
import accord.txn.Keys;
import com.google.common.base.Preconditions;
diff --git a/accord-core/src/main/java/accord/topology/KeyRanges.java b/accord-core/src/main/java/accord/topology/KeyRanges.java
index de2ce60..3f87f88 100644
--- a/accord-core/src/main/java/accord/topology/KeyRanges.java
+++ b/accord-core/src/main/java/accord/topology/KeyRanges.java
@@ -1,7 +1,6 @@
package accord.topology;
import accord.api.Key;
-import accord.api.KeyRange;
import accord.txn.Keys;
import com.google.common.base.Preconditions;
import com.google.common.collect.Iterators;
@@ -161,6 +160,17 @@ public class KeyRanges implements Iterable<KeyRange>
return false;
}
+ public int findFirstIntersecting(Keys keys)
+ {
+ for (int i=0; i<ranges.length; i++)
+ {
+ int lowKeyIndex = ranges[i].lowKeyIndex(keys);
+ if (lowKeyIndex >= 0)
+ return lowKeyIndex;
+ }
+ return -1;
+ }
+
/**
* Subtracts the given set of key ranges from this
* @param that
@@ -212,7 +222,7 @@ public class KeyRanges implements Iterable<KeyRange>
/**
* Adds a set of non-overlapping ranges
*/
- public KeyRanges union(KeyRanges that)
+ public KeyRanges combine(KeyRanges that)
{
KeyRange[] combined = new KeyRange[this.ranges.length + that.ranges.length];
System.arraycopy(this.ranges, 0, combined, 0, this.ranges.length);
@@ -225,9 +235,9 @@ public class KeyRanges implements Iterable<KeyRange>
return new KeyRanges(combined);
}
- public KeyRanges union(KeyRange range)
+ public KeyRanges combine(KeyRange range)
{
- return union(new KeyRanges(new KeyRange[]{range}));
+ return combine(new KeyRanges(new KeyRange[]{ range}));
}
private static KeyRange tryMerge(KeyRange range1, KeyRange range2)
@@ -237,71 +247,84 @@ public class KeyRanges implements Iterable<KeyRange>
return range1.tryMerge(range2);
}
- public KeyRanges merge(KeyRanges that)
+ // optimised for case where one contains the other
+ public KeyRanges union(KeyRanges that)
{
- List<KeyRange> result = new ArrayList<>(this.size() + that.size());
- int thisIdx = 0, thisSize = this.size();
- int thatIdx = 0, thatSize = that.size();
+ int ai = 0, bi = 0;
+ KeyRange[] as = this.ranges, bs = that.ranges;
+ if (as.length < bs.length)
+ {
+ KeyRange[] tmp = as;
+ as = bs;
+ bs = tmp;
+ }
- KeyRange merging = null;
- while (thisIdx < thisSize || thatIdx < thatSize)
+ // TODO: this doesn't correctly handle as.length == bs.length if bs > as
+ while (ai < as.length && bi < bs.length)
{
- KeyRange merged;
- KeyRange thisRange = thisIdx < thisSize ? this.get(thisIdx) : null;
- KeyRange thatRange = thatIdx < thatSize ? that.get(thatIdx) : null;
- if ((merged = tryMerge(merging, thisRange)) != null)
- {
- merging = merged;
- thisIdx++;
- }
- else if ((merged = tryMerge(merging, thatRange)) != null)
- {
- merging = merged;
- thatIdx++;
- }
- else if (merging != null)
- {
- result.add(merging);
- merging = null;
- }
- else if (thisRange == null)
+ KeyRange a = as[ai];
+ KeyRange b = bs[bi];
+ int c = a.compareIntersecting(b);
+ if (c < 0) ai++;
+ else if (c > 0 || !a.fullyContains(b)) break;
+ else bi++;
+ }
+
+ if (bi == bs.length)
+ return as == this.ranges ? this : that;
+
+ KeyRange[] result = new KeyRange[as.length + (bs.length - bi)];
+ System.arraycopy(as, 0, result, 0, ai);
+ int count = ai;
+
+ while (ai < as.length && bi < bs.length)
+ {
+ KeyRange a = as[ai];
+ KeyRange b = bs[bi];
+
+ int c = a.compareIntersecting(b);
+ if (c < 0)
{
- result.add(thatRange);
- thatIdx++;
+ result[count++] = a;
+ ai++;
}
- else if (thatRange == null)
+ else if (c > 0)
{
- result.add(thisRange);
- thisIdx++;
+ result[count++] = b;
+ bi++;
}
else
{
- int cmp = thisRange.compareIntersecting(thatRange);
- if (cmp > 0)
+ KeyRange merged = a.subRange(c < 0 ? a.start() : b.start(), a.end().compareTo(b.end()) > 0 ? a.end() : b.end());
+ ai++;
+ bi++;
+ while (ai < as.length || bi < bs.length)
{
- result.add(thatRange);
- thatIdx++;
- }
- else if (cmp < 0)
- {
- result.add(thisRange);
- thisIdx++;
- }
- else
- {
- merging = thisRange.tryMerge(thatRange);
- thisIdx++;
- thatIdx++;
- Preconditions.checkState(merging != null);
+ KeyRange min;
+ if (ai == as.length) min = bs[bi];
+ else if (bi == bs.length) min = a = as[ai];
+ else min = as[ai].start().compareTo(bs[bi].start()) < 0 ? a = as[ai] : bs[bi];
+ if (min.start().compareTo(merged.end()) > 0)
+ break;
+ if (min.end().compareTo(merged.end()) > 0)
+ merged = merged.subRange(merged.start(), min.end());
+ if (a == min) ai++;
+ else bi++;
}
+ result[count++] = merged;
}
}
- if (merging != null)
- result.add(merging);
+ while (ai < as.length)
+ result[count++] = as[ai++];
+ while (bi < bs.length)
+ result[count++] = bs[bi++];
- return new KeyRanges(result.toArray(KeyRange[]::new));
+ if (count < result.length)
+ result = Arrays.copyOf(result, count);
+
+ return new KeyRanges(result);
}
public KeyRanges mergeTouching()
diff --git a/accord-core/src/main/java/accord/topology/Shard.java b/accord-core/src/main/java/accord/topology/Shard.java
index 4418807..4814e73 100644
--- a/accord-core/src/main/java/accord/topology/Shard.java
+++ b/accord-core/src/main/java/accord/topology/Shard.java
@@ -5,7 +5,6 @@ import java.util.List;
import java.util.Objects;
import java.util.Set;
-import accord.api.KeyRange;
import accord.local.Node.Id;
import accord.api.Key;
import com.google.common.annotations.VisibleForTesting;
@@ -14,6 +13,7 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
+// TODO: concept of region/locality
public class Shard
{
public final KeyRange range;
diff --git a/accord-core/src/main/java/accord/topology/Topologies.java b/accord-core/src/main/java/accord/topology/Topologies.java
index d9a5631..78c716a 100644
--- a/accord-core/src/main/java/accord/topology/Topologies.java
+++ b/accord-core/src/main/java/accord/topology/Topologies.java
@@ -11,6 +11,10 @@ public interface Topologies
{
Topology current();
+ Topology forEpoch(long epoch);
+
+ long oldestEpoch();
+
default long currentEpoch()
{
return current().epoch;
@@ -27,8 +31,6 @@ public interface Topologies
Set<Node.Id> nodes();
- Topologies removeEpochsBefore(long epoch);
-
default void forEach(IndexedConsumer<Topology> consumer)
{
for (int i=0, mi=size(); i<mi; i++)
@@ -81,7 +83,7 @@ public interface Topologies
StringBuilder sb = new StringBuilder("[");
for (int i=0, mi=t.size(); i<mi; i++)
{
- if (i < 0)
+ if (i > 0)
sb.append(", ");
sb.append(t.get(i).toString());
@@ -90,12 +92,12 @@ public interface Topologies
return sb.toString();
}
- class Singleton implements Topologies
+ class Single implements Topologies
{
private final Topology topology;
private final boolean fastPathPermitted;
- public Singleton(Topology topology, boolean fastPathPermitted)
+ public Single(Topology topology, boolean fastPathPermitted)
{
this.topology = topology;
this.fastPathPermitted = fastPathPermitted;
@@ -107,6 +109,20 @@ public interface Topologies
return topology;
}
+ @Override
+ public Topology forEpoch(long epoch)
+ {
+ if (topology.epoch != epoch)
+ throw new IndexOutOfBoundsException();
+ return topology;
+ }
+
+ @Override
+ public long oldestEpoch()
+ {
+ return currentEpoch();
+ }
+
@Override
public boolean fastPathPermitted()
{
@@ -139,14 +155,6 @@ public interface Topologies
return topology.nodes();
}
- @Override
- public Topologies removeEpochsBefore(long epoch)
- {
- if (epoch > topology.epoch())
- throw new IndexOutOfBoundsException(epoch + " is greater than current epoch " + topology.epoch());
- return this;
- }
-
@Override
public boolean equals(Object obj)
{
@@ -188,6 +196,21 @@ public interface Topologies
return get(0);
}
+ @Override
+ public Topology forEpoch(long epoch)
+ {
+ long index = get(0).epoch - epoch;
+ if (index < 0 || index > size())
+ throw new IndexOutOfBoundsException();
+ return get((int)index);
+ }
+
+ @Override
+ public long oldestEpoch()
+ {
+ return get(size() - 1).epoch;
+ }
+
@Override
public boolean fastPathPermitted()
{
@@ -224,27 +247,6 @@ public interface Topologies
return result;
}
- @Override
- public Topologies removeEpochsBefore(long epoch)
- {
- long current = currentEpoch();
- if (epoch > current)
- throw new IndexOutOfBoundsException(epoch + " is greater than current epoch " + current);
- if (epoch <= topologies.get(0).epoch())
- return this;
- if (epoch == current)
- return new Singleton(current(), fastPathPermitted());
-
- int numEpochs = (int) (current - epoch + 1);
- Topology[] result = new Topology[numEpochs];
- int startIdx = topologies.size() - numEpochs;
- for (int i=0; i<result.length; i++)
- result[i] = topologies.get(startIdx + 1);
- Preconditions.checkState(result[0].epoch() >= epoch);
- Preconditions.checkState(current == result[result.length - 1].epoch());
- return new Multi(result);
- }
-
public void add(Topology topology)
{
Preconditions.checkArgument(topologies.isEmpty() || topology.epoch == topologies.get(topologies.size() - 1).epoch - 1);
diff --git a/accord-core/src/main/java/accord/topology/Topology.java b/accord-core/src/main/java/accord/topology/Topology.java
index dd25071..5bf2ef7 100644
--- a/accord-core/src/main/java/accord/topology/Topology.java
+++ b/accord-core/src/main/java/accord/topology/Topology.java
@@ -4,12 +4,13 @@ import java.util.*;
import java.util.function.IntFunction;
import java.util.stream.IntStream;
-import accord.api.KeyRange;
+import accord.local.CommandStores;
import accord.local.Node.Id;
import accord.api.Key;
import accord.txn.Keys;
import accord.utils.IndexedConsumer;
import accord.utils.IndexedBiFunction;
+import accord.utils.IndexedIntFunction;
import accord.utils.IndexedPredicate;
public class Topology extends AbstractCollection<Shard>
@@ -142,6 +143,7 @@ public class Topology extends AbstractCollection<Shard>
return info != null ? info.ranges : null;
}
+ // TODO: optimised HomeKey concept containing the Key, Shard and Topology to avoid lookups when topology hasn't changed
public Shard forKey(Key key)
{
int i = ranges.rangeIndexForKey(key);
@@ -288,6 +290,37 @@ public class Topology extends AbstractCollection<Shard>
return count;
}
+ public int foldlIntOn(Id on, IndexedIntFunction<Shard> consumer, int offset, int initialValue, int terminalValue)
+ {
+ // TODO: this can be done by divide-and-conquer splitting of the lists and recursion, which should be more efficient
+ NodeInfo info = nodeLookup.get(on);
+ if (info == null)
+ return initialValue;
+ int[] a = supersetRangeIndexes, b = info.supersetIndexes;
+ int ai = 0, bi = 0;
+ while (ai < a.length && bi < b.length)
+ {
+ if (a[ai] == b[bi])
+ {
+ initialValue = consumer.apply(offset + ai, shards[a[ai]], initialValue);
+ if (terminalValue == initialValue)
+ return terminalValue;
+ ++ai; ++bi;
+ }
+ else if (a[ai] < b[bi])
+ {
+ ai = Arrays.binarySearch(a, ai + 1, a.length, b[bi]);
+ if (ai < 0) ai = -1 -ai;
+ }
+ else
+ {
+ bi = Arrays.binarySearch(b, bi + 1, b.length, a[ai]);
+ if (bi < 0) bi = -1 -bi;
+ }
+ }
+ return initialValue;
+ }
+
public void forEach(IndexedConsumer<Shard> consumer)
{
for (int i = 0; i < supersetRangeIndexes.length ; ++i)
diff --git a/accord-core/src/main/java/accord/topology/TopologyManager.java b/accord-core/src/main/java/accord/topology/TopologyManager.java
index d3e9a92..60691cf 100644
--- a/accord-core/src/main/java/accord/topology/TopologyManager.java
+++ b/accord-core/src/main/java/accord/topology/TopologyManager.java
@@ -1,11 +1,14 @@
package accord.topology;
import accord.api.ConfigurationService;
+import accord.api.Key;
import accord.coordinate.tracking.QuorumTracker;
import accord.local.Node;
+import accord.messages.EpochRequest;
import accord.messages.Request;
-import accord.messages.TxnRequest;
+import accord.topology.Topologies.Single;
import accord.txn.Keys;
+import accord.txn.Timestamp;
import accord.txn.Txn;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
@@ -25,11 +28,14 @@ import java.util.function.LongConsumer;
* * clean up obsolete data
*
* Assumes a topology service that won't report epoch n without having n-1 etc also available
+ *
+ * TODO: make TopologyManager a Topologies and copy-on-write update to it, so we can always just take a reference for
+ * transactions instead of copying every time (and index into it by the txnId.epoch)
*/
public class TopologyManager implements ConfigurationService.Listener
{
private static final Future<Void> SUCCESS = ImmediateFuture.success(null);
- class EpochState
+ static class EpochState
{
private final Topology global;
private final Topology local;
@@ -37,12 +43,12 @@ public class TopologyManager implements ConfigurationService.Listener
private boolean syncComplete = false;
private boolean prevSynced;
- EpochState(Topology global, boolean prevSynced)
+ EpochState(Node.Id node, Topology global, boolean prevSynced)
{
- Preconditions.checkArgument(!global.isSubset());
this.global = global;
this.local = global.forNode(node);
- this.syncTracker = new QuorumTracker(new Topologies.Singleton(global, false));
+ Preconditions.checkArgument(!global().isSubset());
+ this.syncTracker = new QuorumTracker(new Single(global(), false));
this.prevSynced = prevSynced;
}
@@ -53,13 +59,22 @@ public class TopologyManager implements ConfigurationService.Listener
public void recordSyncComplete(Node.Id node)
{
- syncTracker.recordSuccess(node);
- syncComplete = syncTracker.hasReachedQuorum();
+ syncComplete = syncTracker.success(node);
+ }
+
+ Topology global()
+ {
+ return global;
+ }
+
+ Topology local()
+ {
+ return local;
}
long epoch()
{
- return global.epoch;
+ return global().epoch;
}
boolean syncComplete()
@@ -76,7 +91,7 @@ public class TopologyManager implements ConfigurationService.Listener
return false;
if (syncComplete)
return true;
- Boolean result = global.foldl(keys, (i, shard, acc) -> {
+ Boolean result = global().foldl(keys, (i, shard, acc) -> {
if (acc == Boolean.FALSE)
return acc;
return Boolean.valueOf(syncTracker.unsafeGet(i).hasReachedQuorum());
@@ -139,7 +154,7 @@ public class TopologyManager implements ConfigurationService.Listener
public Topology current()
{
- return epochs.length > 0 ? epochs[0].global : Topology.EMPTY;
+ return epochs.length > 0 ? epochs[0].global() : Topology.EMPTY;
}
/**
@@ -171,26 +186,22 @@ public class TopologyManager implements ConfigurationService.Listener
private EpochState get(long epoch)
{
- if (epoch > currentEpoch || epoch < currentEpoch - epochs.length)
+ if (epoch > currentEpoch || epoch <= currentEpoch - epochs.length)
return null;
return epochs[(int) (currentEpoch - epoch)];
}
- long maxUnknownEpoch(TxnRequest.Scope scope)
- {
- if (currentEpoch < scope.minRequiredEpoch())
- return scope.minRequiredEpoch();
- return 0;
- }
-
- boolean requiresHistoricalTopologiesFor(Keys keys)
+ boolean requiresHistoricalTopologiesFor(Keys keys, long epoch)
{
- return epochs.length > 1 && !epochs[1].syncCompleteFor(keys);
+ Preconditions.checkState(epoch <= currentEpoch);
+ EpochState state = get(epoch - 1);
+ return state != null && !state.syncCompleteFor(keys);
}
}
private final Node.Id node;
+ // TODO: this is unused!?
private final LongConsumer epochReporter;
private volatile Epochs epochs;
@@ -219,7 +230,7 @@ public class TopologyManager implements ConfigurationService.Listener
System.arraycopy(current.epochs, 0, nextEpochs, 1, current.epochs.length);
boolean prevSynced = current.epochs.length == 0 || current.epochs[0].syncComplete();
- nextEpochs[0] = new EpochState(topology, prevSynced);
+ nextEpochs[0] = new EpochState(node, topology, prevSynced);
List<AsyncPromise<Void>> futureEpochFutures = new ArrayList<>(current.futureEpochFutures);
AsyncPromise<Void> toComplete = !futureEpochFutures.isEmpty() ? futureEpochFutures.remove(0) : null;
@@ -255,59 +266,110 @@ public class TopologyManager implements ConfigurationService.Listener
return epochs.get(epoch);
}
- public Topologies forKeys(Keys keys, long minEpoch)
+ public Topologies withUnsyncedEpochs(Keys keys, long minEpoch, long maxEpoch)
{
Epochs snapshot = epochs;
- EpochState epochState = snapshot.get(snapshot.currentEpoch);
- Topology currentTopology = epochState.global.forKeys(keys);
+ if (maxEpoch == Long.MAX_VALUE) maxEpoch = snapshot.currentEpoch;
+ else Preconditions.checkState(snapshot.currentEpoch >= maxEpoch);
- if (minEpoch == Long.MAX_VALUE)
- minEpoch = currentTopology.epoch();
- else
- Preconditions.checkArgument(minEpoch <= currentTopology.epoch() && minEpoch >= 0);
+ EpochState maxEpochState = snapshot.get(maxEpoch);
+ if (minEpoch == maxEpoch && !snapshot.requiresHistoricalTopologiesFor(keys, maxEpoch))
+ return new Single(maxEpochState.global.forKeys(keys), true);
- if (currentTopology.epoch() == minEpoch && !epochs.requiresHistoricalTopologiesFor(keys))
+ int i = (int)(snapshot.currentEpoch - maxEpoch);
+ int limit = (int)(Math.min(1 + snapshot.currentEpoch - minEpoch, snapshot.epochs.length));
+ int count = limit - i;
+ while (limit < snapshot.epochs.length && !snapshot.epochs[limit].syncCompleteFor(keys))
{
- return new Topologies.Singleton(currentTopology, true);
+ ++count;
+ ++limit;
}
- else
+
+ Topologies.Multi topologies = new Topologies.Multi(count);
+ while (i < limit)
{
- Topologies.Multi topologies = new Topologies.Multi(2);
- topologies.add(currentTopology);
- for (int i=1; i<snapshot.epochs.length; i++)
- {
- epochState = snapshot.epochs[i];
- if (i > 1 && epochState.syncCompleteFor(keys) && epochState.epoch() < minEpoch)
- break;
-
- if (epochState.epoch() < minEpoch)
- topologies.add(epochState.global.forKeys(keys, epochState::shardIsUnsynced));
- else
- topologies.add(epochState.global.forKeys(keys));
- }
- return topologies;
+ EpochState epochState = snapshot.epochs[i++];
+ if (epochState.epoch() < minEpoch)
+ topologies.add(epochState.global.forKeys(keys, epochState::shardIsUnsynced));
+ else
+ topologies.add(epochState.global.forKeys(keys));
}
+
+ return topologies;
}
- public Topologies forKeys(Keys keys)
+ public Topologies withUnsyncedEpochs(Keys keys, long epoch)
{
- return forKeys(keys, Long.MAX_VALUE);
+ return withUnsyncedEpochs(keys, epoch, epoch);
+ }
+
+ public Topologies preciseEpochs(Keys keys, long minEpoch, long maxEpoch)
+ {
+ Epochs snapshot = epochs;
+
+ if (minEpoch == maxEpoch)
+ return new Single(snapshot.get(minEpoch).global().forKeys(keys), true);
+
+ int count = (int)(1 + maxEpoch - minEpoch);
+ Topologies.Multi topologies = new Topologies.Multi(count);
+ for (int i = 0 ; i < count ; ++i)
+ topologies.add(snapshot.get(minEpoch + count).global.forKeys(keys));
+
+ return topologies;
+ }
+
+ public Topologies preciseEpochs(Keys keys, long epoch)
+ {
+ return preciseEpochs(keys, epoch, epoch);
}
public Topologies forEpoch(Keys keys, long epoch)
{
- return new Topologies.Singleton(epochs.get(epoch).global.forKeys(keys), true);
+ return new Single(epochs.get(epoch).global().forKeys(keys), true);
+ }
+
+ public Shard forEpochIfKnown(Key key, long epoch)
+ {
+ EpochState epochState = epochs.get(epoch);
+ if (epochState == null)
+ return null;
+ return epochState.global().forKey(key);
+ }
+
+ public boolean hasEpoch(long epoch)
+ {
+ return epochs.get(epoch) != null;
+ }
+
+ public Topologies forTxn(Txn txn, long epoch)
+ {
+ return withUnsyncedEpochs(txn.keys(), epoch, epoch);
+ }
+
+ public Topologies forTxn(Txn txn, long minEpoch, long maxEpoch)
+ {
+ return withUnsyncedEpochs(txn.keys(), minEpoch, maxEpoch);
}
- public Topologies forTxn(Txn txn, long minEpoch)
+ public Topologies forTxn(Txn txn, Timestamp min, Timestamp max)
{
- return forKeys(txn.keys(), minEpoch);
+ return withUnsyncedEpochs(txn.keys(), min.epoch, max.epoch);
}
- public Topologies forTxn(Txn txn)
+ public Topologies unsyncForTxn(Txn txn, long epoch)
{
- return forKeys(txn.keys(), Long.MAX_VALUE);
+ return preciseEpochs(txn.keys(), epoch, epoch);
+ }
+
+ public Topology localForEpoch(long epoch)
+ {
+ return epochs.get(epoch).local();
+ }
+
+ public Topology globalForEpoch(long epoch)
+ {
+ return epochs.get(epoch).global();
}
public Topologies forEpoch(Txn txn, long epoch)
@@ -317,9 +379,13 @@ public class TopologyManager implements ConfigurationService.Listener
public long maxUnknownEpoch(Request request)
{
- if (!(request instanceof TxnRequest))
+ if (!(request instanceof EpochRequest))
return 0;
- return epochs.maxUnknownEpoch(((TxnRequest) request).scope());
+ long waitForEpoch = ((EpochRequest) request).waitForEpoch();
+ if (epochs.currentEpoch < waitForEpoch)
+ return waitForEpoch;
+
+ return 0;
}
}
diff --git a/accord-core/src/main/java/accord/txn/Dependencies.java b/accord-core/src/main/java/accord/txn/Dependencies.java
index 4e9e114..bc42dbc 100644
--- a/accord-core/src/main/java/accord/txn/Dependencies.java
+++ b/accord-core/src/main/java/accord/txn/Dependencies.java
@@ -1,41 +1,62 @@
package accord.txn;
+import java.util.Collections;
import java.util.Iterator;
+import java.util.Map;
import java.util.Map.Entry;
import java.util.NavigableMap;
import java.util.Objects;
import java.util.TreeMap;
+import accord.api.Key;
import accord.local.Command;
import accord.local.CommandStore;
+import accord.topology.KeyRanges;
+
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
// TODO: do not send Txn
// TODO: implementation efficiency
public class Dependencies implements Iterable<Entry<TxnId, Txn>>
{
+ // TEMPORARY: this is used ONLY for SimpleProgressLog.blockingState which may not know the homeKey
+ // or be able to invalidate a transaction it cannot witness anywhere
+ public static class TxnAndHomeKey
+ {
+ public final Txn txn;
+ public final Key homeKey;
+
+ TxnAndHomeKey(Txn txn, Key homeKey)
+ {
+ this.txn = txn;
+ this.homeKey = homeKey;
+ }
+ }
+
// TODO: encapsulate
- public final NavigableMap<TxnId, Txn> deps;
+ public final NavigableMap<TxnId, TxnAndHomeKey> deps;
public Dependencies()
{
this.deps = new TreeMap<>();
}
- public Dependencies(NavigableMap<TxnId, Txn> deps)
+ public Dependencies(NavigableMap<TxnId, TxnAndHomeKey> deps)
{
this.deps = deps;
}
public void add(Command command)
{
- add(command.txnId(), command.txn());
+ add(command.txnId(), command.txn(), command.homeKey());
}
@VisibleForTesting
- public Dependencies add(TxnId txnId, Txn txn)
+ public Dependencies add(TxnId txnId, Txn txn, Key homeKey)
{
- deps.put(txnId, txn);
+ Preconditions.checkState(homeKey != null);
+ deps.put(txnId, new TxnAndHomeKey(txn, homeKey));
return this;
}
@@ -61,21 +82,30 @@ public class Dependencies implements Iterable<Entry<TxnId, Txn>>
public Txn get(TxnId txnId)
{
- return deps.get(txnId);
+ return deps.get(txnId).txn;
}
- public Iterable<TxnId> on(CommandStore commandStore)
+ public Iterable<TxnId> on(CommandStore commandStore, Timestamp executeAt)
{
+ KeyRanges ranges = commandStore.ranges().since(executeAt.epoch);
+ if (ranges == null)
+ return Collections.emptyList();
+
return deps.entrySet()
.stream()
- .filter(e -> commandStore.intersects(e.getValue().keys()))
+ .filter(e -> commandStore.intersects(e.getValue().txn.keys(), ranges))
.map(Entry::getKey)::iterator;
}
+ public Key homeKey(TxnId txnId)
+ {
+ return deps.get(txnId).homeKey;
+ }
+
@Override
public Iterator<Entry<TxnId, Txn>> iterator()
{
- return deps.entrySet().iterator();
+ return deps.entrySet().stream().map(e -> Map.entry(e.getKey(), e.getValue().txn)).iterator();
}
public int size()
diff --git a/accord-core/src/main/java/accord/txn/Keys.java b/accord-core/src/main/java/accord/txn/Keys.java
index f5bfc97..b09985c 100644
--- a/accord-core/src/main/java/accord/txn/Keys.java
+++ b/accord-core/src/main/java/accord/txn/Keys.java
@@ -6,7 +6,7 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import accord.api.Key;
-import accord.api.KeyRange;
+import accord.topology.KeyRange;
import accord.topology.KeyRanges;
@SuppressWarnings("rawtypes")
@@ -53,6 +53,11 @@ public class Keys implements Iterable<Key>
return Arrays.binarySearch(keys, key);
}
+ public boolean contains(Key key)
+ {
+ return indexOf(key) >= 0;
+ }
+
public Key get(int indexOf)
{
return keys[indexOf];
@@ -102,11 +107,12 @@ public class Keys implements Iterable<Key>
return true;
}
- public Keys merge(Keys that)
+ public Keys union(Keys that)
{
int thisIdx = 0;
int thatIdx = 0;
- Key[] result = new Key[this.size() + that.size()];
+ Key[] noOp = keys.length >= that.keys.length ? this.keys : that.keys;
+ Key[] result = noOp;
int resultSize = 0;
while (thisIdx < this.size() && thatIdx < that.size())
@@ -117,23 +123,53 @@ public class Keys implements Iterable<Key>
Key minKey;
if (cmp == 0)
{
- minKey = thisKey;
thisIdx++;
thatIdx++;
+ if (result == noOp)
+ continue;
+ minKey = thisKey;
}
else if (cmp < 0)
{
- minKey = thisKey;
thisIdx++;
+ if (result == keys)
+ continue;
+ minKey = thisKey;
+ if (result == noOp)
+ {
+ resultSize = thatIdx;
+ result = new Key[resultSize + (keys.length - (thisIdx - 1)) + (that.keys.length - thatIdx)];
+ System.arraycopy(that.keys, 0, result, 0, resultSize);
+ }
}
else
{
- minKey = thatKey;
thatIdx++;
+ if (result == that.keys)
+ continue;
+ minKey = thatKey;
+ if (result == noOp)
+ {
+ resultSize = thisIdx;
+ result = new Key[resultSize + (keys.length - thisIdx) + (that.keys.length - (thatIdx - 1))];
+ System.arraycopy(this.keys, 0, result, 0, resultSize);
+ }
}
result[resultSize++] = minKey;
}
+ if (result == noOp)
+ {
+ if (noOp == keys && thatIdx == that.keys.length)
+ return this;
+ if (noOp == that.keys && thisIdx == keys.length)
+ return that;
+
+ resultSize = noOp == keys ? thisIdx : thatIdx;
+ result = new Key[resultSize + (keys.length - thisIdx) + (that.keys.length - thatIdx)];
+ System.arraycopy(noOp, 0, result, 0, resultSize);
+ }
+
while (thisIdx < this.size())
result[resultSize++] = this.keys[thisIdx++];
@@ -146,6 +182,48 @@ public class Keys implements Iterable<Key>
return new Keys(result);
}
+ public Keys intersect(Keys that)
+ {
+ int thisIdx = 0;
+ int thatIdx = 0;
+ Key[] noOp = keys.length <= that.keys.length ? this.keys : that.keys;
+ Key[] result = noOp;
+ int resultSize = 0;
+
+ while (thisIdx < this.size() && thatIdx < that.size())
+ {
+ Key thisKey = this.keys[thisIdx];
+ Key thatKey = that.keys[thatIdx];
+ int cmp = thisKey.compareTo(thatKey);
+ if (cmp == 0)
+ {
+ thisIdx++;
+ thatIdx++;
+ if (result != noOp)
+ result[resultSize] = thisKey;
+ resultSize++;
+ }
+ else
+ {
+ if (cmp < 0) thisIdx++;
+ else thatIdx++;
+ if (result == noOp)
+ {
+ result = new Key[resultSize + Math.min(keys.length - thisIdx, that.keys.length - thatIdx)];
+ System.arraycopy(noOp, 0, result, 0, resultSize);
+ }
+ }
+ }
+
+ if (result == noOp && resultSize == noOp.length)
+ return noOp == keys ? this : that;
+
+ if (resultSize < result.length)
+ result = Arrays.copyOf(result, resultSize);
+
+ return new Keys(result);
+ }
+
public boolean isEmpty()
{
return keys.length == 0;
@@ -173,6 +251,21 @@ public class Keys implements Iterable<Key>
return ceilIndex(0, keys.length, key);
}
+ public Keys with(Key key)
+ {
+ int insertPos = Arrays.binarySearch(keys, key);
+ if (insertPos >= 0)
+ return this;
+ insertPos = -1 - insertPos;
+
+ Key[] src = keys;
+ Key[] trg = new Key[src.length + 1];
+ System.arraycopy(src, 0, trg, 0, insertPos);
+ trg[insertPos] = key;
+ System.arraycopy(src, insertPos, trg, insertPos + 1, src.length - insertPos);
+ return new Keys(trg);
+ }
+
public Stream<Key> stream()
{
return Stream.of(keys);
@@ -214,7 +307,7 @@ public class Keys implements Iterable<Key>
return new Keys(keys);
}
- public Keys intersection(KeyRanges ranges)
+ public Keys intersect(KeyRanges ranges)
{
Key[] result = null;
int resultSize = 0;
diff --git a/accord-core/src/main/java/accord/txn/Txn.java b/accord-core/src/main/java/accord/txn/Txn.java
index ae2619d..4e6fec6 100644
--- a/accord-core/src/main/java/accord/txn/Txn.java
+++ b/accord-core/src/main/java/accord/txn/Txn.java
@@ -1,8 +1,6 @@
package accord.txn;
-import java.util.Comparator;
import java.util.Objects;
-import java.util.stream.Stream;
import accord.api.*;
import accord.local.*;
@@ -84,12 +82,12 @@ public class Txn
public String toString()
{
- return "read:" + read.toString() + (update != null ? ", update:" + update : "");
+ return "{read:" + read.toString() + (update != null ? ", update:" + update : "") + '}';
}
- public Data read(Command command, Keys keyScope)
+ public Data read(Command command, Keys keys)
{
- return keyScope.foldl(command.commandStore.ranges(), (key, accumulate) -> {
+ return keys.foldl(command.commandStore.ranges().at(command.executeAt().epoch), (key, accumulate) -> {
CommandStore commandStore = command.commandStore;
if (!commandStore.hashIntersects(key))
return accumulate;
@@ -98,69 +96,4 @@ public class Txn
return accumulate != null ? accumulate.merge(result) : result;
}, null);
}
-
- public Timestamp maxConflict(CommandStore commandStore)
- {
- return maxConflict(commandStore, keys());
- }
-
- public Stream<Command> conflictsMayExecuteBefore(CommandStore commandStore, Timestamp mayExecuteBefore)
- {
- return keys().stream().flatMap(key -> {
- CommandsForKey forKey = commandStore.commandsForKey(key);
- return Stream.concat(
- forKey.uncommitted.headMap(mayExecuteBefore, false).values().stream(),
- // TODO: only return latest of Committed?
- forKey.committedByExecuteAt.headMap(mayExecuteBefore, false).values().stream()
- );
- });
- }
-
- public Stream<Command> uncommittedStartedBefore(CommandStore commandStore, TxnId startedBefore)
- {
- return keys().stream().flatMap(key -> {
- CommandsForKey forKey = commandStore.commandsForKey(key);
- return forKey.uncommitted.headMap(startedBefore, false).values().stream();
- });
- }
-
- public Stream<Command> committedStartedBefore(CommandStore commandStore, TxnId startedBefore)
- {
- return keys().stream().flatMap(key -> {
- CommandsForKey forKey = commandStore.commandsForKey(key);
- return forKey.committedById.headMap(startedBefore, false).values().stream();
- });
- }
-
- public Stream<Command> uncommittedStartedAfter(CommandStore commandStore, TxnId startedAfter)
- {
- return keys().stream().flatMap(key -> {
- CommandsForKey forKey = commandStore.commandsForKey(key);
- return forKey.uncommitted.tailMap(startedAfter, false).values().stream();
- });
- }
-
- public Stream<Command> committedExecutesAfter(CommandStore commandStore, TxnId startedAfter)
- {
- return keys().stream().flatMap(key -> {
- CommandsForKey forKey = commandStore.commandsForKey(key);
- return forKey.committedByExecuteAt.tailMap(startedAfter, false).values().stream();
- });
- }
-
- public void register(CommandStore commandStore, Command command)
- {
- assert commandStore == command.commandStore;
- keys().forEach(key -> commandStore.commandsForKey(key).register(command));
- }
-
- protected Timestamp maxConflict(CommandStore commandStore, Keys keys)
- {
- return keys.stream()
- .map(commandStore::commandsForKey)
- .map(CommandsForKey::max)
- .max(Comparator.naturalOrder())
- .orElse(Timestamp.NONE);
- }
-
}
diff --git a/accord-core/src/main/java/accord/txn/TxnId.java b/accord-core/src/main/java/accord/txn/TxnId.java
index 47c9aa8..752a59b 100644
--- a/accord-core/src/main/java/accord/txn/TxnId.java
+++ b/accord-core/src/main/java/accord/txn/TxnId.java
@@ -2,6 +2,7 @@ package accord.txn;
import accord.local.Node.Id;
+// TODO: consider moving homeKey concept here? might make later optimisations of its storage trickier
public class TxnId extends Timestamp
{
public TxnId(Timestamp timestamp)
diff --git a/accord-core/src/main/java/accord/txn/Writes.java b/accord-core/src/main/java/accord/txn/Writes.java
index 67bc51c..b919ad4 100644
--- a/accord-core/src/main/java/accord/txn/Writes.java
+++ b/accord-core/src/main/java/accord/txn/Writes.java
@@ -2,6 +2,7 @@ package accord.txn;
import accord.api.Write;
import accord.local.CommandStore;
+import accord.topology.KeyRanges;
public class Writes
{
@@ -21,7 +22,11 @@ public class Writes
if (write == null)
return;
- keys.foldl(commandStore.ranges(), (key, accumulate) -> {
+ KeyRanges ranges = commandStore.ranges().since(executeAt.epoch);
+ if (ranges == null)
+ return;
+
+ keys.foldl(ranges, (key, accumulate) -> {
if (commandStore.hashIntersects(key))
write.apply(key, executeAt, commandStore.store());
return accumulate;
diff --git a/accord-core/src/main/java/accord/utils/DeterministicIdentitySet.java b/accord-core/src/main/java/accord/utils/DeterministicIdentitySet.java
index 90ba39d..70fd7ca 100644
--- a/accord-core/src/main/java/accord/utils/DeterministicIdentitySet.java
+++ b/accord-core/src/main/java/accord/utils/DeterministicIdentitySet.java
@@ -98,4 +98,10 @@ public class DeterministicIdentitySet<T> extends AbstractSet<T>
next = next.next;
}
}
+
+ @Override
+ public String toString()
+ {
+ return super.toString();
+ }
}
diff --git a/accord-core/src/main/java/accord/utils/IndexedIntFunction.java b/accord-core/src/main/java/accord/utils/IndexedIntFunction.java
new file mode 100644
index 0000000..90a552b
--- /dev/null
+++ b/accord-core/src/main/java/accord/utils/IndexedIntFunction.java
@@ -0,0 +1,6 @@
+package accord.utils;
+
+public interface IndexedIntFunction<T>
+{
+ int apply(int i, T t, int v);
+}
diff --git a/accord-core/src/main/java/accord/utils/IntrusiveLinkedList.java b/accord-core/src/main/java/accord/utils/IntrusiveLinkedList.java
new file mode 100644
index 0000000..43579d3
--- /dev/null
+++ b/accord-core/src/main/java/accord/utils/IntrusiveLinkedList.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package accord.utils;
+
+import java.util.Iterator;
+import java.util.Spliterator;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+
+import static java.util.Spliterators.spliteratorUnknownSize;
+
+/**
+ * TODO: copied from Cassandra; extract shared utilities library
+ *
+ * A simple intrusive double-linked list for maintaining a list of tasks,
+ * useful for invalidating queued ordered tasks
+ */
+@SuppressWarnings("unchecked")
+public class IntrusiveLinkedList<O extends IntrusiveLinkedListNode> extends IntrusiveLinkedListNode
+{
+ public IntrusiveLinkedList()
+ {
+ prev = next = this;
+ }
+
+ public void add(O add)
+ {
+ assert add.prev == null && add.next == null;
+ IntrusiveLinkedListNode after = this;
+ IntrusiveLinkedListNode before = prev;
+ add.next = after;
+ add.prev = before;
+ before.next = add;
+ after.prev = add;
+ }
+
+ public O poll()
+ {
+ if (isEmpty())
+ return null;
+
+ IntrusiveLinkedListNode next = this.next;
+ next.remove();
+ return (O) next;
+ }
+
+ public boolean isEmpty()
+ {
+ return next == this;
+ }
+
+ public Stream<O> stream()
+ {
+ Iterator<O> iterator = new Iterator<O>()
+ {
+ IntrusiveLinkedListNode next = IntrusiveLinkedList.this.next;
+
+ @Override
+ public boolean hasNext()
+ {
+ return next != IntrusiveLinkedList.this;
+ }
+
+ @Override
+ public O next()
+ {
+ O result = (O)next;
+ next = next.next;
+ return result;
+ }
+ };
+
+ return StreamSupport.stream(spliteratorUnknownSize(iterator, Spliterator.IMMUTABLE), false);
+ }
+}
+
diff --git a/accord-core/src/main/java/accord/utils/IntrusiveLinkedListNode.java b/accord-core/src/main/java/accord/utils/IntrusiveLinkedListNode.java
new file mode 100644
index 0000000..a021a5c
--- /dev/null
+++ b/accord-core/src/main/java/accord/utils/IntrusiveLinkedListNode.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package accord.utils;
+
+/**
+ * TODO: copied from Cassandra; extract shared utilities library
+ */
+public abstract class IntrusiveLinkedListNode
+{
+ IntrusiveLinkedListNode prev;
+ IntrusiveLinkedListNode next;
+
+ protected boolean isFree()
+ {
+ return next == null;
+ }
+
+ protected void remove()
+ {
+ if (next != null)
+ {
+ prev.next = next;
+ next.prev = prev;
+ next = null;
+ prev = null;
+ }
+ }
+}
diff --git a/accord-core/src/test/java/accord/KeysTest.java b/accord-core/src/test/java/accord/KeysTest.java
index 1f06656..f039a30 100644
--- a/accord-core/src/test/java/accord/KeysTest.java
+++ b/accord-core/src/test/java/accord/KeysTest.java
@@ -4,8 +4,8 @@ import java.util.ArrayList;
import java.util.List;
import accord.api.Key;
-import accord.api.KeyRange;
import accord.impl.IntKey;
+import accord.topology.KeyRange;
import accord.topology.KeyRanges;
import accord.txn.Keys;
@@ -32,24 +32,24 @@ public class KeysTest
{
assertEquals(keys(150, 250),
keys(100, 150, 200, 250, 300)
- .intersection(ranges(r(125, 175), r(225, 275))));
+ .intersect(ranges(r(125, 175), r(225, 275))));
assertEquals(keys(101, 199, 200),
keys(99, 100, 101, 199, 200, 201)
- .intersection(ranges(r(100, 200))));
+ .intersect(ranges(r(100, 200))));
assertEquals(keys(101, 199, 200, 201, 299, 300),
keys(99, 100, 101, 199, 200, 201, 299, 300, 301)
- .intersection(ranges(r(100, 200), r(200, 300))));
+ .intersect(ranges(r(100, 200), r(200, 300))));
}
@Test
void mergeTest()
{
assertEquals(keys(0, 1, 2, 3, 4),
- keys(0, 1, 2, 3, 4).merge(keys(0, 1, 2, 3, 4)));
+ keys(0, 1, 2, 3, 4).union(keys(0, 1, 2, 3, 4)));
assertEquals(keys(0, 1, 2, 3, 4),
- keys(0, 1).merge(keys(2, 3, 4)));
+ keys(0, 1).union(keys(2, 3, 4)));
assertEquals(keys(0, 1, 2, 3, 4),
- keys(0, 2, 4).merge(keys(1, 3)));
+ keys(0, 2, 4).union(keys(1, 3)));
}
@Test
diff --git a/accord-core/src/test/java/accord/Utils.java b/accord-core/src/test/java/accord/Utils.java
index d41860d..9ad4bcb 100644
--- a/accord-core/src/test/java/accord/Utils.java
+++ b/accord-core/src/test/java/accord/Utils.java
@@ -1,6 +1,6 @@
package accord;
-import accord.api.KeyRange;
+import accord.topology.KeyRange;
import accord.local.Node;
import accord.impl.mock.MockStore;
import accord.topology.KeyRanges;
@@ -66,12 +66,12 @@ public class Utils
public static Txn writeTxn(Keys keys)
{
- return new Txn(keys, MockStore.READ, MockStore.QUERY, MockStore.UPDATE);
+ return new Txn(keys, MockStore.read(keys), MockStore.QUERY, MockStore.UPDATE);
}
public static Txn readTxn(Keys keys)
{
- return new Txn(keys, MockStore.READ, MockStore.QUERY);
+ return new Txn(keys, MockStore.read(keys), MockStore.QUERY);
}
public static Shard shard(KeyRange range, List<Node.Id> nodes, Set<Node.Id> fastPath)
diff --git a/accord-core/src/test/java/accord/burn/BurnTest.java b/accord-core/src/test/java/accord/burn/BurnTest.java
index 0501417..1fb9663 100644
--- a/accord-core/src/test/java/accord/burn/BurnTest.java
+++ b/accord-core/src/test/java/accord/burn/BurnTest.java
@@ -219,8 +219,8 @@ public class BurnTest
{
for (int i = 0 ; i < requests.length ; ++i)
{
- logger.debug("{}", requests[i]);
- logger.debug("\t\t" + replies[i]);
+ logger.info("{}", requests[i]);
+ logger.info("\t\t" + replies[i]);
}
throw new AssertionError("Incomplete set of responses");
}
diff --git a/accord-core/src/test/java/accord/burn/TopologyUpdate.java b/accord-core/src/test/java/accord/burn/TopologyUpdate.java
index ae22d60..9125c0d 100644
--- a/accord-core/src/test/java/accord/burn/TopologyUpdate.java
+++ b/accord-core/src/test/java/accord/burn/TopologyUpdate.java
@@ -1,11 +1,12 @@
package accord.burn;
-import accord.api.KeyRange;
+import accord.api.Key;
import accord.api.Result;
import accord.api.TestableConfigurationService;
import accord.local.Command;
import accord.local.Node;
import accord.local.Status;
+import accord.topology.KeyRange;
import accord.topology.KeyRanges;
import accord.topology.Shard;
import accord.topology.Topology;
@@ -13,7 +14,6 @@ import accord.txn.*;
import accord.utils.MessageTask;
import com.google.common.base.Preconditions;
import com.google.common.collect.Sets;
-import org.apache.cassandra.utils.concurrent.AsyncPromise;
import org.apache.cassandra.utils.concurrent.Future;
import org.apache.cassandra.utils.concurrent.ImmediateFuture;
import org.slf4j.Logger;
@@ -21,7 +21,6 @@ import org.slf4j.LoggerFactory;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
import java.util.function.Function;
@@ -38,41 +37,55 @@ public class TopologyUpdate
private final Status status;
private final TxnId txnId;
private final Txn txn;
+ private final Key homeKey;
private final Timestamp executeAt;
+ private final long epoch;
private final Dependencies deps;
private final Writes writes;
private final Result result;
- public CommandSync(Command command)
+ public CommandSync(Command command, long epoch)
{
Preconditions.checkArgument(command.hasBeen(Status.PreAccepted));
this.txnId = command.txnId();
this.txn = command.txn();
+ this.homeKey = command.homeKey();
this.status = command.status();
this.executeAt = command.executeAt();
this.deps = command.savedDeps();
this.writes = command.writes();
this.result = command.result();
+ this.epoch = epoch;
}
+
public void process(Node node)
{
- node.forEachLocal(txn, commandStore -> {
+ if (!node.topology().hasEpoch(txnId.epoch))
+ {
+ node.configService().fetchTopologyForEpoch(txnId.epoch);
+ node.topology().awaitEpoch(txnId.epoch).addListener(() -> process(node));
+ return;
+ }
+
+ Key progressKey = node.trySelectProgressKey(txnId, txn.keys, homeKey); // likely to be null, unless flip-flop of ownership
+ // TODO: can skip the homeKey if it's not a participating key in the transaction
+ node.forEachLocalSince(txn.keys, epoch, commandStore -> {
switch (status)
{
case PreAccepted:
- commandStore.command(txnId).witness(txn);
+ commandStore.command(txnId).preaccept(txn, homeKey, progressKey);
break;
case Accepted:
- commandStore.command(txnId).accept(Ballot.ZERO, txn, executeAt, deps);
+ commandStore.command(txnId).accept(Ballot.ZERO, txn, homeKey, progressKey, executeAt, deps);
break;
case Committed:
case ReadyToExecute:
- commandStore.command(txnId).commit(txn, deps, executeAt);
+ commandStore.command(txnId).commit(txn, homeKey, progressKey, executeAt, deps);
break;
case Executed:
case Applied:
- commandStore.command(txnId).apply(txn, deps, executeAt, writes, result);
+ commandStore.command(txnId).apply(txn, homeKey, progressKey, executeAt, deps, writes, result);
break;
default:
throw new IllegalStateException();
@@ -120,7 +133,7 @@ public class TopologyUpdate
private static Stream<MessageTask> syncEpochCommands(Node node, long epoch, KeyRanges ranges, Function<CommandSync, Collection<Node.Id>> recipients, long forEpoch, boolean committedOnly)
{
Map<TxnId, CommandSync> syncMessages = new ConcurrentHashMap<>();
- Consumer<Command> commandConsumer = command -> syncMessages.put(command.txnId(), new CommandSync(command));
+ Consumer<Command> commandConsumer = command -> syncMessages.put(command.txnId(), new CommandSync(command, epoch));
if (committedOnly)
node.forEachLocal(commandStore -> commandStore.forCommittedInEpoch(ranges, epoch, commandConsumer));
else
@@ -137,7 +150,7 @@ public class TopologyUpdate
{
Topology syncTopology = node.configService().getTopologyForEpoch(syncEpoch);
Topology localTopology = syncTopology.forNode(node.id());
- Function<CommandSync, Collection<Node.Id>> allNodes = cmd -> node.topology().forTxn(cmd.txn).nodes();
+ Function<CommandSync, Collection<Node.Id>> allNodes = cmd -> node.topology().forTxn(cmd.txn, syncEpoch).nodes();
KeyRanges ranges = localTopology.ranges();
Stream<MessageTask> messageStream = Stream.empty();
@@ -158,7 +171,7 @@ public class TopologyUpdate
Topology localTopology = syncTopology.forNode(node.id());
Topology nextTopology = node.configService().getTopologyForEpoch(nextEpoch);
Function<CommandSync, Collection<Node.Id>> nextNodes = cmd -> allNodesFor(cmd.txn, nextTopology);
- Function<CommandSync, Collection<Node.Id>> allNodes = cmd -> node.topology().forTxn(cmd.txn).nodes();
+ Function<CommandSync, Collection<Node.Id>> allNodes = cmd -> node.topology().forTxn(cmd.txn, syncEpoch).nodes();
// backfill new replicas with operations from prior epochs
Stream<MessageTask> messageStream = Stream.empty();
diff --git a/accord-core/src/test/java/accord/coordinate/CoordinateTest.java b/accord-core/src/test/java/accord/coordinate/CoordinateTest.java
index e306dae..0bd9361 100644
--- a/accord-core/src/test/java/accord/coordinate/CoordinateTest.java
+++ b/accord-core/src/test/java/accord/coordinate/CoordinateTest.java
@@ -10,6 +10,7 @@ import accord.txn.TxnId;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import static accord.Utils.id;
import static accord.Utils.ids;
import static accord.Utils.writeTxn;
import static accord.impl.IntKey.keys;
@@ -26,7 +27,7 @@ public class CoordinateTest
TxnId txnId = new TxnId(1, 100, 0, node.id());
Txn txn = writeTxn(keys(10));
- Result result = Coordinate.execute(node, txnId, txn).get();
+ Result result = Coordinate.execute(node, txnId, txn, txn.keys().get(0)).get();
Assertions.assertEquals(MockStore.RESULT, result);
}
}
@@ -41,9 +42,8 @@ public class CoordinateTest
Node node = cluster.get(1);
Assertions.assertNotNull(node);
- TxnId txnId = new TxnId(1, 100, 0, node.id());
Txn txn = writeTxn(keys(10));
- Result result = Coordinate.execute(node, txnId, txn).get();
+ Result result = cluster.get(id(1)).coordinate(txn).get();
Assertions.assertEquals(MockStore.RESULT, result);
}
}
@@ -52,7 +52,7 @@ public class CoordinateTest
{
TxnId txnId = new TxnId(1, clock, 0, node.id());
Txn txn = writeTxn(keys);
- Result result = Coordinate.execute(node, txnId, txn).get();
+ Result result = Coordinate.execute(node, txnId, txn, node.selectHomeKey(txnId, txn.keys)).get();
Assertions.assertEquals(MockStore.RESULT, result);
return txnId;
}
diff --git a/accord-core/src/test/java/accord/coordinate/PreacceptTrackerTest.java b/accord-core/src/test/java/accord/coordinate/PreacceptTrackerTest.java
index b357cfc..5ae5316 100644
--- a/accord-core/src/test/java/accord/coordinate/PreacceptTrackerTest.java
+++ b/accord-core/src/test/java/accord/coordinate/PreacceptTrackerTest.java
@@ -1,20 +1,17 @@
package accord.coordinate;
import accord.Utils;
-import accord.api.KeyRange;
import accord.coordinate.tracking.FastPathTracker;
import accord.impl.TopologyUtils;
import accord.local.Node;
import accord.topology.KeyRanges;
import accord.topology.Shard;
import accord.topology.Topology;
-import accord.txn.Keys;
+
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import static accord.Utils.*;
-import static accord.impl.IntKey.keys;
-import static accord.impl.IntKey.range;
public class PreacceptTrackerTest
{
@@ -99,10 +96,10 @@ public class PreacceptTrackerTest
responses.recordSuccess(ids[0], true);
assertResponseState(responses, false, false, false, true);
- responses.recordFailure(ids[1]);
+ responses.failure(ids[1]);
assertResponseState(responses, false, false, false, true);
- responses.recordFailure(ids[2]);
+ responses.failure(ids[2]);
assertResponseState(responses, false, false, true, false);
}
diff --git a/accord-core/src/test/java/accord/coordinate/RecoverTest.java b/accord-core/src/test/java/accord/coordinate/RecoverTest.java
index 67b1405..481d3ed 100644
--- a/accord-core/src/test/java/accord/coordinate/RecoverTest.java
+++ b/accord-core/src/test/java/accord/coordinate/RecoverTest.java
@@ -78,13 +78,13 @@ public class RecoverTest
TxnId txnId1 = new TxnId(1, 100, 0, id(100));
Txn txn1 = writeTxn(Keys.of(key));
- assertTimeout(Coordinate.execute(cluster.get(1), txnId1, txn1));
+ assertTimeout(Coordinate.execute(cluster.get(1), txnId1, txn1, key));
TxnId txnId2 = new TxnId(1, 50, 0, id(101));
Txn txn2 = writeTxn(Keys.of(key));
cluster.networkFilter.clear();
cluster.networkFilter.isolate(ids(1, 7));
- assertTimeout(Coordinate.execute(cluster.get(9), txnId2, txn2));
+ assertTimeout(Coordinate.execute(cluster.get(9), txnId2, txn2, key));
cluster.nodes(ids(1, 4)).forEach(n -> assertStatus(n, key, txnId1, Status.Accepted));
cluster.nodes(ids(5, 6)).forEach(n -> assertStatus(n, key, txnId1, Status.PreAccepted));
@@ -96,7 +96,7 @@ public class RecoverTest
//
cluster.networkFilter.clear();
cluster.networkFilter.isolate(ids(1, 4));
- Coordinate.recover(cluster.get(8), txnId2, txn2).get();
+ Coordinate.recover(cluster.get(8), txnId2, txn2, key).get();
List<Node> nodes = cluster.nodes(ids(5, 9));
Assertions.assertTrue(txnId2.compareTo(txnId1) < 0);
diff --git a/accord-core/src/test/java/accord/coordinate/TopologyChangeTest.java b/accord-core/src/test/java/accord/coordinate/TopologyChangeTest.java
index de05398..3668c85 100644
--- a/accord-core/src/test/java/accord/coordinate/TopologyChangeTest.java
+++ b/accord-core/src/test/java/accord/coordinate/TopologyChangeTest.java
@@ -1,6 +1,5 @@
package accord.coordinate;
-import accord.api.KeyRange;
import accord.impl.mock.EpochSync;
import accord.impl.mock.MockCluster;
import accord.impl.mock.MockConfigurationService;
@@ -9,6 +8,7 @@ import accord.local.Command;
import accord.local.Node;
import accord.local.Status;
import accord.messages.Accept;
+import accord.topology.KeyRange;
import accord.topology.Topology;
import accord.txn.Keys;
import accord.txn.Txn;
@@ -17,6 +17,7 @@ import accord.utils.EpochFunction;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import java.util.ArrayList;
import java.util.Set;
import java.util.stream.Collectors;
@@ -55,7 +56,7 @@ public class TopologyChangeTest
TxnId txnId1 = node1.nextTxnId();
Txn txn1 = writeTxn(keys);
node1.coordinate(txnId1, txn1).get();
- node1.forEachLocal(keys, commands -> {
+ node1.forEachLocal(keys, 1, commands -> {
Command command = commands.command(txnId1);
Assertions.assertTrue(command.savedDeps().isEmpty());
});
@@ -69,7 +70,7 @@ public class TopologyChangeTest
// new nodes should have the previous epochs operation as a dependency
cluster.nodes(4, 5, 6).forEach(node -> {
- node.forEachLocal(keys, commands -> {
+ node.forEachLocal(keys, 2, commands -> {
Command command = commands.command(txnId2);
Assertions.assertTrue(command.savedDeps().contains(txnId1));
});
@@ -77,9 +78,9 @@ public class TopologyChangeTest
// ...and participated in consensus
cluster.nodes(1, 2, 3).forEach(node -> {
- node.forEachLocal(keys, commands -> {
+ node.forEachLocal(keys, 1, commands -> {
Command command = commands.command(txnId2);
- Assertions.assertTrue(command.hasBeen(Status.Committed));
+ Assertions.assertTrue(command.hasBeen(Status.Accepted));
});
});
}
@@ -100,21 +101,20 @@ public class TopologyChangeTest
RecordingMessageSink messageSink = (RecordingMessageSink) node1.messageSink();
messageSink.clearHistory();
TxnId txnId1 = coordinate(node1, keys);
- node1.forEachLocal(keys, commands -> {
+ node1.forEachLocal(keys, 1, commands -> {
Command command = commands.command(txnId1);
Assertions.assertTrue(command.savedDeps().isEmpty());
});
// check there was no accept phase
- Assertions.assertFalse(messageSink.requests.stream().anyMatch(env -> env.payload instanceof Accept));
-
+ Assertions.assertFalse(new ArrayList<>(messageSink.requests).stream().anyMatch(env -> env.payload instanceof Accept));
cluster.configServices(1, 2, 3).forEach(config -> config.reportTopology(topology2));
messageSink.clearHistory();
// post epoch change, there _should_ be accepts, but with the original timestamp
TxnId txnId2 = coordinate(node1, keys);
- Set<Node.Id> accepts = messageSink.requests.stream()
+ Set<Node.Id> accepts = new ArrayList<>(messageSink.requests).stream()
.filter(env -> env.payload instanceof Accept).map(env -> {
Accept accept = (Accept) env.payload;
Assertions.assertEquals(txnId2, accept.txnId);
@@ -122,7 +122,7 @@ public class TopologyChangeTest
}).collect(Collectors.toSet());
Assertions.assertEquals(idSet(1, 2, 3), accepts);
- node1.forEachLocal(keys, commands -> {
+ node1.forEachLocal(keys, 2, commands -> {
Command command = commands.command(txnId2);
Assertions.assertTrue(command.hasBeen(Status.Committed));
Assertions.assertTrue(command.savedDeps().contains(txnId1));
@@ -134,8 +134,8 @@ public class TopologyChangeTest
// post sync, fast path should be working again, and there should be no accept phase
messageSink.clearHistory();
TxnId txnId3 = coordinate(node1, keys);
- Assertions.assertFalse(messageSink.requests.stream().anyMatch(env -> env.payload instanceof Accept));
- node1.forEachLocal(keys, commands -> {
+ Assertions.assertFalse(new ArrayList<>(messageSink.requests).stream().anyMatch(env -> env.payload instanceof Accept));
+ node1.forEachLocal(keys, 2, commands -> {
Command command = commands.command(txnId3);
Assertions.assertTrue(command.hasBeen(Status.Committed));
Assertions.assertTrue(command.savedDeps().contains(txnId1));
diff --git a/accord-core/src/test/java/accord/coordinate/tracking/QuorumTrackerTest.java b/accord-core/src/test/java/accord/coordinate/tracking/QuorumTrackerTest.java
index da5f03e..8b59624 100644
--- a/accord-core/src/test/java/accord/coordinate/tracking/QuorumTrackerTest.java
+++ b/accord-core/src/test/java/accord/coordinate/tracking/QuorumTrackerTest.java
@@ -1,9 +1,9 @@
package accord.coordinate.tracking;
import accord.Utils;
-import accord.api.KeyRange;
import accord.impl.TopologyUtils;
import accord.local.Node;
+import accord.topology.KeyRange;
import accord.topology.KeyRanges;
import accord.topology.Shard;
import accord.topology.Topology;
@@ -40,13 +40,13 @@ public class QuorumTrackerTest
Topology subTopology = topology(topology.get(0));
QuorumTracker responses = new QuorumTracker(topologies(subTopology));
- responses.recordSuccess(ids[0]);
+ responses.success(ids[0]);
assertResponseState(responses, false, false, true);
- responses.recordSuccess(ids[1]);
+ responses.success(ids[1]);
assertResponseState(responses, true, false, true);
- responses.recordSuccess(ids[2]);
+ responses.success(ids[2]);
assertResponseState(responses, true, false, false);
}
@@ -59,14 +59,14 @@ public class QuorumTrackerTest
Topology subTopology = topology(topology.get(0));
QuorumTracker responses = new QuorumTracker(topologies(subTopology));
- responses.recordSuccess(ids[0]);
+ responses.success(ids[0]);
assertResponseState(responses, false, false, true);
- responses.recordSuccess(ids[1]);
+ responses.success(ids[1]);
assertResponseState(responses, true, false, true);
Assertions.assertFalse(subTopology.get(0).nodes.contains(ids[4]));
- responses.recordSuccess(ids[4]);
+ responses.success(ids[4]);
assertResponseState(responses, true, false, true);
}
@@ -76,13 +76,13 @@ public class QuorumTrackerTest
Topology subTopology = topology(topology.get(0));
QuorumTracker responses = new QuorumTracker(topologies(subTopology));
- responses.recordSuccess(ids[0]);
+ responses.success(ids[0]);
assertResponseState(responses, false, false, true);
- responses.recordFailure(ids[1]);
+ responses.failure(ids[1]);
assertResponseState(responses, false, false, true);
- responses.recordFailure(ids[2]);
+ responses.failure(ids[2]);
assertResponseState(responses, false, true, false);
}
@@ -100,11 +100,11 @@ public class QuorumTrackerTest
Assertions.assertSame(subTopology.get(1), responses.unsafeGet(1).shard);
Assertions.assertSame(subTopology.get(2), responses.unsafeGet(2).shard);
- responses.recordSuccess(ids[1]);
+ responses.success(ids[1]);
assertResponseState(responses, false, false, true);
- responses.recordSuccess(ids[2]);
+ responses.success(ids[2]);
assertResponseState(responses, false, false, true);
- responses.recordSuccess(ids[3]);
+ responses.success(ids[3]);
assertResponseState(responses, true, false, true);
}
@@ -117,14 +117,14 @@ public class QuorumTrackerTest
QuorumTracker responses = new QuorumTracker(topologies(topology2, topology1));
- responses.recordSuccess(id(1));
+ responses.success(id(1));
assertResponseState(responses, false, false, true);
- responses.recordSuccess(id(2));
+ responses.success(id(2));
assertResponseState(responses, false, false, true);
- responses.recordSuccess(id(4));
+ responses.success(id(4));
assertResponseState(responses, false, false, true);
- responses.recordSuccess(id(5));
+ responses.success(id(5));
assertResponseState(responses, true, false, true);
}
@@ -137,14 +137,14 @@ public class QuorumTrackerTest
QuorumTracker responses = new QuorumTracker(topologies(topology2, topology1));
- responses.recordSuccess(id(1));
+ responses.success(id(1));
assertResponseState(responses, false, false, true);
- responses.recordSuccess(id(2));
+ responses.success(id(2));
assertResponseState(responses, false, false, true);
- responses.recordFailure(id(4));
+ responses.failure(id(4));
assertResponseState(responses, false, false, true);
- responses.recordFailure(id(5));
+ responses.failure(id(5));
assertResponseState(responses, false, true, true);
}
}
diff --git a/accord-core/src/test/java/accord/impl/IntHashKey.java b/accord-core/src/test/java/accord/impl/IntHashKey.java
index 796656b..2be0bc4 100644
--- a/accord-core/src/test/java/accord/impl/IntHashKey.java
+++ b/accord-core/src/test/java/accord/impl/IntHashKey.java
@@ -6,7 +6,7 @@ import java.util.Objects;
import java.util.zip.CRC32C;
import accord.api.Key;
-import accord.api.KeyRange;
+import accord.topology.KeyRange;
import accord.topology.KeyRanges;
import accord.txn.Keys;
diff --git a/accord-core/src/test/java/accord/impl/IntKey.java b/accord-core/src/test/java/accord/impl/IntKey.java
index 6292658..ca84f27 100644
--- a/accord-core/src/test/java/accord/impl/IntKey.java
+++ b/accord-core/src/test/java/accord/impl/IntKey.java
@@ -6,7 +6,7 @@ import java.util.Objects;
import java.util.function.BiFunction;
import accord.api.Key;
-import accord.api.KeyRange;
+import accord.topology.KeyRange;
import accord.topology.KeyRanges;
import accord.txn.Keys;
diff --git a/accord-core/src/test/java/accord/impl/TopologyFactory.java b/accord-core/src/test/java/accord/impl/TopologyFactory.java
index 7481867..2009121 100644
--- a/accord-core/src/test/java/accord/impl/TopologyFactory.java
+++ b/accord-core/src/test/java/accord/impl/TopologyFactory.java
@@ -3,7 +3,7 @@ package accord.impl;
import accord.local.Node;
import accord.api.Key;
-import accord.api.KeyRange;
+import accord.topology.KeyRange;
import accord.topology.KeyRanges;
import accord.topology.Topology;
diff --git a/accord-core/src/test/java/accord/impl/TopologyUtils.java b/accord-core/src/test/java/accord/impl/TopologyUtils.java
index 490af35..46cf530 100644
--- a/accord-core/src/test/java/accord/impl/TopologyUtils.java
+++ b/accord-core/src/test/java/accord/impl/TopologyUtils.java
@@ -1,7 +1,7 @@
package accord.impl;
import accord.api.Key;
-import accord.api.KeyRange;
+import accord.topology.KeyRange;
import accord.local.Node;
import accord.topology.KeyRanges;
import accord.topology.Shard;
diff --git a/accord-core/src/test/java/accord/impl/basic/Cluster.java b/accord-core/src/test/java/accord/impl/basic/Cluster.java
index 2a1429d..d965a36 100644
--- a/accord-core/src/test/java/accord/impl/basic/Cluster.java
+++ b/accord-core/src/test/java/accord/impl/basic/Cluster.java
@@ -18,6 +18,7 @@ import java.util.function.Supplier;
import accord.api.MessageSink;
import accord.burn.BurnTestConfigurationService;
import accord.local.CommandStores;
+import accord.impl.SimpleProgressLog;
import accord.local.Node;
import accord.local.Node.Id;
import accord.api.Scheduler;
@@ -35,6 +36,7 @@ import org.slf4j.LoggerFactory;
public class Cluster implements Scheduler
{
private static final Logger logger = LoggerFactory.getLogger(Cluster.class);
+ private static final Logger trace = LoggerFactory.getLogger("accord.impl.basic.Trace");
final Function<Id, Node> lookup;
final PendingQueue pending;
@@ -62,7 +64,7 @@ public class Cluster implements Scheduler
private void add(Packet packet)
{
boolean isReply = packet.message instanceof Reply;
- logger.trace("{} {} {}", clock++, isReply ? "RPLY" : "SEND", packet);
+ trace.trace("{} {} {}", clock++, isReply ? "RPLY" : "SEND", packet);
if (lookup.apply(packet.dst) == null) responseSink.accept(packet);
else pending.add(packet);
}
@@ -90,16 +92,18 @@ public class Cluster implements Scheduler
{
Packet deliver = (Packet) next;
Node on = lookup.apply(deliver.dst);
+
+ // TODO (now): random drop chance independent of partition; also port flaky connections etc. from simulator
// Drop the message if it goes across the partition
boolean drop = ((Packet) next).src.id >= 0 &&
!(partitionSet.contains(deliver.src) && partitionSet.contains(deliver.dst)
|| !partitionSet.contains(deliver.src) && !partitionSet.contains(deliver.dst));
if (drop)
{
- logger.debug("{} DROP[{}] {}", clock++, on.epoch(), deliver);
+ logger.trace("{} DROP[{}] {}", clock++, on.epoch(), deliver);
return true;
}
- logger.debug("{} RECV[{}] {}", clock++, on.epoch(), deliver);
+ trace.trace("{} RECV[{}] {}", clock++, on.epoch(), deliver);
if (deliver.message instanceof Reply)
{
Reply reply = (Reply) deliver.message;
@@ -153,7 +157,8 @@ public class Cluster implements Scheduler
MessageSink messageSink = sinks.create(node, randomSupplier.get());
BurnTestConfigurationService configService = new BurnTestConfigurationService(node, messageSink, randomSupplier, topology, lookup::get);
lookup.put(node, new Node(node, messageSink, configService,
- nowSupplier.get(), () -> new ListStore(node), ListAgent.INSTANCE, sinks, CommandStores.Synchronized::new));
+ nowSupplier.get(), () -> new ListStore(node), ListAgent.INSTANCE,
+ randomSupplier.get(), sinks, SimpleProgressLog::new, CommandStores.Synchronized::new));
}
List<Id> nodesList = new ArrayList<>(Arrays.asList(nodes));
@@ -170,6 +175,7 @@ public class Cluster implements Scheduler
sinks.add(next);
while (sinks.processPending());
+ System.out.println();
}
finally
{
diff --git a/accord-core/src/test/java/accord/impl/list/ListRead.java b/accord-core/src/test/java/accord/impl/list/ListRead.java
index 389c32e..a8af384 100644
--- a/accord-core/src/test/java/accord/impl/list/ListRead.java
+++ b/accord-core/src/test/java/accord/impl/list/ListRead.java
@@ -18,7 +18,13 @@ public class ListRead implements Read
}
@Override
- public Data read(Key key, Timestamp executeAt, Store store)
+ public Keys keys()
+ {
+ return keys;
+ }
+
+ @Override
+ public Data read(Key key, Timestamp executeAt, DataStore store)
{
ListStore s = (ListStore)store;
ListData result = new ListData();
diff --git a/accord-core/src/test/java/accord/impl/list/ListRequest.java b/accord-core/src/test/java/accord/impl/list/ListRequest.java
index 9a4a823..731c967 100644
--- a/accord-core/src/test/java/accord/impl/list/ListRequest.java
+++ b/accord-core/src/test/java/accord/impl/list/ListRequest.java
@@ -18,7 +18,7 @@ public class ListRequest implements Request
public void process(Node node, Id client, ReplyContext replyContext)
{
- // TODO (now): error handling
+ // TODO: error handling
node.coordinate(txn).addCallback((success, fail) -> {
if (success != null)
node.reply(client, replyContext, (ListResult) success);
diff --git a/accord-core/src/test/java/accord/impl/list/ListStore.java b/accord-core/src/test/java/accord/impl/list/ListStore.java
index b619113..f2daca8 100644
--- a/accord-core/src/test/java/accord/impl/list/ListStore.java
+++ b/accord-core/src/test/java/accord/impl/list/ListStore.java
@@ -4,11 +4,11 @@ import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import accord.api.Key;
-import accord.api.Store;
import accord.local.Node;
+import accord.api.DataStore;
import accord.utils.Timestamped;
-public class ListStore implements Store
+public class ListStore implements DataStore
{
static final int[] EMPTY = new int[0];
final Map<Key, Timestamped<int[]>> data = new ConcurrentHashMap<>();
diff --git a/accord-core/src/test/java/accord/impl/list/ListUpdate.java b/accord-core/src/test/java/accord/impl/list/ListUpdate.java
index b5db73e..4d89ec5 100644
--- a/accord-core/src/test/java/accord/impl/list/ListUpdate.java
+++ b/accord-core/src/test/java/accord/impl/list/ListUpdate.java
@@ -3,6 +3,7 @@ package accord.impl.list;
import java.util.Arrays;
import java.util.Map;
import java.util.TreeMap;
+import java.util.stream.Collectors;
import accord.api.Key;
import accord.api.Data;
@@ -26,4 +27,12 @@ public class ListUpdate extends TreeMap<Key, Integer> implements Update
to[to.length - 1] = append;
return to;
}
+
+ @Override
+ public String toString()
+ {
+ return entrySet().stream()
+ .map(e -> e.getKey() + ":" + e.getValue())
+ .collect(Collectors.joining(", ", "{", "}"));
+ }
}
diff --git a/accord-core/src/test/java/accord/impl/list/ListWrite.java b/accord-core/src/test/java/accord/impl/list/ListWrite.java
index 4be25f3..68b321b 100644
--- a/accord-core/src/test/java/accord/impl/list/ListWrite.java
+++ b/accord-core/src/test/java/accord/impl/list/ListWrite.java
@@ -5,7 +5,7 @@ import java.util.TreeMap;
import java.util.stream.Collectors;
import accord.api.Key;
-import accord.api.Store;
+import accord.api.DataStore;
import accord.api.Write;
import accord.txn.Timestamp;
import accord.utils.Timestamped;
@@ -16,7 +16,7 @@ public class ListWrite extends TreeMap<Key, int[]> implements Write
{
private static final Logger logger = LoggerFactory.getLogger(ListWrite.class);
@Override
- public void apply(Key key, Timestamp executeAt, Store store)
+ public void apply(Key key, Timestamp executeAt, DataStore store)
{
ListStore s = (ListStore) store;
if (!containsKey(key))
@@ -30,7 +30,7 @@ public class ListWrite extends TreeMap<Key, int[]> implements Write
public String toString()
{
return entrySet().stream()
- .map(e -> e.getKey() + "=" + Arrays.toString(e.getValue()))
+ .map(e -> e.getKey() + ":" + Arrays.toString(e.getValue()))
.collect(Collectors.joining(", ", "{", "}"));
}
}
diff --git a/accord-core/src/test/java/accord/impl/mock/EpochSync.java b/accord-core/src/test/java/accord/impl/mock/EpochSync.java
index 80ad2d2..9d105be 100644
--- a/accord-core/src/test/java/accord/impl/mock/EpochSync.java
+++ b/accord-core/src/test/java/accord/impl/mock/EpochSync.java
@@ -1,9 +1,10 @@
package accord.impl.mock;
+import accord.api.Key;
import accord.coordinate.tracking.QuorumTracker;
import accord.local.*;
import accord.messages.*;
-import accord.topology.Topologies;
+import accord.topology.Topologies.Single;
import accord.topology.Topology;
import accord.txn.*;
import com.google.common.base.Preconditions;
@@ -33,18 +34,22 @@ public class EpochSync implements Runnable
this.nextEpoch = syncEpoch + 1;
}
- private static class SyncMessage implements Request
+ private static class SyncCommitted implements Request
{
private final TxnId txnId;
private final Txn txn;
+ private final Key homeKey;
private final Timestamp executeAt;
private final Dependencies deps;
+ private final long epoch;
- public SyncMessage(Command command)
+ public SyncCommitted(Command command, long epoch)
{
+ this.epoch = epoch;
Preconditions.checkArgument(command.hasBeen(Status.Committed));
this.txnId = command.txnId();
this.txn = command.txn();
+ this.homeKey = command.homeKey();
this.executeAt = command.executeAt();
this.deps = command.savedDeps();
}
@@ -52,9 +57,10 @@ public class EpochSync implements Runnable
@Override
public void process(Node node, Node.Id from, ReplyContext replyContext)
{
- node.forEachLocal(commandStore -> {
+ Key progressKey = node.trySelectProgressKey(txnId, txn.keys, homeKey);
+ node.forEachLocalSince(txn.keys, epoch, commandStore -> {
Command command = commandStore.command(txnId);
- command.commit(txn, deps, executeAt);
+ command.commit(txn, homeKey, progressKey, executeAt, deps);
});
node.reply(from, replyContext, SyncAck.INSTANCE);
}
@@ -82,30 +88,30 @@ public class EpochSync implements Runnable
{
private final QuorumTracker tracker;
- public CommandSync(Node node, SyncMessage message, Topology topology)
+ public CommandSync(Node node, SyncCommitted message, Topology topology)
{
Keys keys = message.txn.keys();
- this.tracker = new QuorumTracker(new Topologies.Singleton(topology.forKeys(keys), false));
+ this.tracker = new QuorumTracker(new Single(topology.forKeys(keys), false));
node.send(tracker.nodes(), message, this);
}
@Override
public synchronized void onSuccess(Node.Id from, SyncAck response)
{
- tracker.recordSuccess(from);
+ tracker.success(from);
if (tracker.hasReachedQuorum())
- setSuccess(null);
+ trySuccess(null);
}
@Override
public synchronized void onFailure(Node.Id from, Throwable throwable)
{
- tracker.recordFailure(from);
+ tracker.failure(from);
if (tracker.hasFailed())
tryFailure(throwable);
}
- public static void sync(Node node, SyncMessage message, Topology topology)
+ public static void sync(Node node, SyncCommitted message, Topology topology)
{
try
{
@@ -156,11 +162,11 @@ public class EpochSync implements Runnable
@Override
public void run()
{
- Map<TxnId, SyncMessage> syncMessages = new ConcurrentHashMap<>();
- Consumer<Command> commandConsumer = command -> syncMessages.put(command.txnId(), new SyncMessage(command));
+ Map<TxnId, SyncCommitted> syncMessages = new ConcurrentHashMap<>();
+ Consumer<Command> commandConsumer = command -> syncMessages.put(command.txnId(), new SyncCommitted(command, syncEpoch));
node.forEachLocal(commandStore -> commandStore.forCommittedInEpoch(syncTopology.ranges(), syncEpoch, commandConsumer));
- for (SyncMessage message : syncMessages.values())
+ for (SyncCommitted message : syncMessages.values())
CommandSync.sync(node, message, nextTopology);
SyncComplete syncComplete = new SyncComplete(syncEpoch);
diff --git a/accord-core/src/test/java/accord/impl/mock/MockCluster.java b/accord-core/src/test/java/accord/impl/mock/MockCluster.java
index 0a70760..d75d49a 100644
--- a/accord-core/src/test/java/accord/impl/mock/MockCluster.java
+++ b/accord-core/src/test/java/accord/impl/mock/MockCluster.java
@@ -5,6 +5,8 @@ import accord.api.MessageSink;
import accord.coordinate.Timeout;
import accord.impl.TopologyUtils;
import accord.local.CommandStores;
+import accord.impl.SimpleProgressLog;
+import accord.local.CommandStore;
import accord.local.Node;
import accord.local.Node.Id;
import accord.topology.KeyRanges;
@@ -88,7 +90,9 @@ public class MockCluster implements Network, AutoCloseable, Iterable<Node>
nowSupplier,
() -> store,
new TestAgent(),
+ new Random(),
new ThreadPoolScheduler(),
+ SimpleProgressLog::new,
CommandStores.SingleThread::new);
}
diff --git a/accord-core/src/test/java/accord/impl/mock/MockStore.java b/accord-core/src/test/java/accord/impl/mock/MockStore.java
index 7899a78..c72a85c 100644
--- a/accord-core/src/test/java/accord/impl/mock/MockStore.java
+++ b/accord-core/src/test/java/accord/impl/mock/MockStore.java
@@ -1,14 +1,17 @@
package accord.impl.mock;
import accord.api.Data;
+import accord.api.Key;
import accord.api.Query;
import accord.api.Read;
import accord.api.Result;
-import accord.api.Store;
+import accord.api.DataStore;
import accord.api.Update;
import accord.api.Write;
+import accord.txn.Keys;
+import accord.txn.Timestamp;
-public class MockStore implements Store
+public class MockStore implements DataStore
{
public static final Data DATA = new Data() {
@Override
@@ -19,8 +22,25 @@ public class MockStore implements Store
};
public static final Result RESULT = new Result() {};
- public static final Read READ = (key, executeAt, store) -> DATA;
public static final Query QUERY = data -> RESULT;
public static final Write WRITE = (key, executeAt, store) -> {};
public static final Update UPDATE = data -> WRITE;
+
+ public static Read read(Keys keys)
+ {
+ return new Read()
+ {
+ @Override
+ public Keys keys()
+ {
+ return keys;
+ }
+
+ @Override
+ public Data read(Key key, Timestamp executeAt, DataStore store)
+ {
+ return DATA;
+ }
+ };
+ }
}
diff --git a/accord-core/src/test/java/accord/impl/mock/RecordingMessageSink.java b/accord-core/src/test/java/accord/impl/mock/RecordingMessageSink.java
index 512c2b2..8e4b458 100644
--- a/accord-core/src/test/java/accord/impl/mock/RecordingMessageSink.java
+++ b/accord-core/src/test/java/accord/impl/mock/RecordingMessageSink.java
@@ -9,6 +9,7 @@ import accord.messages.Request;
import org.junit.jupiter.api.Assertions;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
public class RecordingMessageSink extends SimpleMessageSink
@@ -27,8 +28,8 @@ public class RecordingMessageSink extends SimpleMessageSink
}
}
- public final List<Envelope<Request>> requests = new ArrayList<>();
- public final List<Envelope<Reply>> responses = new ArrayList<>();
+ public final List<Envelope<Request>> requests = Collections.synchronizedList(new ArrayList<>());
+ public final List<Envelope<Reply>> responses = Collections.synchronizedList(new ArrayList<>());
public RecordingMessageSink(Node.Id node, Network network)
{
diff --git a/accord-core/src/test/java/accord/local/CommandStoreTest.java b/accord-core/src/test/java/accord/local/CommandStoreTest.java
new file mode 100644
index 0000000..269aabf
--- /dev/null
+++ b/accord-core/src/test/java/accord/local/CommandStoreTest.java
@@ -0,0 +1,92 @@
+//package accord.local;
+//
+//import accord.Utils;
+//import accord.topology.KeyRange;
+//import accord.impl.IntKey;
+//import accord.impl.TopologyUtils;
+//import accord.topology.KeyRanges;
+//import accord.topology.Shard;
+//import accord.topology.Topology;
+//import org.junit.jupiter.api.Assertions;
+//import org.junit.jupiter.api.Test;
+//
+//import java.util.List;
+//
+//import static accord.impl.IntKey.key;
+//
+//public class CommandStoreTest
+//{
+// @Test
+// void topologyChangeTest()
+// {
+// List<Node.Id> ids = Utils.ids(5);
+// KeyRanges ranges = TopologyUtils.initialRanges(5, 500);
+// Topology topology = TopologyUtils.initialTopology(ids, ranges, 3);
+// Topology local = topology.forNode(ids.get(0));
+//
+// KeyRanges shards = CommandStores.shardRanges(local.ranges(), 10).get(0);
+// Assertions.assertEquals(ranges(r(0, 10), r(300, 310), r(400, 410)), shards);
+//
+// CommandStore commandStore = new CommandStore.Synchronized(0, null, null, null, null, ignore -> null, null);
+// commandStore.updateTopology(topology, shards.union(r(350, 360)), KeyRanges.EMPTY);
+// commandStore.commandsForKey(key(355));
+// commandStore.commandsForKey(key(356));
+// commandStore.commandsForKey(key(357));
+//
+// Assertions.assertTrue(commandStore.hasCommandsForKey(key(355)));
+// Assertions.assertTrue(commandStore.hasCommandsForKey(key(356)));
+// Assertions.assertTrue(commandStore.hasCommandsForKey(key(357)));
+//
+// commandStore.updateTopology(topology, KeyRanges.EMPTY, ranges(r(355, 360)));
+//
+// Assertions.assertTrue(commandStore.hasCommandsForKey(key(355)));
+// Assertions.assertFalse(commandStore.hasCommandsForKey(key(356)));
+// Assertions.assertFalse(commandStore.hasCommandsForKey(key(357)));
+// }
+//
+// private static Shard[] shards(Topology topology, int... indexes)
+// {
+// Shard[] shards = new Shard[indexes.length];
+//
+// for (int i=0; i<indexes.length; i++)
+// shards[i] = topology.get(indexes[i]);
+// return shards;
+// }
+//
+// private static void assertMapping(KeyRanges ranges, Shard[] shards, CommandStore.RangeMapping mapping)
+// {
+// Assertions.assertEquals(ranges, mapping.ranges);
+// Assertions.assertArrayEquals(shards, mapping.shards);
+// }
+//
+// private static KeyRanges ranges(KeyRange... ranges)
+// {
+// return new KeyRanges(ranges);
+// }
+//
+// private static KeyRange<IntKey> r(int start, int end)
+// {
+// return IntKey.range(start, end);
+// }
+//
+// @Test
+// void mapRangesTest()
+// {
+// List<Node.Id> ids = Utils.ids(5);
+// KeyRanges ranges = TopologyUtils.initialRanges(5, 500);
+// Topology topology = TopologyUtils.initialTopology(ids, ranges, 3);
+// Topology local = topology.forNode(ids.get(0));
+//
+// KeyRanges shards = CommandStores.shardRanges(local.ranges(), 10).get(0);
+// Assertions.assertEquals(ranges(r(0, 10), r(300, 310), r(400, 410)), shards);
+//
+// assertMapping(shards, shards(local, 0, 1, 2),
+// CommandStore.mapRanges(shards, local));
+// assertMapping(ranges(r(0, 10), r(300, 310), r(390, 400), r(400, 410)), shards(local, 0, 1, 1, 2),
+// CommandStore.mapRanges(ranges(r(0, 10), r(300, 310), r(390, 410)), local));
+//
+// assertMapping(ranges(r(0, 10), r(300, 310), r(350, 360), r(400, 410)), shards(local, 0, 1, 1, 2),
+// CommandStore.mapRanges(ranges(r(0, 10), r(300, 310), r(350, 360), r(400, 410)), local));
+//
+// }
+//}
diff --git a/accord-core/src/test/java/accord/local/CommandTest.java b/accord-core/src/test/java/accord/local/CommandTest.java
index 7699d3d..ca375ed 100644
--- a/accord-core/src/test/java/accord/local/CommandTest.java
+++ b/accord-core/src/test/java/accord/local/CommandTest.java
@@ -1,10 +1,18 @@
package accord.local;
+import accord.api.Key;
+import accord.api.ProgressLog;
+import accord.api.TestableConfigurationService;
import accord.impl.IntKey;
import accord.impl.TestAgent;
import accord.impl.TopologyFactory;
import accord.impl.mock.MockCluster;
+import accord.impl.mock.MockConfigurationService;
import accord.impl.mock.MockStore;
+import accord.local.CommandStore.RangesForEpoch;
+import accord.local.CommandStores.Synchronized;
+import accord.local.Node.Id;
+import accord.topology.KeyRanges;
import accord.topology.Topology;
import accord.txn.Keys;
import accord.txn.Timestamp;
@@ -14,9 +22,10 @@ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import java.util.List;
+import java.util.Random;
+import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
-import java.util.function.Function;
import static accord.Utils.id;
import static accord.Utils.writeTxn;
@@ -34,12 +43,6 @@ public class CommandTest
{
final AtomicReference<Topology> local = new AtomicReference<>(TOPOLOGY);
final MockStore data = new MockStore();
- final AtomicReference<Timestamp> nextTimestamp = new AtomicReference<>(Timestamp.NONE);
- final Function<Timestamp, Timestamp> uniqueNow = atleast -> {
- Timestamp next = nextTimestamp.get();
- Assertions.assertTrue(next.compareTo(atleast) >= 0);
- return next;
- };
}
private static void setTopologyEpoch(AtomicReference<Topology> topology, long epoch)
@@ -49,15 +52,52 @@ public class CommandTest
private static CommandStore createStore(CommandStoreSupport storeSupport)
{
- return new CommandStore.Synchronized(0,
- 0,
- 1,
- ID1,
- storeSupport.uniqueNow,
- new TestAgent(),
- storeSupport.data,
- storeSupport.local.get().ranges(),
- storeSupport.local::get);
+ return createNode(ID1, storeSupport).unsafeByIndex(0);
+ }
+
+ private static class NoOpProgressLog implements ProgressLog
+ {
+ @Override
+ public void preaccept(TxnId txnId, boolean isProgressShard, boolean isHomeShard)
+ {
+ }
+
+ @Override
+ public void accept(TxnId txnId, boolean isProgressShard, boolean isHomeShard)
+ {
+ }
+
+ @Override
+ public void commit(TxnId txnId, boolean isProgressShard, boolean isHomeShard)
+ {
+ }
+
+ @Override
+ public void readyToExecute(TxnId txnId, boolean isProgressShard, boolean isHomeShard)
+ {
+ }
+
+ @Override
+ public void execute(TxnId txnId, boolean isProgressShard, boolean isHomeShard)
+ {
+ }
+
+ @Override
+ public void executedOnAllShards(TxnId txnId, Set<Id> persistedOn)
+ {
+ }
+
+ @Override
+ public void waiting(TxnId blockedBy, Key homeKey)
+ {
+ }
+ }
+
+ private static Node createNode(Id id, CommandStoreSupport storeSupport)
+ {
+ return new Node(id, null, new MockConfigurationService(null, (epoch, service) -> { }, storeSupport.local.get()),
+ new MockCluster.Clock(100), () -> storeSupport.data, new TestAgent(), new Random(), null,
+ ignore -> ignore2 -> new NoOpProgressLog(), Synchronized::new);
}
@Test
@@ -73,7 +113,7 @@ public class CommandTest
Assertions.assertEquals(Status.NotWitnessed, command.status());
Assertions.assertNull(command.executeAt());
- command.witness(txn);
+ command.preaccept(txn, KEY, KEY);
Assertions.assertEquals(Status.PreAccepted, command.status());
Assertions.assertEquals(txnId, command.executeAt());
}
@@ -83,19 +123,18 @@ public class CommandTest
{
CommandStoreSupport support = new CommandStoreSupport();
CommandStore commands = createStore(support);
- MockCluster.Clock clock = new MockCluster.Clock(100);
- TxnId txnId = clock.idForNode(1, 1);
+ TxnId txnId = commands.node().nextTxnId();
+ ((MockCluster.Clock)commands.node().unsafeGetNowSupplier()).increment(10);
Txn txn = writeTxn(Keys.of(KEY));
-
Command command = new Command(commands, txnId);
Assertions.assertEquals(Status.NotWitnessed, command.status());
Assertions.assertNull(command.executeAt());
setTopologyEpoch(support.local, 2);
+ ((TestableConfigurationService)commands.node().configService()).reportTopology(support.local.get().withEpoch(2));
Timestamp expectedTimestamp = new Timestamp(2, 110, 0, ID1);
- support.nextTimestamp.set(expectedTimestamp);
- commands.process((Consumer<? super CommandStore>) cstore -> command.witness(txn));
+ commands.process((Consumer<? super CommandStore>) cstore -> command.preaccept(txn, KEY, KEY));
Assertions.assertEquals(Status.PreAccepted, command.status());
Assertions.assertEquals(expectedTimestamp, command.executeAt());
}
diff --git a/accord-core/src/test/java/accord/messages/PreAcceptTest.java b/accord-core/src/test/java/accord/messages/PreAcceptTest.java
index e3b384f..75b7e47 100644
--- a/accord-core/src/test/java/accord/messages/PreAcceptTest.java
+++ b/accord-core/src/test/java/accord/messages/PreAcceptTest.java
@@ -1,6 +1,8 @@
package accord.messages;
+import accord.api.Key;
import accord.impl.mock.*;
+import accord.impl.SimpleProgressLog;
import accord.local.Node;
import accord.local.Node.Id;
import accord.api.MessageSink;
@@ -22,6 +24,7 @@ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import java.util.List;
+import java.util.Random;
import static accord.Utils.id;
import static accord.Utils.writeTxn;
@@ -47,18 +50,15 @@ public class PreAcceptTest
clock,
() -> store,
new TestAgent(),
+ new Random(),
scheduler,
+ SimpleProgressLog::new,
CommandStores.SingleThread::new);
}
- private static TxnRequest.Scope scope(TxnId txnId, Txn txn)
+ private static PreAccept preAccept(TxnId txnId, Txn txn, Key homeKey)
{
- return new TxnRequest.Scope(txnId.epoch, txn.keys());
- }
-
- private static PreAccept preAccept(TxnId txnId, Txn txn)
- {
- return new PreAccept(scope(txnId, txn), txnId, txn);
+ return new PreAccept(txn.keys, txnId.epoch, txnId, txn, homeKey);
}
@Test
@@ -77,7 +77,7 @@ public class PreAcceptTest
TxnId txnId = clock.idForNode(1, ID2);
Txn txn = writeTxn(Keys.of(key));
- PreAccept preAccept = preAccept(txnId, txn);
+ PreAccept preAccept = preAccept(txnId, txn, key);
clock.increment(10);
preAccept.process(node, ID2, REPLY_CONTEXT);
@@ -109,7 +109,7 @@ public class PreAcceptTest
TxnId txnId = clock.idForNode(1, ID2);
Txn txn = writeTxn(Keys.of(key));
- PreAccept preAccept = preAccept(txnId, txn);
+ PreAccept preAccept = preAccept(txnId, txn, key);
preAccept.process(node, ID2, REPLY_CONTEXT);
}
finally
@@ -133,13 +133,13 @@ public class PreAcceptTest
{
IntKey key1 = IntKey.key(10);
- PreAccept preAccept1 = preAccept(clock.idForNode(1, ID2), writeTxn(Keys.of(key1)));
+ PreAccept preAccept1 = preAccept(clock.idForNode(1, ID2), writeTxn(Keys.of(key1)), key1);
preAccept1.process(node, ID2, REPLY_CONTEXT);
messageSink.clearHistory();
IntKey key2 = IntKey.key(11);
TxnId txnId2 = new TxnId(1, 50, 0, ID3);
- PreAccept preAccept2 = preAccept(txnId2, writeTxn(Keys.of(key1, key2)));
+ PreAccept preAccept2 = preAccept(txnId2, writeTxn(Keys.of(key1, key2)), key2);
clock.increment(10);
preAccept2.process(node, ID3, REPLY_CONTEXT);
@@ -170,7 +170,7 @@ public class PreAcceptTest
try
{
TxnId txnId = new TxnId(1, 110, 0, ID2);
- PreAccept preAccept = preAccept(txnId, writeTxn(Keys.of(key)));
+ PreAccept preAccept = preAccept(txnId, writeTxn(Keys.of(key)), key);
preAccept.process(node, ID2, REPLY_CONTEXT);
messageSink.assertHistorySizes(0, 1);
@@ -202,7 +202,7 @@ public class PreAcceptTest
TxnId txnId = clock.idForNode(1, ID2);
Txn txn = writeTxn(Keys.of(key));
- PreAccept preAccept = preAccept(txnId, txn);
+ PreAccept preAccept = preAccept(txnId, txn, key);
clock.increment(10);
preAccept.process(node, ID2, REPLY_CONTEXT);
diff --git a/accord-core/src/test/java/accord/messages/TxnRequestScopeTest.java b/accord-core/src/test/java/accord/messages/TxnRequestScopeTest.java
index 3a0dca4..fc16858 100644
--- a/accord-core/src/test/java/accord/messages/TxnRequestScopeTest.java
+++ b/accord-core/src/test/java/accord/messages/TxnRequestScopeTest.java
@@ -1,7 +1,6 @@
package accord.messages;
-import accord.api.KeyRange;
-import accord.messages.TxnRequest.Scope;
+import accord.topology.KeyRange;
import accord.topology.Topologies;
import accord.topology.Topology;
import accord.txn.Keys;
@@ -15,11 +14,6 @@ import static accord.impl.IntKey.range;
public class TxnRequestScopeTest
{
- private static Scope scope(long epoch, Keys keys)
- {
- return new Scope(epoch, keys);
- }
-
@Test
void createDisjointScopeTest()
{
@@ -32,10 +26,10 @@ public class TxnRequestScopeTest
topologies.add(topology2);
topologies.add(topology1);
- Assertions.assertEquals(scope(1, keys(150)),
- Scope.forTopologies(id(1), topologies, keys));
- Assertions.assertEquals(scope(2, keys(150)),
- Scope.forTopologies(id(4), topologies, keys));
+ Assertions.assertEquals(keys(150), TxnRequest.computeScope(id(1), topologies, keys));
+ Assertions.assertEquals(1, TxnRequest.computeWaitForEpoch(id(1), topologies, keys));
+ Assertions.assertEquals(keys(150), TxnRequest.computeScope(id(4), topologies, keys));
+ Assertions.assertEquals(2, TxnRequest.computeWaitForEpoch(id(4), topologies, keys));
}
@Test
@@ -55,9 +49,10 @@ public class TxnRequestScopeTest
Topologies.Multi topologies = new Topologies.Multi();
topologies.add(topology2);
topologies.add(topology1);
- Assertions.assertEquals(scope(2, keys(150, 250)),
- Scope.forTopologies(id(1), topologies, keys));
- Assertions.assertEquals(scope(2, keys(250, 150)),
- Scope.forTopologies(id(4), topologies, keys));
+
+ Assertions.assertEquals(keys(150, 250), TxnRequest.computeScope(id(1), topologies, keys));
+ Assertions.assertEquals(2, TxnRequest.computeWaitForEpoch(id(1), topologies, keys));
+ Assertions.assertEquals(keys(150, 250), TxnRequest.computeScope(id(4), topologies, keys));
+ Assertions.assertEquals(2, TxnRequest.computeWaitForEpoch(id(4), topologies, keys));
}
}
diff --git a/accord-core/src/test/java/accord/topology/TopologyManagerTest.java b/accord-core/src/test/java/accord/topology/TopologyManagerTest.java
index 68d6338..de1b116 100644
--- a/accord-core/src/test/java/accord/topology/TopologyManagerTest.java
+++ b/accord-core/src/test/java/accord/topology/TopologyManagerTest.java
@@ -1,6 +1,5 @@
package accord.topology;
-import accord.api.KeyRange;
import accord.local.Node;
import accord.txn.Keys;
import org.junit.jupiter.api.Assertions;
@@ -139,12 +138,12 @@ public class TopologyManagerTest
Keys keys = keys(150);
Assertions.assertEquals(topologies(topology2.forKeys(keys), topology1.forKeys(keys)),
- service.forKeys(keys));
+ service.withUnsyncedEpochs(keys, 2, 2));
service.onEpochSyncComplete(id(1), 1);
service.onEpochSyncComplete(id(2), 1);
Assertions.assertEquals(topologies(topology2.forKeys(keys)),
- service.forKeys(keys));
+ service.withUnsyncedEpochs(keys, 2, 2));
}
/**
@@ -167,12 +166,12 @@ public class TopologyManagerTest
// no acks, so all epoch 1 shards should be included
Assertions.assertEquals(topologies(topology2, topology1),
- service.forKeys(keys(150, 250)));
+ service.withUnsyncedEpochs(keys(150, 250), 2, 2));
// first topology acked, so only the second shard should be included
service.onEpochSyncComplete(id(1), 1);
service.onEpochSyncComplete(id(2), 1);
- Topologies actual = service.forKeys(keys(150, 250));
+ Topologies actual = service.withUnsyncedEpochs(keys(150, 250), 2, 2);
Assertions.assertEquals(topologies(topology2, topology(1, shard(range(200, 300), idList(4, 5, 6), idSet(4, 5)))),
actual);
Assertions.assertFalse(actual.fastPathPermitted());
diff --git a/accord-core/src/test/java/accord/topology/TopologyRandomizer.java b/accord-core/src/test/java/accord/topology/TopologyRandomizer.java
index 06f088f..46390c9 100644
--- a/accord-core/src/test/java/accord/topology/TopologyRandomizer.java
+++ b/accord-core/src/test/java/accord/topology/TopologyRandomizer.java
@@ -1,6 +1,5 @@
package accord.topology;
-import accord.api.KeyRange;
import accord.burn.TopologyUpdate;
import accord.impl.IntHashKey;
import accord.local.Node;
@@ -218,7 +217,7 @@ public class TopologyRandomizer
{
KeyRanges previous = previouslyReplicated.getOrDefault(entry.getKey(), KeyRanges.EMPTY);
KeyRanges added = entry.getValue();
- KeyRanges merged = previous.merge(added).mergeTouching();
+ KeyRanges merged = previous.union(added).mergeTouching();
previouslyReplicated.put(entry.getKey(), merged);
}
diff --git a/accord-core/src/test/java/accord/topology/TopologyTest.java b/accord-core/src/test/java/accord/topology/TopologyTest.java
index 76ed574..643f73f 100644
--- a/accord-core/src/test/java/accord/topology/TopologyTest.java
+++ b/accord-core/src/test/java/accord/topology/TopologyTest.java
@@ -2,7 +2,6 @@ package accord.topology;
import accord.Utils;
import accord.api.Key;
-import accord.api.KeyRange;
import accord.impl.IntKey;
import accord.impl.TopologyFactory;
import accord.local.Node;
diff --git a/accord-core/src/test/java/accord/utils/KeyRangeTest.java b/accord-core/src/test/java/accord/utils/KeyRangeTest.java
index ff0fdc4..7d2c0f7 100644
--- a/accord-core/src/test/java/accord/utils/KeyRangeTest.java
+++ b/accord-core/src/test/java/accord/utils/KeyRangeTest.java
@@ -1,7 +1,7 @@
package accord.utils;
import accord.api.Key;
-import accord.api.KeyRange;
+import accord.topology.KeyRange;
import accord.impl.IntKey;
import accord.topology.KeyRanges;
import accord.txn.Keys;
diff --git a/accord-core/src/test/java/accord/utils/KeyRangesTest.java b/accord-core/src/test/java/accord/utils/KeyRangesTest.java
index 62a0b5f..52fc17b 100644
--- a/accord-core/src/test/java/accord/utils/KeyRangesTest.java
+++ b/accord-core/src/test/java/accord/utils/KeyRangesTest.java
@@ -1,6 +1,6 @@
package accord.utils;
-import accord.api.KeyRange;
+import accord.topology.KeyRange;
import accord.impl.IntKey;
import accord.topology.KeyRanges;
import org.junit.jupiter.api.Assertions;
@@ -61,15 +61,15 @@ public class KeyRangesTest
void addTest()
{
Assertions.assertEquals(ranges(r(0, 50), r(50, 100), r(100, 150), r(150, 200)),
- ranges(r(0, 50), r(100, 150)).union(ranges(r(50, 100), r(150, 200))));
+ ranges(r(0, 50), r(100, 150)).combine(ranges(r(50, 100), r(150, 200))));
- Assertions.assertThrows(IllegalArgumentException.class, () -> ranges(r(0, 50)).union(ranges(r(25, 75))));
+ Assertions.assertThrows(IllegalArgumentException.class, () -> ranges(r(0, 50)).combine(ranges(r(25, 75))));
}
private static void assertMergeResult(KeyRanges expected, KeyRanges input1, KeyRanges input2)
{
- Assertions.assertEquals(expected, input1.merge(input2));
- Assertions.assertEquals(expected, input2.merge(input1));
+ Assertions.assertEquals(expected, input1.union(input2));
+ Assertions.assertEquals(expected, input2.union(input1));
}
@Test
diff --git a/accord-core/src/test/java/accord/verify/StrictSerializabilityVerifier.java b/accord-core/src/test/java/accord/verify/StrictSerializabilityVerifier.java
index f29e3a4..cf648e0 100644
--- a/accord-core/src/test/java/accord/verify/StrictSerializabilityVerifier.java
+++ b/accord-core/src/test/java/accord/verify/StrictSerializabilityVerifier.java
@@ -547,12 +547,12 @@ public class StrictSerializabilityVerifier
for (int i = 0, max = Math.min(sequence.length, this.sequence.length) ; i < max ; ++i)
{
if (sequence[i] != this.sequence[i])
- throw new HistoryViolation(key, "Inconsistent sequences: " + Arrays.toString(this.sequence) + " vs " + Arrays.toString(sequence));
+ throw new HistoryViolation(key, "Inconsistent sequences on " + key + ": " + Arrays.toString(this.sequence) + " vs " + Arrays.toString(sequence));
}
if (this.sequence.length > sequence.length)
{
if (maybeWrite >= 0 && maybeWrite != this.sequence[sequence.length])
- throw new HistoryViolation(key, "Inconsistent sequences: " + Arrays.toString(this.sequence) + " vs " + Arrays.toString(sequence) + "+" + maybeWrite);
+ throw new HistoryViolation(key, "Inconsistent sequences on " + key + ": " + Arrays.toString(this.sequence) + " vs " + Arrays.toString(sequence) + "+" + maybeWrite);
}
else
{
diff --git a/accord-core/src/test/resources/burn-logback.xml b/accord-core/src/test/resources/burn-logback-trace.xml
similarity index 65%
copy from accord-core/src/test/resources/burn-logback.xml
copy to accord-core/src/test/resources/burn-logback-trace.xml
index 5e846ba..460c302 100644
--- a/accord-core/src/test/resources/burn-logback.xml
+++ b/accord-core/src/test/resources/burn-logback-trace.xml
@@ -6,6 +6,16 @@
<encoder>
<pattern>%-5level ${bySecond} %F:%L - %msg%n</pattern>
</encoder>
+ <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+ <level>INFO</level>
+ </filter>
+ </appender>
+
+ <appender name="TRACE" class="ch.qos.logback.core.FileAppender">
+ <file>./build/burn-test/logs/burn-${startTime}.trace</file>
+ <encoder>
+ <pattern>%msg%n</pattern>
+ </encoder>
</appender>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
@@ -21,4 +31,8 @@
<appender-ref ref="FILE"/>
<appender-ref ref="STDOUT"/>
</root>
+
+ <logger name="accord.impl.basic.Trace" level="TRACE">
+ <appender-ref ref="TRACE" />
+ </logger>
</configuration>
diff --git a/accord-core/src/test/resources/burn-logback.xml b/accord-core/src/test/resources/burn-logback.xml
index 5e846ba..dc6c225 100644
--- a/accord-core/src/test/resources/burn-logback.xml
+++ b/accord-core/src/test/resources/burn-logback.xml
@@ -6,6 +6,9 @@
<encoder>
<pattern>%-5level ${bySecond} %F:%L - %msg%n</pattern>
</encoder>
+ <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+ <level>INFO</level>
+ </filter>
</appender>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
diff --git a/accord-maelstrom/build.gradle b/accord-maelstrom/build.gradle
index 33d5e72..40ec833 100644
--- a/accord-maelstrom/build.gradle
+++ b/accord-maelstrom/build.gradle
@@ -14,7 +14,7 @@ compileJava {
}
dependencies {
- compile project(':accord-core')
+ implementation project(':accord-core')
implementation 'com.google.code.gson:gson:2.8.7'
implementation 'com.google.guava:guava:30.1.1-jre'
implementation 'ch.qos.logback:logback-classic:1.2.3'
diff --git a/accord-maelstrom/src/main/java/accord/maelstrom/Cluster.java b/accord-maelstrom/src/main/java/accord/maelstrom/Cluster.java
index eb0eeeb..bb02a81 100644
--- a/accord-maelstrom/src/main/java/accord/maelstrom/Cluster.java
+++ b/accord-maelstrom/src/main/java/accord/maelstrom/Cluster.java
@@ -15,6 +15,7 @@ import java.util.function.Supplier;
import accord.coordinate.Timeout;
import accord.local.CommandStores;
+import accord.impl.SimpleProgressLog;
import accord.local.Node;
import accord.local.Node.Id;
import accord.api.MessageSink;
@@ -264,7 +265,8 @@ public class Cluster implements Scheduler
{
MessageSink messageSink = sinks.create(node, randomSupplier.get());
lookup.put(node, new Node(node, messageSink, new SimpleConfigService(topology),
- nowSupplier.get(), MaelstromStore::new, MaelstromAgent.INSTANCE, sinks, CommandStores.SingleThread::new));
+ nowSupplier.get(), MaelstromStore::new, MaelstromAgent.INSTANCE,
+ randomSupplier.get(), sinks, SimpleProgressLog::new, CommandStores.SingleThread::new));
}
List<Id> nodesList = new ArrayList<>(Arrays.asList(nodes));
diff --git a/accord-maelstrom/src/main/java/accord/maelstrom/Json.java b/accord-maelstrom/src/main/java/accord/maelstrom/Json.java
index 88001ae..010d1d4 100644
--- a/accord-maelstrom/src/main/java/accord/maelstrom/Json.java
+++ b/accord-maelstrom/src/main/java/accord/maelstrom/Json.java
@@ -11,6 +11,7 @@ import java.util.TreeSet;
import accord.local.Node;
import accord.api.Result;
+import accord.txn.Dependencies.TxnAndHomeKey;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.TypeAdapter;
@@ -300,11 +301,14 @@ public class Json
public void write(JsonWriter out, Dependencies value) throws IOException
{
out.beginArray();
- for (Map.Entry<TxnId, Txn> e : value.deps.entrySet())
+ for (Map.Entry<TxnId, TxnAndHomeKey> e : value.deps.entrySet())
{
out.beginArray();
GSON.toJson(e.getKey(), TxnId.class, out);
- TXN_ADAPTER.write(out, e.getValue());
+ out.beginArray();
+ ((MaelstromKey)e.getValue().homeKey).write(out);
+ TXN_ADAPTER.write(out, e.getValue().txn);
+ out.endArray();
out.endArray();
}
out.endArray();
@@ -313,7 +317,7 @@ public class Json
@Override
public Dependencies read(JsonReader in) throws IOException
{
- NavigableMap<TxnId, Txn> deps = new TreeMap<>();
+ Dependencies deps = new Dependencies();
in.beginArray();
if (!in.hasNext())
{
@@ -325,12 +329,15 @@ public class Json
{
in.beginArray();
TxnId txnId = GSON.fromJson(in, TxnId.class);
+ in.beginArray();
+ Key homeKey = MaelstromKey.read(in);
Txn txn = TXN_ADAPTER.read(in);
- deps.put(txnId, txn);
+ in.endArray();
+ deps.add(txnId, txn, homeKey);
in.endArray();
}
in.endArray();
- return new Dependencies(deps);
+ return deps;
}
};
diff --git a/accord-maelstrom/src/main/java/accord/maelstrom/MaelstromKey.java b/accord-maelstrom/src/main/java/accord/maelstrom/MaelstromKey.java
index 1f54292..e98993d 100644
--- a/accord-maelstrom/src/main/java/accord/maelstrom/MaelstromKey.java
+++ b/accord-maelstrom/src/main/java/accord/maelstrom/MaelstromKey.java
@@ -3,7 +3,7 @@ package accord.maelstrom;
import java.io.IOException;
import accord.api.Key;
-import accord.api.KeyRange;
+import accord.topology.KeyRange;
import accord.topology.KeyRanges;
import com.google.common.base.Objects;
import com.google.common.base.Preconditions;
diff --git a/accord-maelstrom/src/main/java/accord/maelstrom/MaelstromRead.java b/accord-maelstrom/src/main/java/accord/maelstrom/MaelstromRead.java
index 6f96e70..5d03c55 100644
--- a/accord-maelstrom/src/main/java/accord/maelstrom/MaelstromRead.java
+++ b/accord-maelstrom/src/main/java/accord/maelstrom/MaelstromRead.java
@@ -14,7 +14,13 @@ public class MaelstromRead implements Read
}
@Override
- public Data read(Key key, Timestamp executeAt, Store store)
+ public Keys keys()
+ {
+ return keys;
+ }
+
+ @Override
+ public Data read(Key key, Timestamp executeAt, DataStore store)
{
MaelstromStore s = (MaelstromStore)store;
MaelstromData result = new MaelstromData();
diff --git a/accord-maelstrom/src/main/java/accord/maelstrom/MaelstromRequest.java b/accord-maelstrom/src/main/java/accord/maelstrom/MaelstromRequest.java
index 6d84c7c..c3046c4 100644
--- a/accord-maelstrom/src/main/java/accord/maelstrom/MaelstromRequest.java
+++ b/accord-maelstrom/src/main/java/accord/maelstrom/MaelstromRequest.java
@@ -29,7 +29,7 @@ public class MaelstromRequest extends Body implements Request
public void process(Node node, Id client, ReplyContext replyContext)
{
- // TODO (now): error handling
+ // TODO: error handling
node.coordinate(txn).addCallback((success, fail) -> {
if (success != null) node.reply(client, replyContext, new MaelstromReply(MaelstromReplyContext.messageIdFor(replyContext), (MaelstromResult) success));
// else node.reply(client, messageId, new Error(messageId, 13, fail.getMessage()));
diff --git a/accord-maelstrom/src/main/java/accord/maelstrom/MaelstromStore.java b/accord-maelstrom/src/main/java/accord/maelstrom/MaelstromStore.java
index 23051e5..ccd9b5c 100644
--- a/accord-maelstrom/src/main/java/accord/maelstrom/MaelstromStore.java
+++ b/accord-maelstrom/src/main/java/accord/maelstrom/MaelstromStore.java
@@ -4,10 +4,10 @@ import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import accord.api.Key;
-import accord.api.Store;
+import accord.api.DataStore;
import accord.utils.Timestamped;
-public class MaelstromStore implements Store
+public class MaelstromStore implements DataStore
{
final Map<Key, Timestamped<Value>> data = new ConcurrentHashMap<>();
diff --git a/accord-maelstrom/src/main/java/accord/maelstrom/MaelstromWrite.java b/accord-maelstrom/src/main/java/accord/maelstrom/MaelstromWrite.java
index 8316712..cf1feb8 100644
--- a/accord-maelstrom/src/main/java/accord/maelstrom/MaelstromWrite.java
+++ b/accord-maelstrom/src/main/java/accord/maelstrom/MaelstromWrite.java
@@ -1,7 +1,7 @@
package accord.maelstrom;
import accord.api.Key;
-import accord.api.Store;
+import accord.api.DataStore;
import accord.api.Write;
import accord.txn.Timestamp;
import accord.utils.Timestamped;
@@ -11,7 +11,7 @@ import java.util.TreeMap;
public class MaelstromWrite extends TreeMap<Key, Value> implements Write
{
@Override
- public void apply(Key key, Timestamp executeAt, Store store)
+ public void apply(Key key, Timestamp executeAt, DataStore store)
{
MaelstromStore s = (MaelstromStore) store;
if (containsKey(key))
diff --git a/accord-maelstrom/src/main/java/accord/maelstrom/Main.java b/accord-maelstrom/src/main/java/accord/maelstrom/Main.java
index 1c4e7f9..d0aecd2 100644
--- a/accord-maelstrom/src/main/java/accord/maelstrom/Main.java
+++ b/accord-maelstrom/src/main/java/accord/maelstrom/Main.java
@@ -14,6 +14,7 @@ import java.util.function.LongSupplier;
import java.util.function.Supplier;
import accord.coordinate.Timeout;
+import accord.impl.SimpleProgressLog;
import accord.local.CommandStore;
import accord.local.CommandStores;
import accord.local.Node;
@@ -141,7 +142,8 @@ public class Main
topology = topologyFactory.toTopology(init.cluster);
sink = new StdoutSink(System::currentTimeMillis, scheduler, start, init.self, out, err);
on = new Node(init.self, sink, new SimpleConfigService(topology), System::currentTimeMillis,
- MaelstromStore::new, MaelstromAgent.INSTANCE, scheduler, CommandStores.SingleThread::new);
+ MaelstromStore::new, MaelstromAgent.INSTANCE, new Random(), scheduler,
+ SimpleProgressLog::new, CommandStores.SingleThread::new);
err.println("Initialized node " + init.self);
err.flush();
sink.send(packet.src, new Body(Type.init_ok, Body.SENTINEL_MSG_ID, init.msg_id));
diff --git a/accord-maelstrom/src/main/java/accord/maelstrom/Packet.java b/accord-maelstrom/src/main/java/accord/maelstrom/Packet.java
index 4e4eaaf..4b08864 100644
--- a/accord-maelstrom/src/main/java/accord/maelstrom/Packet.java
+++ b/accord-maelstrom/src/main/java/accord/maelstrom/Packet.java
@@ -33,7 +33,6 @@ public class Packet implements ReplyContext
Apply(Apply.class, Json.DEFAULT_ADAPTER),
Read(ReadData.class, Json.DEFAULT_ADAPTER),
ReadOk(ReadData.ReadOk.class, Json.DEFAULT_ADAPTER),
- ReadWaiting(ReadData.ReadWaiting.class, Json.DEFAULT_ADAPTER),
ReadNack(ReadData.ReadNack.class, Json.DEFAULT_ADAPTER),
WaitOnCommit(accord.messages.WaitOnCommit.class, Json.DEFAULT_ADAPTER),
WaitOnCommitOk(accord.messages.WaitOnCommit.WaitOnCommitOk.class, Json.DEFAULT_ADAPTER),
diff --git a/accord-maelstrom/src/main/java/accord/maelstrom/TopologyFactory.java b/accord-maelstrom/src/main/java/accord/maelstrom/TopologyFactory.java
index 390032b..777d1fa 100644
--- a/accord-maelstrom/src/main/java/accord/maelstrom/TopologyFactory.java
+++ b/accord-maelstrom/src/main/java/accord/maelstrom/TopologyFactory.java
@@ -6,7 +6,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import accord.api.KeyRange;
+import accord.topology.KeyRange;
import accord.local.Node.Id;
import accord.maelstrom.Datum.Kind;
import accord.topology.Shard;
diff --git a/build.gradle b/build.gradle
index 54b9419..a4d7c69 100644
--- a/build.gradle
+++ b/build.gradle
@@ -1,5 +1,6 @@
plugins {
id 'java'
+ id 'java-test-fixtures'
}
group 'accord'
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org