You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by be...@apache.org on 2025/09/19 11:21:30 UTC
(cassandra-accord) 01/01: Rebootstrap Also Refine: - Introduce MapReduceConsumeCommandStore for enforcing message refusal comprehensively - Refactor to RedundantStatus: - use int rather than short to prepare to support more statuses - merge PreBootstrap and Unready (into Unready) Also Fix: - Propagate Erase to no longer participating replica - RecoverAwait to original coordinator was not correctly notifying - Cancel messages if we encounter an exception so we do not attempt to process other shards after [...]
This is an automated email from the ASF dual-hosted git repository.
benedict pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git
commit 657f344eb3b3570966bf8cff7731bef6eeea98f1
Author: Benedict Elliott Smith <be...@apache.org>
AuthorDate: Wed Sep 10 11:17:35 2025 +0100
Rebootstrap
Also Refine:
- Introduce MapReduceConsumeCommandStore for enforcing message refusal comprehensively
- Refactor to RedundantStatus:
- use int rather than short to prepare to support more statuses
- merge PreBootstrap and Unready (into Unready)
Also Fix:
- Propagate Erase to no longer participating replica
- RecoverAwait to original coordinator was not correctly notifying
- Cancel messages if we encounter an exception so we do not attempt to process other shards after clearing the message state
- Consistent participant filtering and Cleanup across ownership loss
- CheckStatus.foundKnown could erroneously override validFor ranges with weaker validForAll, leading to a shard not being treated as fully truncated and preventing recovery progress
patch by Benedict and Alex Petrov for CASSANDRA-20908
---
accord-core/src/main/java/accord/api/Agent.java | 6 +-
.../main/java/accord/api/ConfigurationService.java | 9 +-
.../src/main/java/accord/api/DataStore.java | 10 +-
accord-core/src/main/java/accord/api/Journal.java | 7 +-
.../java/accord/api/OwnershipEventListener.java | 39 +++
.../accord/coordinate/CoordinateTransaction.java | 15 +-
.../accord/coordinate/CoordinationAdapter.java | 2 +-
.../accord/coordinate/ExecuteEphemeralRead.java | 11 +-
.../main/java/accord/coordinate/ExecuteTxn.java | 35 ++-
.../main/java/accord/coordinate/FetchRoute.java | 20 +-
.../src/main/java/accord/coordinate/Infer.java | 24 +-
.../main/java/accord/coordinate/Invalidate.java | 43 ++-
.../main/java/accord/coordinate/KeyBarriers.java | 43 ++-
.../accord/impl/AbstractConfigurationService.java | 80 ++----
.../src/main/java/accord/impl/CommandChange.java | 27 +-
.../java/accord/impl/InMemoryCommandStore.java | 28 +-
.../accord/impl/progresslog/PackedKeyTracker.java | 5 +-
.../src/main/java/accord/local/Bootstrap.java | 56 ++--
.../src/main/java/accord/local/Cleanup.java | 40 ++-
.../src/main/java/accord/local/Command.java | 2 +-
.../src/main/java/accord/local/CommandStore.java | 284 ++++++++++++++-----
.../src/main/java/accord/local/CommandStores.java | 303 +++++++++------------
.../src/main/java/accord/local/Commands.java | 20 +-
.../LogUnavailableException.java} | 26 +-
.../java/accord/local/MapReduceCommandStores.java | 68 +++++
.../local/MapReduceConsumeCommandStores.java | 95 +++++++
accord-core/src/main/java/accord/local/Node.java | 68 ++---
.../main/java/accord/local/RedundantBefore.java | 269 +++++++++---------
.../main/java/accord/local/RedundantStatus.java | 201 ++++++--------
.../main/java/accord/local/SafeCommandStore.java | 30 +-
.../main/java/accord/local/cfk/CommandsForKey.java | 130 ++++-----
.../main/java/accord/local/cfk/PostProcess.java | 36 +--
.../src/main/java/accord/local/cfk/Pruning.java | 24 +-
.../src/main/java/accord/local/cfk/Serialize.java | 38 +--
.../src/main/java/accord/local/cfk/Updating.java | 42 +--
.../accord/local/durability/DurabilityService.java | 1 +
.../accord/local/durability/GlobalDurability.java | 3 +-
.../accord/local/durability/ShardDurability.java | 1 -
.../src/main/java/accord/messages/Accept.java | 28 +-
.../src/main/java/accord/messages/Apply.java | 11 +-
.../accord/messages/ApplyThenWaitUntilApplied.java | 6 +-
.../src/main/java/accord/messages/Await.java | 75 ++---
.../java/accord/messages/BeginInvalidation.java | 15 +-
.../main/java/accord/messages/BeginRecovery.java | 7 +-
.../src/main/java/accord/messages/CheckStatus.java | 49 ++--
.../src/main/java/accord/messages/Commit.java | 93 +++++--
.../java/accord/messages/GetEphemeralReadDeps.java | 6 +-
.../main/java/accord/messages/GetLatestDeps.java | 6 +-
.../main/java/accord/messages/GetMaxConflict.java | 6 +-
.../main/java/accord/messages/InformDurable.java | 6 +-
.../{AbstractRequest.java => NoWaitRequest.java} | 33 ++-
.../{TxnRequest.java => ParticipantsRequest.java} | 74 +----
.../src/main/java/accord/messages/PreAccept.java | 6 +-
.../src/main/java/accord/messages/Propagate.java | 43 ++-
.../src/main/java/accord/messages/ReadData.java | 42 +--
.../java/accord/messages/ReadEphemeralTxnData.java | 19 +-
.../main/java/accord/messages/RouteRequest.java | 78 ++++++
.../main/java/accord/messages/SetShardDurable.java | 18 +-
.../main/java/accord/messages/StableThenRead.java | 12 +-
.../main/java/accord/primitives/LatestDeps.java | 2 +-
.../src/main/java/accord/primitives/Ranges.java | 18 +-
.../main/java/accord/utils/DeterministicSet.java | 6 +
.../java/accord/utils/IntrusivePriorityHeap.java | 6 +
.../main/java/accord/utils/MapReduceConsume.java | 11 +-
.../main/java/accord/utils/async/AsyncResults.java | 17 +-
.../src/test/java/accord/burn/BurnTestBase.java | 20 +-
.../accord/burn/BurnTestConfigurationService.java | 13 +-
.../src/test/java/accord/burn/TopologyUpdates.java | 26 --
.../java/accord/coordinate/TopologyChangeTest.java | 9 +-
.../impl/AbstractConfigurationServiceTest.java | 6 +-
.../impl/AbstractTestConfigurationService.java | 62 +++++
.../src/test/java/accord/impl/MessageListener.java | 6 +-
.../src/test/java/accord/impl/TestAgent.java | 9 +-
.../src/test/java/accord/impl/basic/Cluster.java | 137 ++++++----
.../accord/impl/basic/DelayedCommandStores.java | 12 -
.../java/accord/impl/basic/InMemoryJournal.java | 24 +-
.../java/accord/impl/basic/LoggingJournal.java | 4 +-
.../java/accord/impl/basic/RandomDelayQueue.java | 102 +++++--
.../src/test/java/accord/impl/list/ListAgent.java | 33 ++-
.../src/test/java/accord/impl/list/ListData.java | 3 +-
.../src/test/java/accord/impl/list/ListStore.java | 2 +-
.../accord/impl/mock/MockConfigurationService.java | 4 +-
.../src/test/java/accord/impl/mock/MockStore.java | 2 +-
.../java/accord/local/RedundantBeforeTest.java | 17 +-
.../java/accord/local/cfk/CommandsForKeyTest.java | 11 +-
.../test/java/accord/messages/ReadDataTest.java | 4 +-
...stScopeTest.java => RouteRequestScopeTest.java} | 22 +-
.../java/accord/topology/TopologyManagerTest.java | 2 +-
.../java/accord/topology/TopologyRandomizer.java | 131 +++++++--
.../test/java/accord/utils/ExtendedAssertions.java | 4 +-
.../src/main/java/accord/maelstrom/Cluster.java | 2 +-
.../main/java/accord/maelstrom/MaelstromAgent.java | 9 +-
.../main/java/accord/maelstrom/MaelstromStore.java | 2 +-
.../java/accord/maelstrom/SimpleConfigService.java | 3 +-
94 files changed, 2115 insertions(+), 1400 deletions(-)
diff --git a/accord-core/src/main/java/accord/api/Agent.java b/accord-core/src/main/java/accord/api/Agent.java
index 99e72e30..9219e4a4 100644
--- a/accord-core/src/main/java/accord/api/Agent.java
+++ b/accord-core/src/main/java/accord/api/Agent.java
@@ -27,10 +27,8 @@ import accord.local.Node;
import accord.local.SafeCommandStore;
import accord.local.TimeService;
import accord.messages.ReplyContext;
-import accord.primitives.Ranges;
import accord.primitives.Routable.Domain;
import accord.primitives.Status.Phase;
-import accord.primitives.Timestamp;
import accord.primitives.Txn;
import accord.primitives.Txn.Kind;
import accord.primitives.TxnId;
@@ -44,9 +42,7 @@ public interface Agent extends UncaughtExceptionListener
{
default @Nullable Tracing trace(TxnId txnId, TraceEventType eventType) { return null; }
- void onFailedBootstrap(int attempt, String phase, Ranges ranges, Runnable retry, Throwable failure);
-
- void onStale(Timestamp staleSince, Ranges ranges);
+ OwnershipEventListener ownershipEvents();
default CoordinatorEventListener coordinatorEvents()
{
diff --git a/accord-core/src/main/java/accord/api/ConfigurationService.java b/accord-core/src/main/java/accord/api/ConfigurationService.java
index 636f3b8a..4c1035c6 100644
--- a/accord-core/src/main/java/accord/api/ConfigurationService.java
+++ b/accord-core/src/main/java/accord/api/ConfigurationService.java
@@ -101,6 +101,11 @@ public interface ConfigurationService
this.reads = reads;
}
+ public AsyncResult<Void> metadata() { return metadata; }
+ public AsyncResult<Void> coordinate() { return coordinate; }
+ public AsyncResult<Void> data() { return data; }
+ public AsyncResult<Void> reads() { return reads; }
+
public static EpochReady done(long epoch)
{
return new EpochReady(epoch, DONE, DONE, DONE, DONE);
@@ -187,8 +192,10 @@ public interface ConfigurationService
* Called after this node learns of an epoch as part of the {@code Listener#onTopologyUpdate} call.
* On invocation the system is not necessarily ready to process the epoch, and the BootstrapReady parameter
* provides indications of when the bootstrap has completed various phases of setup.
+ *
+ * returns false is epoch has already been truncated
*/
- void acknowledgeEpoch(EpochReady ready, boolean startSync);
+ boolean acknowledgeEpoch(EpochReady ready, boolean startSync);
void reportEpochClosed(Ranges ranges, long epoch);
diff --git a/accord-core/src/main/java/accord/api/DataStore.java b/accord-core/src/main/java/accord/api/DataStore.java
index 750daa44..e1ebdaac 100644
--- a/accord-core/src/main/java/accord/api/DataStore.java
+++ b/accord-core/src/main/java/accord/api/DataStore.java
@@ -38,6 +38,11 @@ import accord.utils.async.AsyncResult;
*/
public interface DataStore
{
+ enum FetchKind
+ {
+ Image, Sync
+ }
+
interface StartingRangeFetch
{
/**
@@ -117,8 +122,11 @@ public interface DataStore
/**
* Initiate a bulk fetch of the requested ranges from the queried node, ensuring that the snapshot reflects
* data at least as of the provided SyncPoint.
+ *
+ * If RequestKind#Sync is used, fetches only a minimal subset of data, assuming there has been some data locally
+ * (it is to up store's implementer to know how to achieve this)
*/
- FetchResult fetch(Node node, SafeCommandStore safeStore, Ranges ranges, SyncPoint syncPoint, FetchRanges callback);
+ FetchResult fetch(Node node, SafeCommandStore safeStore, Ranges ranges, SyncPoint syncPoint, FetchRanges callback, FetchKind kind);
/**
* Logical fsync-like operation: anything written to the store prior to the invocation of this method
diff --git a/accord-core/src/main/java/accord/api/Journal.java b/accord-core/src/main/java/accord/api/Journal.java
index b6492b31..5ed70dc2 100644
--- a/accord-core/src/main/java/accord/api/Journal.java
+++ b/accord-core/src/main/java/accord/api/Journal.java
@@ -67,7 +67,12 @@ public interface Journal
void saveTopology(TopologyUpdate topologyUpdate, Runnable onFlush);
void purge(CommandStores commandStores, EpochSupplier minEpoch);
- void replay(CommandStores commandStores);
+
+ /**
+ * Replays all messages from journal to rehydrate CommandStores state. Returns whether it has seen (and ignored)
+ * any exceptions during replay.
+ */
+ boolean replay(CommandStores commandStores);
RedundantBefore loadRedundantBefore(int store);
NavigableMap<TxnId, Ranges> loadBootstrapBeganAt(int store);
diff --git a/accord-core/src/main/java/accord/api/OwnershipEventListener.java b/accord-core/src/main/java/accord/api/OwnershipEventListener.java
new file mode 100644
index 00000000..d2dbe1c5
--- /dev/null
+++ b/accord-core/src/main/java/accord/api/OwnershipEventListener.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package accord.api;
+
+import accord.local.CommandStore;
+import accord.primitives.Ranges;
+import accord.primitives.Timestamp;
+
+import static accord.utils.Invariants.illegalState;
+
+public interface OwnershipEventListener
+{
+ default void onSuccessfulBootstrap(CommandStore commandStore, int attempt, long epoch, Ranges ranges) {}
+ void onFailedBootstrap(int attempt, String phase, Ranges ranges, Runnable retry, Throwable failure);
+ void onStale(Timestamp staleSince, Ranges ranges);
+
+ OwnershipEventListener FAIL = new OwnershipEventListener()
+ {
+ @Override public void onSuccessfulBootstrap(CommandStore commandStore, int attempt, long epoch, Ranges ranges) { throw illegalState(); }
+ @Override public void onFailedBootstrap(int attempt, String phase, Ranges ranges, Runnable retry, Throwable failure) { throw illegalState(); }
+ @Override public void onStale(Timestamp staleSince, Ranges ranges) { throw illegalState(); }
+ };
+}
diff --git a/accord-core/src/main/java/accord/coordinate/CoordinateTransaction.java b/accord-core/src/main/java/accord/coordinate/CoordinateTransaction.java
index ee0185dc..42aa3683 100644
--- a/accord-core/src/main/java/accord/coordinate/CoordinateTransaction.java
+++ b/accord-core/src/main/java/accord/coordinate/CoordinateTransaction.java
@@ -34,13 +34,14 @@ import accord.local.Commands;
import accord.local.DepsCalculator;
import accord.local.LoadKeys;
import accord.local.LoadKeysFor;
-import accord.local.PreLoadContext;
+import accord.local.MapReduceConsumeCommandStores;
import accord.local.SafeCommand;
import accord.local.SafeCommandStore;
import accord.local.SequentialAsyncExecutor;
import accord.local.StoreParticipants;
import accord.messages.PreAccept.PreAcceptNack;
import accord.messages.PreAccept.PreAcceptReply;
+import accord.primitives.Route;
import accord.primitives.Status;
import accord.primitives.Unseekables;
import accord.topology.Topologies;
@@ -52,7 +53,6 @@ import accord.primitives.FullRoute;
import accord.primitives.Timestamp;
import accord.primitives.Txn;
import accord.primitives.TxnId;
-import accord.utils.MapReduceConsume;
import accord.utils.SortedListMap;
import accord.utils.async.AsyncChain;
import accord.utils.async.AsyncChains;
@@ -203,16 +203,21 @@ public class CoordinateTransaction extends CoordinatePreAccept<Result>
}
enum LocalExecuteState { PENDING, SUCCESS, TIMEOUT}
- class LocalExecute implements PreLoadContext, MapReduceConsume<SafeCommandStore, PreAcceptReply>, Timeouts.Timeout
+ class LocalExecute extends MapReduceConsumeCommandStores<Route<?>, PreAcceptReply> implements Timeouts.Timeout
{
LocalExecuteState state = LocalExecuteState.PENDING;
Cancellable cancel;
RegisteredTimeout timeout;
+ protected LocalExecute()
+ {
+ super(route);
+ }
+
void start()
{
markSelfContacted();
- Cancellable cancel = node.mapReduceConsumeLocal(this, route, topologies.oldestEpoch(), topologies.currentEpoch(), this);
+ Cancellable cancel = node.commandStores().mapReduceConsume(topologies.oldestEpoch(), topologies.currentEpoch(), this);
long expiresAt = node.agent().selfExpiresAt(txnId, Status.Phase.PreAccept, MICROSECONDS);
RegisteredTimeout timeout = expiresAt <= 0 ? null : node.timeouts().registerAt(this, expiresAt, MICROSECONDS);
synchronized (this)
@@ -265,7 +270,7 @@ public class CoordinateTransaction extends CoordinatePreAccept<Result>
}
@Override
- public PreAcceptReply apply(SafeCommandStore safeStore)
+ public PreAcceptReply applyInternal(SafeCommandStore safeStore)
{
long minEpoch = topologies.oldestEpoch();
StoreParticipants participants = StoreParticipants.update(safeStore, route, minEpoch, txnId, txnId.epoch());
diff --git a/accord-core/src/main/java/accord/coordinate/CoordinationAdapter.java b/accord-core/src/main/java/accord/coordinate/CoordinationAdapter.java
index 4ceb90f0..6d0fc834 100644
--- a/accord-core/src/main/java/accord/coordinate/CoordinationAdapter.java
+++ b/accord-core/src/main/java/accord/coordinate/CoordinationAdapter.java
@@ -208,7 +208,7 @@ public interface CoordinationAdapter<R>
@Override
public void stabiliseOnly(Node node, SequentialAsyncExecutor executor, Route<?> require, Route<?> sendTo, SelectNodeOwnership selectNodeOwnership, FullRoute<?> route, Ballot ballot, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer<? super Deps, Throwable> callback)
{
- if (!node.topology().hasEpoch(executeAt.epoch()))
+ if (!node.topology().hasAtLeastEpoch(executeAt.epoch()))
{
node.withEpochAtLeast(executeAt.epoch(), executor, (success, fail) -> {
if (fail != null) callback.accept(null, fail);
diff --git a/accord-core/src/main/java/accord/coordinate/ExecuteEphemeralRead.java b/accord-core/src/main/java/accord/coordinate/ExecuteEphemeralRead.java
index 13eb541f..3e3bb369 100644
--- a/accord-core/src/main/java/accord/coordinate/ExecuteEphemeralRead.java
+++ b/accord-core/src/main/java/accord/coordinate/ExecuteEphemeralRead.java
@@ -182,11 +182,11 @@ public class ExecuteEphemeralRead extends ReadCoordinator<Result, ReadReply>
}
@Override
- public CommitOrReadNack apply(SafeCommandStore safeStore)
+ public CommitOrReadNack applyInternal(SafeCommandStore safeStore)
{
StoreParticipants participants = StoreParticipants.execute(safeStore, route, txnId, minEpoch(), executeAtEpoch);
SafeCommand safeCommand = safeStore.get(txnId, participants);
- return apply(safeStore, safeCommand, participants);
+ return applyInternal(safeStore, safeCommand, participants);
}
@Override
@@ -215,13 +215,14 @@ public class ExecuteEphemeralRead extends ReadCoordinator<Result, ReadReply>
}
@Override
- public void timeout()
+ protected boolean timeoutInternal()
{
- if (!super.cancel())
- return;
+ if (!super.timeoutInternal())
+ return false;
// TODO (desired): if we fail to commit locally we can submit a slow/medium path request
callback.failure(node.id(), new Timeout(txnId, route.homeKey(), "Could not promptly read from local coordinator"));
+ return true;
}
@Override
diff --git a/accord-core/src/main/java/accord/coordinate/ExecuteTxn.java b/accord-core/src/main/java/accord/coordinate/ExecuteTxn.java
index e7ef3382..43a8679a 100644
--- a/accord-core/src/main/java/accord/coordinate/ExecuteTxn.java
+++ b/accord-core/src/main/java/accord/coordinate/ExecuteTxn.java
@@ -23,9 +23,6 @@ import java.util.function.BiConsumer;
import javax.annotation.Nullable;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import accord.api.Data;
import accord.api.Result;
import accord.api.Timeouts;
@@ -35,6 +32,7 @@ import accord.coordinate.tracking.QuorumIdTracker;
import accord.coordinate.tracking.RequestStatus;
import accord.local.Commands;
import accord.local.Commands.CommitOutcome;
+import accord.local.LogUnavailableException;
import accord.local.Node;
import accord.local.Node.Id;
import accord.local.SafeCommand;
@@ -96,8 +94,6 @@ import static java.util.concurrent.TimeUnit.MICROSECONDS;
// TODO (expected): by default, if we can execute locally, never contact a remote replica regardless of local outcome
public class ExecuteTxn extends ReadCoordinator<Result, ReadReply>
{
- private static final Logger logger = LoggerFactory.getLogger(ExecuteTxn.class);
-
class StableTracker extends QuorumIdTracker implements Callback<ReadReply>
{
private boolean isDone;
@@ -190,7 +186,7 @@ public class ExecuteTxn extends ReadCoordinator<Result, ReadReply>
Node.Id self = node.id();
if (permitLocalExecution() && tryIfUniversal(self))
{
- isPrivilegedVoteCommitting = true;
+ isPrivilegedVoteCommitting = txnId.hasPrivilegedCoordinator() && path == FAST;
ExecuteFlags executeFlags = flags.get(self);
/*
This decides if our local execution may fast execute with the provided flags.
@@ -203,8 +199,8 @@ public class ExecuteTxn extends ReadCoordinator<Result, ReadReply>
ensures PREAPPLIED is recorded at the local coordinator before any other replicas are sent Apply, but
for now we simply disable this optimisation in this case.
*/
- boolean mayFastExecute = executeFlags.contains(READY_TO_EXECUTE)
- && (!txnId.hasPrivilegedCoordinator() || path != FAST)
+ boolean mayFastExecute = !isPrivilegedVoteCommitting
+ && executeFlags.contains(READY_TO_EXECUTE)
&& fastReadsMayBypassSafeStore(txnId);
if (mayFastExecute)
@@ -470,20 +466,28 @@ public class ExecuteTxn extends ReadCoordinator<Result, ReadReply>
}
@Override
- public CommitOrReadNack apply(SafeCommandStore safeStore)
+ public CommitOrReadNack applyInternal(SafeCommandStore safeStore)
{
StoreParticipants participants = StoreParticipants.execute(safeStore, route, txnId, minEpoch(), executeAtEpoch);
SafeCommand safeCommand = safeStore.get(txnId, participants);
- return apply(safeStore, safeCommand, participants);
+ return applyInternal(safeStore, safeCommand, participants);
}
@Override
- protected CommitOrReadNack apply(SafeCommandStore safeStore, SafeCommand safeCommand, StoreParticipants participants)
+ protected CommitOrReadNack applyInternal(SafeCommandStore safeStore, SafeCommand safeCommand, StoreParticipants participants)
{
if (CommitOutcome.Rejected == Commands.commit(safeStore, safeCommand, participants, Stable, Ballot.ZERO, txnId, route, txn, executeAt, stableDeps, commitKind()))
return CommitOrReadNack.Rejected;
- return super.apply(safeStore, safeCommand, participants);
+ return super.applyInternal(safeStore, safeCommand, participants);
+ }
+
+ @Override
+ protected CommitOrReadNack refuseInternal(SafeCommandStore safeStore)
+ {
+ if (isPrivilegedVoteCommitting)
+ throw new LogUnavailableException();
+ return super.refuseInternal(safeStore);
}
@Override
@@ -516,14 +520,15 @@ public class ExecuteTxn extends ReadCoordinator<Result, ReadReply>
}
@Override
- public void timeout()
+ protected boolean timeoutInternal()
{
- if (!super.cancel())
- return;
+ if (!super.timeoutInternal())
+ return false;
slowTimeout = null;
if (committed) callback.failure(node.id(), new Timeout(txnId, route.homeKey(), "Could not promptly read from local coordinator"));
else callback.failure(node.id(), new Timeout(txnId, route.homeKey(), "Could not promptly commit to local coordinator"));
+ return true;
}
@Override
diff --git a/accord-core/src/main/java/accord/coordinate/FetchRoute.java b/accord-core/src/main/java/accord/coordinate/FetchRoute.java
index 258303d6..f8e00e7a 100644
--- a/accord-core/src/main/java/accord/coordinate/FetchRoute.java
+++ b/accord-core/src/main/java/accord/coordinate/FetchRoute.java
@@ -24,18 +24,18 @@ import javax.annotation.Nullable;
import accord.api.Tracing;
import accord.local.Commands;
+import accord.local.MapReduceConsumeCommandStores;
import accord.local.Node;
-import accord.local.PreLoadContext;
import accord.local.SafeCommand;
import accord.local.SafeCommandStore;
import accord.primitives.Known;
import accord.messages.CheckStatus.CheckStatusOk;
import accord.messages.CheckStatus.IncludeInfo;
+import accord.primitives.RoutingKeys;
import accord.primitives.WithQuorum;
import accord.primitives.Participants;
import accord.primitives.Route;
import accord.primitives.TxnId;
-import accord.utils.MapReduceConsume;
import static accord.api.TraceEventType.FETCH;
import static accord.coordinate.Infer.InvalidIf.NotKnownToBeInvalid;
@@ -99,8 +99,20 @@ public class FetchRoute extends CheckShards<Route<?>, Participants<?>>
else
{
StoreSelector selector = reportTo.refine(txnId, null, query);
- node.mapReduceConsumeLocal(PreLoadContext.contextFor(txnId, "Report Route"), selector, new MapReduceConsume<>()
+ node.commandStores().mapReduceConsume(selector, new MapReduceConsumeCommandStores<>(RoutingKeys.EMPTY)
{
+ @Override
+ public TxnId primaryTxnId()
+ {
+ return txnId;
+ }
+
+ @Override
+ public String reason()
+ {
+ return "Report Route";
+ }
+
@Override
public void accept(Object result, Throwable failure)
{
@@ -108,7 +120,7 @@ public class FetchRoute extends CheckShards<Route<?>, Participants<?>>
}
@Override
- public Object apply(SafeCommandStore safeStore)
+ public Object applyInternal(SafeCommandStore safeStore)
{
SafeCommand safeCommand = safeStore.ifInitialised(txnId);
if (safeCommand != null)
diff --git a/accord-core/src/main/java/accord/coordinate/Infer.java b/accord-core/src/main/java/accord/coordinate/Infer.java
index 5c515200..a50ba81d 100644
--- a/accord-core/src/main/java/accord/coordinate/Infer.java
+++ b/accord-core/src/main/java/accord/coordinate/Infer.java
@@ -27,8 +27,8 @@ import accord.local.Command;
import accord.local.CommandStores.StoreFinder;
import accord.local.CommandStores.StoreSelector;
import accord.local.Commands;
+import accord.local.MapReduceConsumeCommandStores;
import accord.local.Node;
-import accord.local.PreLoadContext;
import accord.local.SafeCommand;
import accord.local.SafeCommandStore;
import accord.primitives.Status;
@@ -37,7 +37,6 @@ import accord.local.StoreParticipants;
import accord.primitives.Participants;
import accord.primitives.TxnId;
import accord.utils.Invariants;
-import accord.utils.MapReduceConsume;
import static accord.primitives.Status.PreCommitted;
@@ -105,24 +104,23 @@ public class Infer
}
// TODO (required): audit all use cases
- private static abstract class CleanupAndCallback<T> implements MapReduceConsume<SafeCommandStore, Void>, PreLoadContext
+ private static abstract class CleanupAndCallback<T> extends MapReduceConsumeCommandStores<Participants<?>, Void>
{
final Node node;
final TxnId txnId;
// TODO (expected): more consistent handling of transactions that only MAY intersect a commandStore
// (e.g. dependencies from an earlier epoch that have not yet committed, or commands that are proposed to execute in a later epoch than eventually agreed)
final StoreSelector reportTo;
- final Participants<?> participants;
final T param;
final BiConsumer<T, Throwable> callback;
final Tracing tracing;
private CleanupAndCallback(Node node, TxnId txnId, StoreSelector reportTo, Participants<?> participants, T param, BiConsumer<T, Throwable> callback, Tracing tracing)
{
+ super(participants);
this.node = node;
this.txnId = txnId;
this.reportTo = reportTo;
- this.participants = participants;
this.param = param;
this.callback = callback;
this.tracing = tracing;
@@ -130,17 +128,23 @@ public class Infer
void start()
{
- node.mapReduceConsumeLocal(this, reportTo.refine(txnId, null, participants), this);
+ node.commandStores().mapReduceConsume(reportTo.refine(txnId, null, scope), this);
}
@Override
- public Void apply(SafeCommandStore safeStore)
+ public Void applyInternal(SafeCommandStore safeStore)
{
// we're applying an invalidation, so the record will not be cleaned up until the whole range is truncated
- StoreParticipants participants = StoreParticipants.notAccept(safeStore, this.participants, txnId);
+ StoreParticipants participants = StoreParticipants.notAccept(safeStore, scope, txnId);
return apply(safeStore, safeStore.get(txnId, participants));
}
+ @Override
+ protected Void refuseInternal(SafeCommandStore safeStore)
+ {
+ return null;
+ }
+
abstract Void apply(SafeCommandStore safeStore, SafeCommand safeCommand);
@Override
@@ -188,14 +192,14 @@ public class Infer
if (tracing != null)
tracing.trace(safeStore.commandStore(), "Invalidating (from %s)", command.saveStatus());
Invariants.require(!command.hasBeen(PreCommitted) || command.hasBeen(Status.Truncated), "Unexpected status for %s", command);
- Commands.commitInvalidate(safeStore, safeCommand, participants);
+ Commands.commitInvalidate(safeStore, safeCommand, scope);
return null;
}
@Override
public String reason()
{
- return "Invalidate " + txnId + " with " + participants;
+ return "Invalidate";
}
}
}
diff --git a/accord-core/src/main/java/accord/coordinate/Invalidate.java b/accord-core/src/main/java/accord/coordinate/Invalidate.java
index b6a18422..bfd471f4 100644
--- a/accord-core/src/main/java/accord/coordinate/Invalidate.java
+++ b/accord-core/src/main/java/accord/coordinate/Invalidate.java
@@ -282,14 +282,41 @@ public class Invalidate extends AbstractCoordination<Outcome, InvalidateReply, I
{
// TODO (desired): merge with FetchData.InvalidateOnDone
// TODO (desired): when sending to network, register a callback for when local application of commitInvalidate message ahs been performed, so no need to special-case
- node.forEachLocal(PreLoadContext.contextFor(txnId, "Local Commit Invalidate"), reportTo.refine(txnId, null, commitTo), safeStore -> {
- // TODO (expected): consid
- StoreParticipants participants = StoreParticipants.notAccept(safeStore, commitTo, txnId);
- Commands.commitInvalidate(safeStore, safeStore.get(txnId, participants), commitTo);
- }).begin((s, f) -> {
- callback.accept(INVALIDATED, null);
- if (f != null) // TODO (required): consider exception handling more carefully: should we catch these prior to passing to callbacks?
- node.agent().onUncaughtException(f);
+ node.commandStores().mapReduceConsume(reportTo, new MapReduceConsumeCommandStores<Participants<?>, Void>(commitTo)
+ {
+ @Override
+ public Void reduce(Void o1, Void o2)
+ {
+ return null;
+ }
+
+ @Override
+ public void accept(Void result, Throwable failure)
+ {
+ callback.accept(INVALIDATED, null);
+ if (failure != null) // TODO (required): consider exception handling more carefully: should we catch these prior to passing to callbacks?
+ node.agent().onUncaughtException(failure);
+ }
+
+ @Override
+ public TxnId primaryTxnId()
+ {
+ return txnId;
+ }
+
+ @Override
+ public String reason()
+ {
+ return "Local Commit Invalidate";
+ }
+
+ @Override
+ protected Void applyInternal(SafeCommandStore safeStore)
+ {
+ StoreParticipants participants = StoreParticipants.notAccept(safeStore, commitTo, txnId);
+ Commands.commitInvalidate(safeStore, safeStore.get(txnId, participants), commitTo);
+ return null;
+ }
});
}
diff --git a/accord-core/src/main/java/accord/coordinate/KeyBarriers.java b/accord-core/src/main/java/accord/coordinate/KeyBarriers.java
index 8434e2c8..f6801a01 100644
--- a/accord-core/src/main/java/accord/coordinate/KeyBarriers.java
+++ b/accord-core/src/main/java/accord/coordinate/KeyBarriers.java
@@ -25,8 +25,8 @@ import org.slf4j.LoggerFactory;
import accord.api.RoutingKey;
import accord.local.CommandSummaries;
+import accord.local.MapReduceConsumeCommandStores;
import accord.local.Node;
-import accord.local.PreLoadContext;
import accord.local.SafeCommandStore;
import accord.local.SequentialAsyncExecutor;
import accord.local.durability.DurabilityService.SyncLocal;
@@ -38,9 +38,9 @@ import accord.primitives.Timestamp;
import accord.primitives.TxnId;
import accord.primitives.Unseekable;
import accord.topology.Topologies;
-import accord.utils.MapReduce;
import accord.utils.async.AsyncChain;
import accord.utils.async.AsyncChains;
+import accord.utils.async.AsyncResult;
import accord.utils.async.AsyncResults;
import accord.utils.async.Cancellable;
@@ -86,9 +86,11 @@ public class KeyBarriers
}
}
- public static AsyncChain<Found> find(Node node, Timestamp min, RoutingKey key, SyncLocal syncLocal, SyncRemote syncRemote)
+ public static AsyncResult<Found> find(Node node, Timestamp min, RoutingKey key, SyncLocal syncLocal, SyncRemote syncRemote)
{
- return node.commandStores().mapReduce((PreLoadContext.Empty) () -> "Key Barrier", RoutingKeys.of(key), min.epoch(), Long.MAX_VALUE, new Find(min, key, syncLocal, syncRemote));
+ Find find = new Find(min, key, syncLocal, syncRemote);
+ node.commandStores().mapReduceConsume(min.epoch(), Long.MAX_VALUE, find);
+ return find.result;
}
/*
@@ -98,8 +100,9 @@ public class KeyBarriers
* For Applied we can return success immediately with the executeAt epoch. For PreApplied we can add
* a listener for when it transitions to Applied and then return success.
*/
- static class Find extends AsyncResults.AbstractResult<Found> implements MapReduce<SafeCommandStore, Found>, CommandSummaries.AllCommandVisitor
+ static class Find extends MapReduceConsumeCommandStores<RoutingKeys, Found> implements CommandSummaries.AllCommandVisitor
{
+ final AsyncResults.SettableByCallback<Found> result = new AsyncResults.SettableByCallback<>();
final Timestamp min;
final RoutingKey find;
final SyncLocal syncLocal;
@@ -108,6 +111,7 @@ public class KeyBarriers
Find(Timestamp min, RoutingKey find, SyncLocal syncLocal, SyncRemote syncRemote)
{
+ super(RoutingKeys.of(find));
this.min = min;
this.find = find;
this.syncLocal = syncLocal;
@@ -115,15 +119,21 @@ public class KeyBarriers
}
@Override
- public Found apply(SafeCommandStore safeStore)
+ public Found applyInternal(SafeCommandStore safeStore)
{
// Barriers are trying to establish that committed transactions are applied before the barrier (or in this case just minEpoch)
// so all existing transaction types should ensure that at this point. An earlier txnid may have an executeAt that is after
// this barrier or the transaction we listen on and that is fine
- safeStore.visit(RoutingKeys.of(find), TxnId.NONE, Ws, STARTED_AFTER, min, IGNORE, this);
+ safeStore.visit(scope, TxnId.NONE, Ws, STARTED_AFTER, min, IGNORE, this);
return found;
}
+ @Override
+ protected Found refuseInternal(SafeCommandStore safeStore)
+ {
+ return null;
+ }
+
@Override
public boolean visit(Unseekable keyOrRange, TxnId txnId, Timestamp executeAt, CommandSummaries.SummaryStatus status, @Nullable CommandSummaries.IsDep dep, Status.Durability minDurability)
{
@@ -147,6 +157,25 @@ public class KeyBarriers
{
throw illegalState("Should not be possible to find multiple transactions");
}
+
+ @Nullable
+ @Override
+ public TxnId primaryTxnId()
+ {
+ return null;
+ }
+
+ @Override
+ public String reason()
+ {
+ return "Find existing transaction";
+ }
+
+ @Override
+ public void accept(Found result, Throwable failure)
+ {
+ this.result.accept(result, failure);
+ }
}
public static AsyncChain<Void> awaitLocal(Node node, TxnId txnId, RoutingKey key)
diff --git a/accord-core/src/main/java/accord/impl/AbstractConfigurationService.java b/accord-core/src/main/java/accord/impl/AbstractConfigurationService.java
index e6c830b0..caa812e6 100644
--- a/accord-core/src/main/java/accord/impl/AbstractConfigurationService.java
+++ b/accord-core/src/main/java/accord/impl/AbstractConfigurationService.java
@@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.Executor;
+import java.util.function.Function;
import javax.annotation.Nullable;
import javax.annotation.concurrent.GuardedBy;
@@ -32,6 +33,7 @@ import org.slf4j.LoggerFactory;
import accord.api.Agent;
import accord.api.ConfigurationService;
+import accord.api.VisibleForImplementation;
import accord.local.Node;
import accord.primitives.Ranges;
import accord.topology.Topology;
@@ -58,7 +60,7 @@ public abstract class AbstractConfigurationService<EpochState extends AbstractCo
protected final AsyncResult.Settable<Topology> received = AsyncResults.settable();
protected final AsyncResult.Settable<Void> acknowledged = AsyncResults.settable();
@GuardedBy("AbstractEpochHistory.this")
- protected AsyncResult<Void> reads = null;
+ protected EpochReady ready = null;
@GuardedBy("AbstractEpochHistory.this")
protected Topology topology = null;
@@ -72,9 +74,9 @@ public abstract class AbstractConfigurationService<EpochState extends AbstractCo
return epoch;
}
- public boolean isReady()
+ public EpochReady ready()
{
- return reads != null && reads.isDone();
+ return ready;
}
@Override
@@ -98,8 +100,8 @@ public abstract class AbstractConfigurationService<EpochState extends AbstractCo
.append(received)
.append(" acknowledged ")
.append(acknowledged)
- .append(" reads ")
- .append(reads);
+ .append(" ready ")
+ .append(ready);
}
@VisibleForTesting
@@ -108,7 +110,7 @@ public abstract class AbstractConfigurationService<EpochState extends AbstractCo
this.topology = topology;
received.setSuccess(topology);
acknowledged.setSuccess(null);
- reads = AsyncResults.success(null);
+ ready = EpochReady.done(topology.epoch());
}
}
@@ -188,7 +190,7 @@ public abstract class AbstractConfigurationService<EpochState extends AbstractCo
return getOrCreate(epoch);
}
- synchronized EpochState getOrCreate(long epoch)
+ protected synchronized EpochState getOrCreate(long epoch)
{
Invariants.requireArgument(!wasTruncated(epoch), "Can not re-create truncated epoch %d. Last truncated: %d", epoch, lastTruncated);
Invariants.requireArgument(epoch >= 0, "Epoch must be non-negative but given %d", epoch);
@@ -276,8 +278,8 @@ public abstract class AbstractConfigurationService<EpochState extends AbstractCo
Invariants.require(lastAcknowledged == epoch - 1 || epoch == 0 || lastAcknowledged == 0,
"Epoch %d != %d + 1", epoch, lastAcknowledged);
state = getOrCreate(epoch);
- Invariants.require(state.reads == null, "Reads result was already set for epoch", epoch);
- state.reads = ready.reads;
+ Invariants.require(state.ready == null, "Ready result was already set for epoch", epoch);
+ state.ready = ready;
lastAcknowledged = epoch;
}
// avoid resolving the future while holding the lock, as the callbacks get called in-line
@@ -306,7 +308,8 @@ public abstract class AbstractConfigurationService<EpochState extends AbstractCo
int highestReadyIdx = -1;
for (int i = trimFrom; i >= 0; i--)
{
- if (epochs.get(i).isReady())
+ EpochReady ready = epochs.get(i).ready;
+ if (ready != null && ready.reads.isDone())
{
highestReadyIdx = i;
break;
@@ -318,7 +321,7 @@ public abstract class AbstractConfigurationService<EpochState extends AbstractCo
return;
List<EpochState> next = new ArrayList<>(epochs.subList(highestReadyIdx, count));
- Invariants.require(next.get(0).reads.isDone());
+ Invariants.require(next.get(0).ready.reads.isDone());
epochs = next;
lastTruncated = next.get(0).epoch - 1;
}
@@ -372,19 +375,18 @@ public abstract class AbstractConfigurationService<EpochState extends AbstractCo
@Override
public abstract void fetchTopologyForEpoch(long epoch);
- // TODO (expected): rename, sync is too ambiguous
- protected abstract void localSyncComplete(Topology topology, boolean startSync);
- protected void localBootstrapsComplete(Topology topology) {}
+ protected abstract void onReadyToCoordinate(Topology topology, boolean isNewEvent);
+ // TODO (expected): startSync should not be necessary if we handle pending propagation work better
@Override
- public void acknowledgeEpoch(EpochReady ready, boolean startSync)
+ public boolean acknowledgeEpoch(EpochReady ready, boolean startSync)
{
if (epochs.wasTruncated(ready.epoch))
- return;
+ return false;
ready.metadata.invokeIfSuccess(() -> epochs.acknowledge(ready)).invoke(agent);
- ready.coordinate.invokeIfSuccess(() -> localSyncComplete(epochs.getOrCreate(ready.epoch).topology, startSync)).invoke(agent);
- ready.reads.invokeIfSuccess(() -> localBootstrapsComplete(epochs.getOrCreate(ready.epoch).topology)).invoke(agent);
+ ready.coordinate.invokeIfSuccess(() -> onReadyToCoordinate(epochs.getOrCreate(ready.epoch).topology, startSync)).invoke(agent);
+ return true;
}
protected void topologyUpdatePostListenerNotify(Topology topology) {}
@@ -482,47 +484,17 @@ public abstract class AbstractConfigurationService<EpochState extends AbstractCo
return sb.toString();
}
- // synchronized because state.reads is written
- public AsyncResult<Void> epochReady(long epoch)
+ @VisibleForImplementation
+ public AsyncResult<Void> epochReady(long epoch, Function<EpochReady, AsyncResult<Void>> get)
{
+ // synchronized for state.ready visibility
synchronized (this)
{
EpochState state = epochs.getOrCreate(epoch);
- if (state.reads != null)
- return state.reads;
-
- return state.acknowledged.flatMap(r -> state.reads);
- }
- }
-
- public abstract static class Minimal extends AbstractConfigurationService<Minimal.EpochState, Minimal.EpochHistory>
- {
- static class EpochState extends AbstractEpochState
- {
- public EpochState(long epoch)
- {
- super(epoch);
- }
- }
+ if (state.ready != null)
+ return get.apply(state.ready);
- static class EpochHistory extends AbstractEpochHistory<EpochState>
- {
- @Override
- protected EpochState createEpochState(long epoch)
- {
- return new EpochState(epoch);
- }
- }
-
- public Minimal(Node.Id node, Agent agent)
- {
- super(node, agent);
- }
-
- @Override
- protected EpochHistory createEpochHistory()
- {
- return new EpochHistory();
+ return state.acknowledged.flatMap(r -> get.apply(state.ready));
}
}
}
\ No newline at end of file
diff --git a/accord-core/src/main/java/accord/impl/CommandChange.java b/accord-core/src/main/java/accord/impl/CommandChange.java
index 7b99f938..6bdcbec6 100644
--- a/accord-core/src/main/java/accord/impl/CommandChange.java
+++ b/accord-core/src/main/java/accord/impl/CommandChange.java
@@ -599,18 +599,11 @@ public class CommandChange
return null;
Invariants.require(txnId != null);
- if (participants != null)
- participants = participants.filter(LOAD, redundantBefore, txnId, saveStatus != null && saveStatus.known.isExecuteAtKnown() ? executeAt : null);
-
- if (durability == null)
- durability = NotDurable;
-
- WaitingOn waitingOn = null;
- if (this.waitingOn != null)
- waitingOn = this.waitingOn.construct(partialDeps(), executesAtLeast, minUniqueHlc);
-
if (cleanup != null)
{
+ // we don't apply participant filtering first because we may have erased e.g. executeAt
+ // but we may not have expunged, so we still need to return the suitable cleanup result;
+ // TODO (desired): should we have dedicated filtering logic here too? It will be filtered by the caller before use anyway.
switch (cleanup)
{
default: throw new UnhandledEnum(cleanup);
@@ -630,6 +623,20 @@ public class CommandChange
}
}
+ // TODO (expected): bitset of expected known fields for cheap and comprehensive expunge check
+ if (executeAt == null && saveStatus != null && saveStatus.known.isExecuteAtKnown())
+ return null;
+
+ if (participants != null)
+ participants = participants.filter(LOAD, redundantBefore, txnId, saveStatus != null && saveStatus.known.isExecuteAtKnown() ? executeAt : null);
+
+ if (durability == null)
+ durability = NotDurable;
+
+ WaitingOn waitingOn = null;
+ if (this.waitingOn != null)
+ waitingOn = this.waitingOn.construct(partialDeps(), executesAtLeast, minUniqueHlc);
+
Invariants.require(saveStatus != null);
switch (saveStatus.status)
{
diff --git a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
index b0918ec0..d1691747 100644
--- a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
+++ b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
@@ -658,7 +658,6 @@ public abstract class InMemoryCommandStore extends CommandStore
{
super(context, commandStore);
- super.setRangesForEpoch(ranges);
this.commands = commands;
this.commandsForKey = commandsForKey;
for (InMemorySafeCommand cmd : commands.values())
@@ -906,31 +905,31 @@ public abstract class InMemoryCommandStore extends CommandStore
if (txnId.is(EphemeralRead)) continue;
Participants<?> intersecting = (txnId.isSyncPoint() ? command.participants().owns(): command.participants().stillWaitsOn()).intersecting(covering, Minimal);
if (intersecting.isEmpty()) continue;
- if (commandStore().unsafeGetRedundantBefore().locallyDefunct(command.txnId(), intersecting) == ALL) continue;
+ if (commandStore().unsafeGetRedundantBefore().isLocallyDefunct(command.txnId(), intersecting, ALL)) continue;
if (txnId.is(Key))
{
// TODO (required): document our invariants around this scenario, where a transaction with a higher txnId
// but lower executeAt than a transaction that is pre-bootstrap is ignore by the CFK (but cannot be validated to be pre-bootstrap independently).
// This invariant requires a timestamp store for correctness at least (as do other protocol assumptions).
// Make sure these invariants are validated in all relevant locations in the burn test.
- boolean isShadowedByPreBootstrap = true;
+ boolean isShadowedByUnready = true;
for (RoutingKey key : (AbstractUnseekableKeys)command.participants().executes())
{
SafeCommandsForKey safeCfk = commandsForKey.get(key);
CommandsForKey cfk = safeCfk.current();
- int i = cfk.indexOf(cfk.bounds().bootstrappedAt);
+ int i = cfk.indexOf(cfk.bounds().readyAt);
if (i < 0) i = -1 - i;
while (--i >= 0)
{
CommandsForKey.TxnInfo txn = cfk.get(i);
- if (txn.isCommittedToExecute() && txn.executeAt.compareTo(cfk.bootstrappedAt()) > 0)
+ if (txn.isCommittedToExecute() && txn.executeAt.compareTo(cfk.readyAt()) > 0)
break;
}
- isShadowedByPreBootstrap &= i >= 0;
+ isShadowedByUnready &= i >= 0;
}
- if (isShadowedByPreBootstrap) continue;
+ if (isShadowedByUnready) continue;
}
- illegalState();
+ illegalState("Prev: %s, updated: %s, command: %s", prev, updated, command);
}
}
}
@@ -1016,7 +1015,7 @@ public abstract class InMemoryCommandStore extends CommandStore
super(id, time, agent, store, progressLogFactory, listenersFactory, epochUpdateHolder, journal);
this.executor = Executors.newSingleThreadExecutor(r -> {
Thread thread = new Thread(r);
- thread.setName(CommandStore.class.getSimpleName() + '[' + id + ',' + time.id() + ']');
+ thread.setName(CommandStore.class.getSimpleName() + '[' + time.id() + ']');
return thread;
});
// "this" is leaked before constructor is completed, but since all fields are "final" and set before "this"
@@ -1205,18 +1204,15 @@ public abstract class InMemoryCommandStore extends CommandStore
* Replay and loading logic
*/
- // redundantBefore, durableBefore, newBootstrapBeganAt, safeToRead, rangesForEpoch are
- // not replayed here. It is assumed that persistence on the application side will ensure
- // they are brought up to latest values _before_ replay.
- public void clear()
+ @VisibleForTesting
+ public void unsafeClearForTesting()
{
- Invariants.require(current == null);
- progressLog.clear();
+ super.unsafeClearForTesting();
commands.clear();
commandsByExecuteAt.clear();
commandsForKey.clear();
rangeCommands.clear();
- listeners.clear();
+ progressLog.clear();
unsafeSetRejectBefore(new RejectBefore());
}
diff --git a/accord-core/src/main/java/accord/impl/progresslog/PackedKeyTracker.java b/accord-core/src/main/java/accord/impl/progresslog/PackedKeyTracker.java
index 2f0c6f30..35d90ee8 100644
--- a/accord-core/src/main/java/accord/impl/progresslog/PackedKeyTracker.java
+++ b/accord-core/src/main/java/accord/impl/progresslog/PackedKeyTracker.java
@@ -68,10 +68,7 @@ public class PackedKeyTracker
static int roundCallbackBitSet(DefaultProgressLog instance, TxnId txnId, Node.Id from, Route<?> route, int callbackId, int roundIndex, int roundSize)
{
if (callbackId != roundIndex)
- {
- Invariants.require(callbackId / roundSize < roundIndex);
- return 0; // stale callback for earlier round
- }
+ return 0; // stale callback for earlier round OR different command store
int start = roundSize * roundIndex;
int end = Math.min((roundSize + 1) * roundIndex, route.size());
diff --git a/accord-core/src/main/java/accord/local/Bootstrap.java b/accord-core/src/main/java/accord/local/Bootstrap.java
index 19b852eb..eed764ea 100644
--- a/accord-core/src/main/java/accord/local/Bootstrap.java
+++ b/accord-core/src/main/java/accord/local/Bootstrap.java
@@ -21,6 +21,7 @@ package accord.local;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.function.BiConsumer;
+import java.util.function.Consumer;
import javax.annotation.Nonnull;
@@ -28,6 +29,7 @@ import accord.api.Agent;
import accord.api.DataStore;
import accord.api.DataStore.FetchRanges;
import accord.api.DataStore.FetchResult;
+import accord.api.DataStore.FetchKind;
import accord.api.DataStore.StartingRangeFetch;
import accord.coordinate.CoordinateSyncPoint;
import accord.primitives.Ranges;
@@ -40,6 +42,7 @@ import accord.utils.ReducingRangeMap;
import accord.utils.async.AsyncResult;
import accord.utils.async.AsyncResults;
+import static accord.api.DataStore.FetchKind.Image;
import static accord.primitives.Routables.Slice.Minimal;
import static accord.primitives.Txn.Kind.ExclusiveSyncPoint;
import static accord.utils.Invariants.illegalState;
@@ -110,12 +113,12 @@ class Bootstrap
this.attempt = attempt;
}
- void start(SafeCommandStore safeStore)
+ TxnId start(SafeCommandStore safeStore)
{
if (valid.isEmpty())
{
maybeComplete();
- return;
+ return globalSyncId;
}
globalSyncId = node.nextTxnIdWithDefaultFlags(ExclusiveSyncPoint, Routable.Domain.Range);
@@ -124,11 +127,11 @@ class Bootstrap
if (!node.topology().hasEpoch(globalSyncId.epoch()))
{
// Ignore timeouts fetching the epoch, always keep trying to bootstrap
- node.withEpochAtLeast(globalSyncId.epoch(), null, (ignored, failure) -> store.execute((PreLoadContext.Empty) () -> "Start Bootstrap", Attempt.this::start, (ignored1, failure2) -> {
+ node.withEpochAtLeast(globalSyncId.epoch(), null, (ignored, failure) -> commandStore.execute((PreLoadContext.Empty) () -> "Start Bootstrap", (Consumer<SafeCommandStore>) Attempt.this::start, (ignored1, failure2) -> {
if (failure2 != null)
node.agent().acceptAndWrap(null, failure2);
}));
- return;
+ return globalSyncId;
}
// we fix here the ranges we use for the synthetic command, even though we may end up only finishing a subset
@@ -139,16 +142,17 @@ class Bootstrap
CoordinateSyncPoint.exclusive(node, globalSyncId, commitRanges)
.flatMap(success -> commandStore.chain((PreLoadContext.Empty) () -> "Mark Bootstrapping", safeStore0 -> {
// we submit a separate execution so that we know markBootstrapping is durable before we initiate the fetch
- store.markBootstrapping(safeStore0, globalSyncId, commitRanges);
+ Bootstrap.this.commandStore.markBootstrapping(safeStore0, globalSyncId, commitRanges);
return success;
}))
- .flatMap(syncPoint -> node.withEpochAtLeast(epoch, null, () -> store.chain((PreLoadContext.Empty) () -> "Start Bootstrap Fetch", safeStore1 -> {
+ .flatMap(syncPoint -> node.withEpochAtLeast(epoch, null, () -> Bootstrap.this.commandStore.chain((PreLoadContext.Empty) () -> "Start Bootstrap Fetch", safeStore1 -> {
if (valid.isEmpty()) // we've lost ownership of the range
return AsyncResults.success(Ranges.EMPTY);
- return fetch = safeStore1.dataStore().fetch(node, safeStore1, valid, syncPoint, this);
+ return fetch = safeStore1.dataStore().fetch(node, safeStore1, valid, syncPoint, this, Image);
})))
.flatMapResult(i -> i)
.begin(this);
+ return globalSyncId;
}
// we no longer want to fetch these ranges (perhaps we no longer own them)
@@ -189,7 +193,7 @@ class Bootstrap
// mark all ranges unsafe to read
// TODO (desired): if we have some ranges as safeToRead then we should really invalidate them immediately and not wait to execute under commandStore;
// could use synchronized to manage updates to these collections
- store.markUnsafeToRead(ranges);
+ commandStore.markUnsafeToRead(ranges);
// find any pre-existing states we may overlap with, mark both as overlaps, and add ourselves to the collection
UnsafeToRead newState = new UnsafeToRead(ranges);
@@ -271,9 +275,9 @@ class Bootstrap
if (hasFailed)
accept(null, failure);
- store.agent().onFailedBootstrap(attempt, "PartialFetch", newFailures, () -> {
+ commandStore.agent().ownershipEvents().onFailedBootstrap(attempt, "PartialFetch", newFailures, () -> {
node.scheduler().selfRecurring(() -> {
- store.execute((PreLoadContext.Empty) () -> "Restart Bootstrap", safeStore -> { restart(safeStore, newFailures.slice(allValid, Minimal), attempt + 1); }, store.agent());
+ commandStore.execute((PreLoadContext.Empty) () -> "Restart Bootstrap", safeStore -> { restart(safeStore, newFailures.slice(allValid, Minimal), attempt + 1); }, commandStore.agent());
}, 0L, TimeUnit.NANOSECONDS);
}, failure);
Invariants.require(!newFailures.intersects(fetchedAndSafeToRead));
@@ -298,7 +302,7 @@ class Bootstrap
safeToReadAts.foldlWithInputAndBounds(newDone, (safeToReadAt, s, start, end, i, j) -> {
Ranges mark = s.slice(Ranges.of(start.rangeFactory().newRange(start, end)), Minimal);
- store.markSafeToRead(globalSyncId, safeToReadAt, mark);
+ commandStore.markSafeToRead(globalSyncId, safeToReadAt, mark);
return s;
}, newDone, i -> false);
fetchedAndSafeToRead = fetchedAndSafeToRead.with(newDone);
@@ -347,17 +351,19 @@ class Bootstrap
complete(this);
if (!retry.isEmpty())
{
- store.agent().onFailedBootstrap(attempt, "Fetch", retry, () -> {
+ commandStore.agent().ownershipEvents().onFailedBootstrap(attempt, "Fetch", retry, () -> {
node.scheduler().selfRecurring(() -> {
- store.execute((PreLoadContext.Empty) () -> "Restart Bootstrap", safeStore -> { restart(safeStore, retry, attempt + 1); }, node.agent());
+ commandStore.execute((PreLoadContext.Empty) () -> "Restart Bootstrap", safeStore -> { restart(safeStore, retry, attempt + 1); }, node.agent());
}, 0L, TimeUnit.NANOSECONDS);
}, fetchOutcome);
}
+ commandStore.agent().ownershipEvents().onSuccessfulBootstrap(commandStore, attempt, epoch, fetchedAndSafeToRead);
}
}
+ final FetchKind kind;
final Node node;
- final CommandStore store;
+ final CommandStore commandStore;
final long epoch;
// TODO (required): make sure this is triggered in event of partial expiration of work to do
final AsyncResult.Settable<Void> data = AsyncResults.settable();
@@ -367,32 +373,38 @@ class Bootstrap
// TODO (expected): handle case where we clear these to empty; should trigger promise immediately
Ranges allValid, remaining;
- public Bootstrap(Node node, CommandStore store, long epoch, Ranges ranges)
+ public Bootstrap(Node node, CommandStore commandStore, long epoch, Ranges ranges)
+ {
+ this(node, commandStore, epoch, ranges, Image);
+ }
+
+ public Bootstrap(Node node, CommandStore commandStore, long epoch, Ranges ranges, FetchKind kind)
{
+ this.kind = kind;
this.node = node;
- this.store = store;
+ this.commandStore = commandStore;
this.epoch = epoch;
this.allValid = ranges;
this.remaining = ranges;
}
- void start(SafeCommandStore safeStore0)
+ TxnId start(SafeCommandStore safeStore0)
{
- restart(safeStore0, allValid, 0);
+ return restart(safeStore0, allValid, 0);
}
- private synchronized void restart(SafeCommandStore safeStore, Ranges ranges, int count)
+ private synchronized TxnId restart(SafeCommandStore safeStore, Ranges ranges, int count)
{
ranges = ranges.slice(allValid);
if (ranges.isEmpty())
- return;
+ return null;
for (Attempt attempt : inProgress)
Invariants.requireArgument(!ranges.intersects(attempt.valid));
Attempt attempt = new Attempt(ranges, count);
inProgress.add(attempt);
- attempt.start(safeStore);
+ return attempt.start(safeStore);
}
synchronized void complete(Attempt attempt)
@@ -405,7 +417,7 @@ class Bootstrap
{
data.setSuccess(null);
reads.setSuccess(null);
- store.complete(this);
+ commandStore.complete(this);
}
}
diff --git a/accord-core/src/main/java/accord/local/Cleanup.java b/accord-core/src/main/java/accord/local/Cleanup.java
index 3d191f60..92e2249d 100644
--- a/accord-core/src/main/java/accord/local/Cleanup.java
+++ b/accord-core/src/main/java/accord/local/Cleanup.java
@@ -40,6 +40,7 @@ import static accord.local.RedundantStatus.Property.LOCALLY_DEFUNCT;
import static accord.local.RedundantStatus.Property.LOCALLY_DURABLE_TO_DATA_STORE;
import static accord.local.RedundantStatus.Property.LOCALLY_REDUNDANT;
import static accord.local.RedundantStatus.Property.NOT_OWNED;
+import static accord.local.RedundantStatus.Property.LOG_UNAVAILABLE;
import static accord.local.RedundantStatus.Property.SHARD_APPLIED;
import static accord.local.RedundantStatus.Property.TRUNCATE_BEFORE;
import static accord.primitives.Known.KnownExecuteAt.ApplyAtKnown;
@@ -179,7 +180,10 @@ public enum Cleanup
FullRoute<?> route = Route.castToFullRoute(participants.route());
// we must not use executeAt here if input == PARTIAL because with partial compaction we might be merging the combination of some old executeAt with a later partial status
RedundantStatus redundant = redundantBefore.status(txnId, input == FULL && saveStatus.known.is(ApplyAtKnown) ? executeAt : null, route);
- Invariants.require(redundant.none(NOT_OWNED),"Command %s that is being loaded is not owned by this shard on route %s", txnId, route);
+ Invariants.require(redundant.none(NOT_OWNED), "Command %s that is being loaded is not owned by this shard on route %s", txnId, route);
+
+ if (redundant.all(LOG_UNAVAILABLE))
+ return logUnavailable(input);
if (redundant.none(LOCALLY_REDUNDANT))
return NO;
@@ -196,7 +200,7 @@ public enum Cleanup
Invariants.paranoid(redundant.all(SHARD_APPLIED));
if (!redundant.all(LOCALLY_DURABLE_TO_DATA_STORE))
- return truncateWithOutcome(txnId, input, redundant, min);
+ return truncateWithOutcome(txnId, input, redundant, min, participants);
HasOutcome test = durability.allShards();
if (saveStatus.compareTo(Vestigial) >= 0)
@@ -211,7 +215,7 @@ public enum Cleanup
test = Durability.HasOutcome.max(test, durableBefore.min(txnId, participants.route()));
if (test != Universal)
- return truncateWithOutcome(txnId, input, redundant, min);
+ return truncateWithOutcome(txnId, input, redundant, min, participants);
if (redundant.all(GC_BEFORE))
return erase(txnId, min);
@@ -230,20 +234,31 @@ public enum Cleanup
if (isCovering && txnId.isSyncPoint() && participants.owns().isEmpty())
{
RedundantStatus redundant = redundantBefore.status(txnId, null, participants.touches());
+ if (redundant.all(LOG_UNAVAILABLE))
+ return logUnavailable(input);
if (redundant.all(SHARD_APPLIED, LOCALLY_REDUNDANT))
return vestigial(txnId);
}
RedundantStatus ownStatus = redundantBefore.status(txnId, null, participants.owns());
+ if (ownStatus.all(LOG_UNAVAILABLE))
+ return logUnavailable(input);
return cleanupUndecided(txnId, ownStatus, isCovering);
}
- private static Cleanup cleanupIfUndecidedWithFullRoute(Input input, TxnId txnId, SaveStatus saveStatus, RedundantStatus redundantStatus)
+ private static Cleanup logUnavailable(Input input)
+ {
+ if (input == FULL)
+ throw new LogUnavailableException();
+ return ERASE;
+ }
+
+ private static Cleanup cleanupIfUndecidedWithFullRoute(Input input, TxnId txnId, SaveStatus saveStatus, RedundantStatus redundant)
{
if (input == PARTIAL || saveStatus.hasBeen(PreCommitted))
return NO;
- return cleanupUndecided(txnId, redundantStatus, true);
+ return cleanupUndecided(txnId, redundant, true);
}
private static Cleanup cleanupUndecided(TxnId txnId, RedundantStatus ownStatus, boolean isCoveringRoute)
@@ -296,10 +311,19 @@ public enum Cleanup
return INVALIDATE;
}
- private static Cleanup truncateWithOutcome(TxnId txnId, Input input, RedundantStatus status, Cleanup atLeast)
+ private static Cleanup truncateWithOutcome(TxnId txnId, Input input, RedundantStatus status, Cleanup atLeast, StoreParticipants participants)
{
- return atLeast.compareTo(TRUNCATE_WITH_OUTCOME) > 0 ? atLeast : input == PARTIAL || !status.all(LOCALLY_DEFUNCT)
- ? TRUNCATE_WITH_OUTCOME : TRUNCATE;
+ if (atLeast.compareTo(TRUNCATE_WITH_OUTCOME) >= 0)
+ return atLeast;
+ if (input == PARTIAL)
+ return TRUNCATE_WITH_OUTCOME;
+ if (status.all(LOCALLY_DEFUNCT))
+ return TRUNCATE;
+ // TODO (expected): tighten constraints when stillExecutes is null (this means undecided; should be handled elsewhere but confirm)
+ Participants<?> stillExecutes = participants.stillExecutes();
+ if (stillExecutes != null && stillExecutes.isEmpty())
+ return TRUNCATE;
+ return TRUNCATE_WITH_OUTCOME;
}
private static Cleanup truncate(TxnId txnId, Cleanup atLeast)
diff --git a/accord-core/src/main/java/accord/local/Command.java b/accord-core/src/main/java/accord/local/Command.java
index 8bce357d..b9d5b812 100644
--- a/accord-core/src/main/java/accord/local/Command.java
+++ b/accord-core/src/main/java/accord/local/Command.java
@@ -1687,7 +1687,7 @@ public abstract class Command implements ICommand
case DepsUnknown:
case DepsErased:
case NoDeps:
- Invariants.require(deps == null);
+ Invariants.require(deps == null, "Deps should have been null, but were %s", deps);
break;
case DepsFromCoordinator:
case DepsProposedFixed:
diff --git a/accord-core/src/main/java/accord/local/CommandStore.java b/accord-core/src/main/java/accord/local/CommandStore.java
index df90e28a..ffd4e9ab 100644
--- a/accord-core/src/main/java/accord/local/CommandStore.java
+++ b/accord-core/src/main/java/accord/local/CommandStore.java
@@ -19,7 +19,9 @@
package accord.local;
import java.util.AbstractMap.SimpleImmutableEntry;
+import java.util.ArrayList;
import java.util.Collections;
+import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
import java.util.Set;
@@ -40,11 +42,15 @@ import org.slf4j.LoggerFactory;
import accord.api.Agent;
import accord.api.ConfigurationService.EpochReady;
import accord.api.DataStore;
+import accord.api.DataStore.FetchKind;
import accord.api.Journal;
import accord.api.LocalListeners;
import accord.api.ProgressLog;
import accord.impl.AbstractAsyncExecutor;
+import accord.coordinate.CoordinateMaxConflict;
+import accord.local.CommandStores.BootstrapRangeAction;
import accord.local.CommandStores.RangesForEpoch;
+import accord.local.PreLoadContext.Empty;
import accord.local.RedundantBefore.Bounds;
import accord.local.RedundantStatus.SomeStatus;
import accord.primitives.Ranges;
@@ -55,21 +61,28 @@ import accord.primitives.TxnId;
import accord.primitives.Unseekables;
import accord.utils.DeterministicIdentitySet;
import accord.utils.Invariants;
+import accord.utils.Reduce;
+import accord.utils.UnhandledEnum;
import accord.utils.async.AsyncChain;
+import accord.utils.async.AsyncChains;
import accord.utils.async.AsyncResult;
import accord.utils.async.AsyncResults;
+import accord.utils.async.AsyncResults.SettableByCallback;
import accord.utils.async.Cancellable;
+import accord.utils.async.AsyncResults.SettableResult;
import org.agrona.collections.LongHashSet;
import static accord.api.ConfigurationService.EpochReady.DONE;
+import static accord.api.DataStore.FetchKind.Image;
import static accord.api.ProtocolModifiers.Toggles.requiresUniqueHlcs;
import static accord.local.RedundantStatus.SomeStatus.GC_BEFORE_AND_LOCALLY_DURABLE;
import static accord.local.RedundantStatus.SomeStatus.LOCALLY_APPLIED_ONLY;
import static accord.local.RedundantStatus.SomeStatus.LOCALLY_DURABLE_TO_COMMAND_STORE_ONLY;
import static accord.local.RedundantStatus.SomeStatus.LOCALLY_DURABLE_TO_DATA_STORE_ONLY;
import static accord.local.RedundantStatus.SomeStatus.LOCALLY_WITNESSED_ONLY;
+import static accord.local.RedundantStatus.SomeStatus.LOG_UNAVAILABLE_ONLY;
import static accord.local.RedundantStatus.SomeStatus.QUORUM_APPLIED_ONLY;
-import static accord.local.RedundantStatus.SomeStatus.PRE_BOOTSTRAP_ONLY;
+import static accord.local.RedundantStatus.SomeStatus.UNREADY_ONLY;
import static accord.local.RedundantStatus.SomeStatus.SHARD_APPLIED_ONLY;
import static accord.primitives.AbstractRanges.UnionMode.MERGE_ADJACENT;
import static accord.primitives.Routables.Slice.Minimal;
@@ -101,7 +114,7 @@ public abstract class CommandStore implements AbstractAsyncExecutor, SequentialA
// TODO (desired): can better encapsulate by accepting only the newRangesForEpoch and deriving the add/remove ranges
public void add(long epoch, RangesForEpoch newRangesForEpoch, Ranges addRanges)
{
- RedundantBefore addRedundantBefore = RedundantBefore.create(addRanges, epoch, Long.MAX_VALUE, TxnId.minForEpoch(epoch), PRE_BOOTSTRAP_ONLY);
+ RedundantBefore addRedundantBefore = RedundantBefore.create(addRanges, epoch, Long.MAX_VALUE, TxnId.minForEpoch(epoch), UNREADY_ONLY);
update(newRangesForEpoch, addRedundantBefore);
}
@@ -148,10 +161,11 @@ public abstract class CommandStore implements AbstractAsyncExecutor, SequentialA
private MaxDecidedRX maxDecidedRX = MaxDecidedRX.EMPTY;
private int maxConflictsUpdates = 0;
protected RangesForEpoch rangesForEpoch;
+ protected @Nullable Ranges refuses;
/**
* safeToRead is related to RedundantBefore, but a distinct concept.
- * While bootstrappedAt defines the txnId bounds we expect to maintain data for locally,
+ * While readyAt defines the txnId bounds we expect to maintain data for locally,
* safeToRead defines executeAt bounds we can safely participate in transaction execution for.
* safeToRead is defined by the no-op transaction we execute after a bootstrap is initiated,
* and creates a global bound before which we know we have complete data from our bootstrap.
@@ -167,19 +181,19 @@ public abstract class CommandStore implements AbstractAsyncExecutor, SequentialA
private final Set<Bootstrap> bootstraps = Collections.synchronizedSet(new DeterministicIdentitySet<>());
@Nullable private RejectBefore rejectBefore;
- static class WaitingOnSync
+ static class WaitingOnVisibility
{
- final AsyncResults.SettableResult<Void> whenDone;
+ final SettableResult<Void> whenDone;
final Ranges allRanges;
Ranges waitingOn, waitingOnDurable;
- WaitingOnSync(AsyncResults.SettableResult<Void> whenDone, Ranges ranges)
+ WaitingOnVisibility(SettableResult<Void> whenDone, Ranges ranges)
{
this.whenDone = whenDone;
this.allRanges = this.waitingOn = this.waitingOnDurable = ranges;
}
}
- private final TreeMap<Long, WaitingOnSync> waitingOnSync = new TreeMap<>();
+ private final TreeMap<Long, WaitingOnVisibility> waitingOnVisibility = new TreeMap<>();
protected CommandStore(int id,
NodeCommandStoreService node,
@@ -214,6 +228,20 @@ public abstract class CommandStore implements AbstractAsyncExecutor, SequentialA
return agent;
}
+ public void unsafeClearForTesting()
+ {
+ progressLog.clear();
+ bootstraps.clear();
+ rangesForEpoch = null;
+ bootstrapBeganAt = emptyBootstrapBeganAt();
+ redundantBefore = RedundantBefore.EMPTY;
+ maxConflicts = MaxConflicts.EMPTY;
+ maxDecidedRX = MaxDecidedRX.EMPTY;
+ safeToRead = emptySafeToRead();
+ listeners.clear();
+ waitingOnVisibility.clear();
+ }
+
public void updateRangesForEpoch(SafeCommandStore safeStore)
{
EpochUpdate update = epochUpdateHolder.get();
@@ -252,7 +280,8 @@ public abstract class CommandStore implements AbstractAsyncExecutor, SequentialA
return maxDecidedRX;
}
- final void unsafeSetRangesForEpoch(RangesForEpoch newRangesForEpoch)
+ @VisibleForTesting
+ public final void unsafeSetRangesForEpoch(RangesForEpoch newRangesForEpoch)
{
rangesForEpoch = nonNull(newRangesForEpoch);
}
@@ -337,6 +366,21 @@ public abstract class CommandStore implements AbstractAsyncExecutor, SequentialA
redundantBefore = RedundantBefore.merge(redundantBefore, addRedundantBefore);
}
+ protected void unsafeRefuseRequests(Ranges refuse)
+ {
+ Invariants.require(refuses == null || !refuses.intersects(refuse));
+ if (refuses == null) refuses = refuse;
+ else refuses = refuses.with(refuse);
+ }
+
+ protected void unsafeAcceptRequests(Ranges accept)
+ {
+ Invariants.require(refuses != null && refuses.containsAll(accept));
+ refuses = refuses.without(accept);
+ if (refuses.isEmpty())
+ refuses = null;
+ }
+
/**
* This method may be invoked on a non-CommandStore thread
*/
@@ -387,11 +431,11 @@ public abstract class CommandStore implements AbstractAsyncExecutor, SequentialA
protected int dumpCounter = 0;
- protected void updateMaxConflicts(Command prev, Command updated)
+ protected void updateMaxConflicts(Command prev, Command updated, boolean force)
{
Timestamp executeAt = updated.executeAt();
if (executeAt == null) return;
- if (prev != null && prev.executeAt() != null && prev.executeAt().compareToStrict(executeAt) >= 0) return;
+ if (prev != null && prev.executeAt() != null && prev.executeAt().compareToStrict(executeAt) >= 0 && !force) return;
executeAt = executeAt.flattenUniqueHlc(); // this is what guarantees a bootstrap recipient can compute uniqueHlc safely
MaxConflicts updatedMaxConflicts = maxConflicts.update(updated.participants().hasTouched(), executeAt);
updateMaxConflicts(executeAt, updatedMaxConflicts);
@@ -521,22 +565,112 @@ public abstract class CommandStore implements AbstractAsyncExecutor, SequentialA
* So, the outer future's success is sufficient for the topology to be acknowledged, and the inner future for the
* bootstrap to be complete.
*/
- final Supplier<EpochReady> bootstrapper(Node node, Ranges newRanges, long epoch)
+ final Supplier<EpochReady> bootstrapper(Node node, Ranges newRanges, long epoch, BootstrapRangeAction action)
{
- return () -> {
- AsyncResult<EpochReady> metadata = submit((PreLoadContext.Empty) () -> "New Epoch", safeStore -> {
- Bootstrap bootstrap = new Bootstrap(node, this, epoch, newRanges);
- bootstraps.add(bootstrap);
- bootstrap.start(safeStore);
- return new EpochReady(epoch, null, null, bootstrap.data, bootstrap.reads);
- });
+ switch (action)
+ {
+ default: throw new UnhandledEnum(action);
+ case NO_BOOTSTRAP:
+ return () -> {
+ AsyncResult<Void> done = execute((Empty) () -> "Initialise New Epoch", (safeStore) -> {
+ // Merge in a base for any ranges that needs to be covered
+ Ranges newBootstrapRanges = newRanges;
+ for (Ranges existing : bootstrapBeganAt.values())
+ newBootstrapRanges = newBootstrapRanges.without(existing);
+ if (!newBootstrapRanges.isEmpty())
+ safeStore.setBootstrapBeganAt(bootstrap(TxnId.NONE, newBootstrapRanges, bootstrapBeganAt));
+ safeStore.setSafeToRead(purgeAndInsert(safeToRead, TxnId.NONE, newRanges));
+ markExclusiveSyncPointDecided(safeStore, TxnId.NONE, newRanges);
+ });
+
+ return new EpochReady(epoch, done, done, done, done);
+ };
+ case SAFE_BOOTSTRAP:
+ return () -> epochReadyAfterBootstrap(newRanges, epoch, startSafeBootstrap(node, newRanges, epoch));
+
+ case UNSAFE_BOOTSTRAP:
+ return () -> epochReadyAfterBootstrap(newRanges, epoch, startUnsafeBootstrap(node, newRanges, epoch, Image));
+ }
+ }
- AsyncResult<Void> readyToCoordinate = readyToCoordinate(newRanges, epoch);
- return new EpochReady(epoch, metadata.map(ignore -> null),
- readyToCoordinate,
- metadata.flatMap(e -> e.data),
- metadata.flatMap(e -> e.reads));
- };
+ private EpochReady epochReadyAfterBootstrap(Ranges newRanges, long epoch, AsyncResult<EpochReady> bootstrap)
+ {
+ AsyncResult<Void> readyToCoordinate = readyToCoordinate(newRanges, epoch);
+ return new EpochReady(epoch, bootstrap.map(ignore -> null),
+ readyToCoordinate,
+ bootstrap.flatMap(e -> e.data),
+ bootstrap.flatMap(e -> e.reads));
+ }
+
+ private AsyncResult<EpochReady> startSafeBootstrap(Node node, Ranges newRanges, long epoch)
+ {
+ return submit((Empty) () -> "New Epoch", safeStore -> {
+ Bootstrap bootstrap = new Bootstrap(node, this, epoch, newRanges);
+ bootstraps.add(bootstrap);
+ bootstrap.start(safeStore);
+ return new EpochReady(epoch, null, null, bootstrap.data, bootstrap.reads);
+ });
+ }
+
+ /**
+ * Rebootstraps some of the ranges for the command store. It follows steps similar to what
+ * bootstrap would go through, with two differences:
+ *
+ * * Marks pre-rebootstrap transactions with LOCALLY_LOST status, which means the node can not
+ * safely participate in pre-rebootstrap transactions, _even_ if they're coming after the node is
+ * done bootstrapping.
+ * * Marks the store as rebootstrapping, which will preclude rebootstrapping node from responding
+ * to PreAccept, Accept, and BeginRecovery and computing dependencies while node is being rebootstrapped,
+ * and ranges aren't ready to coordinate.
+ */
+ protected AsyncResult<EpochReady> startUnsafeBootstrap(Node node, Ranges ranges, long epoch, FetchKind fetch)
+ {
+ return submit((Empty) () -> "Refuse Requests for " + fetch + " Bootstrap", safeStore -> {
+ unsafeRefuseRequests(ranges);
+ safeStore.setSafeToRead(purgeHistory(safeToRead, ranges));
+ return new EpochReady(epoch, new SettableResult<>(), readyToCoordinate(ranges, epoch), new SettableByCallback<>(), new SettableByCallback<>());
+ }).invoke((success, fail) -> {
+ if (fail != null) logger.error("Fatal error initiating {} bootstrap for {}", this, fetch, fail);
+ else rebootstrap(node, ranges, epoch, 1, success, fetch);
+ });
+ }
+
+ private void rebootstrap(Node node, Ranges ranges, long epoch, int attempt, EpochReady ready, FetchKind fetch)
+ {
+ CoordinateMaxConflict
+ .maxConflict(node, ranges)
+ .recover(fail -> {
+ agent.ownershipEvents().onFailedBootstrap(attempt, "Fetch Max Conflict (to mark log safe at)", ranges, () -> rebootstrap(node, ranges, epoch, attempt + 1, ready, fetch), fail);
+ return AsyncChains.failure(fail);
+ }).flatMap(success -> chain((Empty) () -> "Initiate Unsafe " + fetch + " Bootstrap", safeStore -> {
+ node.uniqueNow(success.hlc()); // ensure we pick a higher timestamp than the maximum conflict we found globally
+ // Mark unsafe to read first
+
+ Ranges remaining = ranges.slice(rangesForEpoch.currentRanges());
+ if (remaining.isEmpty())
+ {
+ logger.info("Terminating unsafe {} bootstrap process for {} as no active ranges", fetch, this);
+ return AsyncChains.success(null);
+ }
+
+ Bootstrap bootstrap = new Bootstrap(node, this, epoch, remaining, fetch);
+ bootstraps.add(bootstrap);
+ // If rebootstrap can grab a later timestamp for subsequent attempts, but this timestamp is enough for us
+ // to establish which transactions, for which ranges the node can safely participate in).
+ TxnId unreadyBefore = bootstrap.start(safeStore);
+ safeStore.unsafeUpsertRedundantBefore(RedundantBefore.create(ranges, unreadyBefore, LOG_UNAVAILABLE_ONLY));
+ updateMaxConflicts(ranges, unreadyBefore);
+ // TODO (desired): we could start accepting non-dep requests here
+ ((AsyncResult.Settable<Void>)ready.metadata).setSuccess(null);
+ bootstrap.data.invoke((SettableByCallback<Void>)ready.data);
+ bootstrap.reads.invoke((SettableByCallback<Void>)ready.reads);
+ ready.coordinate.invokeIfSuccess(() -> {
+ execute((Empty)() -> "Accept Dependency Requests", safeStore0 -> {
+ unsafeAcceptRequests(remaining);
+ });
+ });
+ return null;
+ })).begin(agent);
}
/**
@@ -560,11 +694,11 @@ public abstract class CommandStore implements AbstractAsyncExecutor, SequentialA
TxnId minForEpoch = TxnId.minForEpoch(epoch);
Ranges remaining = redundantBefore.removeWitnessed(minForEpoch, ranges);
- AsyncResults.SettableResult<Void> whenDone = new AsyncResults.SettableResult<>();
- WaitingOnSync sync = new WaitingOnSync(whenDone, remaining);
- synchronized (waitingOnSync)
+ SettableResult<Void> whenDone = new SettableResult<>();
+ WaitingOnVisibility sync = new WaitingOnVisibility(whenDone, remaining);
+ synchronized (waitingOnVisibility)
{
- Invariants.require(waitingOnSync.put(epoch, sync) == null);
+ Invariants.require(waitingOnVisibility.put(epoch, sync) == null);
}
ensureReadyToCoordinate(epoch, ranges);
return whenDone;
@@ -584,8 +718,8 @@ public abstract class CommandStore implements AbstractAsyncExecutor, SequentialA
if (!retired.isEmpty())
{
logger.info("Failed to close epoch {} for ranges {} on store {}, but some are retired; marking these as synced.", epoch, ranges, id, fail);
- execute((PreLoadContext.Empty)() -> "Mark Retired Ranges Synced", safeStore -> {
- markSyncedInternal(safeStore, epoch, retired, "(Retired)");
+ execute((Empty)() -> "Mark Retired Ranges Synced", safeStore -> {
+ markVisibleInternal(safeStore, epoch, retired, "(Retired)");
});
}
else if (remaining.isEmpty())
@@ -604,7 +738,7 @@ public abstract class CommandStore implements AbstractAsyncExecutor, SequentialA
Supplier<EpochReady> unbootstrap(long epoch, Ranges removedRanges)
{
return () -> {
- AsyncResult<Void> done = submit((PreLoadContext.Empty) () -> "Unbootstrap", safeStore -> {
+ AsyncResult<Void> done = submit((Empty) () -> "Unbootstrap", safeStore -> {
for (Bootstrap prev : bootstraps)
{
Ranges abort = prev.allValid.slice(removedRanges, Minimal);
@@ -628,7 +762,7 @@ public abstract class CommandStore implements AbstractAsyncExecutor, SequentialA
safeStore.setBootstrapBeganAt(bootstrap(globalSyncId, ranges, bootstrapBeganAt));
safeStore.setSafeToRead(purgeHistory(safeToRead, ranges));
updateMaxConflicts(ranges, globalSyncId);
- RedundantBefore addRedundantBefore = RedundantBefore.create(ranges, Long.MIN_VALUE, Long.MAX_VALUE, globalSyncId, PRE_BOOTSTRAP_ONLY);
+ RedundantBefore addRedundantBefore = RedundantBefore.create(ranges, Long.MIN_VALUE, Long.MAX_VALUE, globalSyncId, UNREADY_ONLY);
safeStore.upsertRedundantBefore(addRedundantBefore);
}
@@ -650,7 +784,7 @@ public abstract class CommandStore implements AbstractAsyncExecutor, SequentialA
if (locallyRedundantBefore.compareTo(globalSyncId) < 0)
{
// TODO (expected): if bootstrapping only part of the range, mark the rest for GC; or relax this as can safely GC behind bootstrap
- TxnId maxBootstrap = safeStore.redundantBefore().max(slicedRanges, Bounds::maxBootstrappedAt);
+ TxnId maxBootstrap = safeStore.redundantBefore().max(slicedRanges, Bounds::maxReadyAt);
if (maxBootstrap.compareTo(globalSyncId) >= 0)
logger.info("Ignoring markShardDurable for a point we are bootstrapping. Bootstrapping: {}, Global: {}, Ranges: {}", maxBootstrap, globalSyncId, slicedRanges);
else
@@ -674,15 +808,43 @@ public abstract class CommandStore implements AbstractAsyncExecutor, SequentialA
listeners.clearBefore(this, clearWaitingBefore);
}
- protected final Ranges isWaitingOnSync(TxnId syncId, Ranges ranges)
+ public AsyncResult<Void> awaitVisibility(long epoch, Ranges ranges)
{
- synchronized (waitingOnSync)
+ synchronized (waitingOnVisibility)
{
- if (waitingOnSync.isEmpty())
+ if (waitingOnVisibility.isEmpty())
+ return AsyncResults.success(null);
+
+ List<AsyncResult<Void>> awaiting = new ArrayList<>();
+ for (Map.Entry<Long, WaitingOnVisibility> e : waitingOnVisibility.entrySet())
+ {
+ if (e.getKey() > epoch)
+ break;
+
+ Ranges remaining = e.getValue().waitingOn;
+ Ranges intersecting = remaining.slice(ranges, Minimal);
+ if (!intersecting.isEmpty())
+ {
+ awaiting.add(e.getValue().whenDone);
+ ranges = ranges.without(intersecting);
+ }
+ }
+
+ if (awaiting.isEmpty())
+ return AsyncResults.success(null);
+ return AsyncResults.reduce(awaiting, Reduce.toNull());
+ }
+ }
+
+ protected final Ranges isWaitingOnVisibility(TxnId syncId, Ranges ranges)
+ {
+ synchronized (waitingOnVisibility)
+ {
+ if (waitingOnVisibility.isEmpty())
return Ranges.EMPTY;
Ranges waitingOn = Ranges.EMPTY;
- for (Map.Entry<Long, WaitingOnSync> e : waitingOnSync.entrySet())
+ for (Map.Entry<Long, WaitingOnVisibility> e : waitingOnVisibility.entrySet())
{
if (e.getKey() > syncId.epoch())
break;
@@ -700,14 +862,14 @@ public abstract class CommandStore implements AbstractAsyncExecutor, SequentialA
}
}
- protected final void markSyncing(TxnId syncId, Ranges ranges)
+ protected final void markingVisible(TxnId syncId, Ranges ranges)
{
- synchronized (waitingOnSync)
+ synchronized (waitingOnVisibility)
{
- if (waitingOnSync.isEmpty())
+ if (waitingOnVisibility.isEmpty())
return;
- for (Map.Entry<Long, WaitingOnSync> e : waitingOnSync.entrySet())
+ for (Map.Entry<Long, WaitingOnVisibility> e : waitingOnVisibility.entrySet())
{
if (e.getKey() > syncId.epoch())
break;
@@ -719,14 +881,14 @@ public abstract class CommandStore implements AbstractAsyncExecutor, SequentialA
}
}
- protected final void unmarkSyncing(TxnId syncId, Ranges ranges)
+ protected final void cancelMarkingVisible(TxnId syncId, Ranges ranges)
{
- synchronized (waitingOnSync)
+ synchronized (waitingOnVisibility)
{
- if (waitingOnSync.isEmpty())
+ if (waitingOnVisibility.isEmpty())
return;
- for (Map.Entry<Long, WaitingOnSync> e : waitingOnSync.entrySet())
+ for (Map.Entry<Long, WaitingOnVisibility> e : waitingOnVisibility.entrySet())
{
if (e.getKey() > syncId.epoch())
break;
@@ -738,23 +900,23 @@ public abstract class CommandStore implements AbstractAsyncExecutor, SequentialA
}
}
- protected final void markSynced(SafeCommandStore safeStore, TxnId syncId, Ranges ranges)
+ protected final void markVisible(SafeCommandStore safeStore, TxnId syncId, Ranges ranges)
{
Invariants.require(syncId.is(VisibilitySyncPoint));
RedundantBefore addRedundantBefore = RedundantBefore.create(ranges, syncId, LOCALLY_WITNESSED_ONLY);
safeStore.upsertRedundantBefore(addRedundantBefore);
- markSyncedInternal(safeStore, syncId.epoch(), ranges, syncId);
+ markVisibleInternal(safeStore, syncId.epoch(), ranges, syncId);
}
- private void markSyncedInternal(SafeCommandStore safeStore, long epoch, Ranges ranges, Object describe)
+ private void markVisibleInternal(SafeCommandStore safeStore, long epoch, Ranges ranges, Object describe)
{
- synchronized (waitingOnSync)
+ synchronized (waitingOnVisibility)
{
- if (waitingOnSync.isEmpty())
+ if (waitingOnVisibility.isEmpty())
return;
LongHashSet remove = null;
- for (Map.Entry<Long, WaitingOnSync> e : waitingOnSync.entrySet())
+ for (Map.Entry<Long, WaitingOnVisibility> e : waitingOnVisibility.entrySet())
{
if (e.getKey() > epoch)
break;
@@ -782,7 +944,7 @@ public abstract class CommandStore implements AbstractAsyncExecutor, SequentialA
}
}
if (remove != null)
- remove.forEach(waitingOnSync::remove);
+ remove.forEach(waitingOnVisibility::remove);
}
}
@@ -803,7 +965,7 @@ public abstract class CommandStore implements AbstractAsyncExecutor, SequentialA
if (ranges.isEmpty())
return;
- agent.onStale(staleSince, ranges);
+ agent.ownershipEvents().onStale(staleSince, ranges);
RedundantBefore addRedundantBefore = RedundantBefore.createStale(ranges, staleUntilAtLeast);
safeStore.upsertRedundantBefore(addRedundantBefore);
@@ -821,7 +983,7 @@ public abstract class CommandStore implements AbstractAsyncExecutor, SequentialA
Supplier<EpochReady> initialise(long epoch, Ranges ranges)
{
return () -> {
- AsyncResult<Void> done = execute((PreLoadContext.Empty) () -> "Initialise New Epoch", (safeStore) -> {
+ AsyncResult<Void> done = execute((Empty) () -> "Initialise New Epoch", (safeStore) -> {
// Merge in a base for any ranges that needs to be covered
Ranges newBootstrapRanges = ranges;
for (Ranges existing : bootstrapBeganAt.values())
@@ -880,7 +1042,7 @@ public abstract class CommandStore implements AbstractAsyncExecutor, SequentialA
{
if (safeToRead.values().stream().anyMatch(r -> r.intersects(ranges)))
{
- execute((PreLoadContext.Empty) () -> "Mark Unsafe To Read", safeStore -> {
+ execute((Empty) () -> "Mark Unsafe To Read", safeStore -> {
safeStore.setSafeToRead(purgeHistory(safeToRead, ranges));
}, agent);
}
@@ -893,7 +1055,7 @@ public abstract class CommandStore implements AbstractAsyncExecutor, SequentialA
final synchronized void markSafeToRead(Timestamp forBootstrapAt, Timestamp at, Ranges ranges)
{
- execute((PreLoadContext.Empty) () -> "Mark Safe To Read", safeStore -> {
+ execute((Empty) () -> "Mark Safe To Read", safeStore -> {
// TODO (required): handle weird edge cases like newer at having a lower HLC than prior existing at, but higher epoch
Ranges validatedSafeToRead = redundantBefore.validateSafeToRead(forBootstrapAt, ranges);
safeStore.setSafeToRead(purgeAndInsert(safeToRead, at, validatedSafeToRead));
@@ -901,20 +1063,16 @@ public abstract class CommandStore implements AbstractAsyncExecutor, SequentialA
}, agent);
}
- public static ImmutableSortedMap<TxnId, Ranges> bootstrap(TxnId at, Ranges ranges, NavigableMap<TxnId, Ranges> bootstrappedAt)
+ public static ImmutableSortedMap<TxnId, Ranges> bootstrap(TxnId at, Ranges ranges, NavigableMap<TxnId, Ranges> readyAt)
{
Invariants.requireArgument(!ranges.isEmpty());
if (at == TxnId.NONE)
{
- for (Ranges rs : bootstrappedAt.values())
+ for (Ranges rs : readyAt.values())
Invariants.require(!ranges.intersects(rs));
}
- else
- {
- bootstrappedAt.floorEntry(at).getValue().containsAll(ranges);
- }
// if we're bootstrapping these ranges, then any period we previously owned the ranges for is effectively invalidated
- return purgeAndInsert(bootstrappedAt, at, ranges);
+ return purgeAndInsert(readyAt, at, ranges);
}
private static <T extends Timestamp> ImmutableSortedMap<T, Ranges> purgeAndInsert(NavigableMap<T, Ranges> in, T insertAt, Ranges insert)
diff --git a/accord-core/src/main/java/accord/local/CommandStores.java b/accord-core/src/main/java/accord/local/CommandStores.java
index daf172af..9d19f871 100644
--- a/accord-core/src/main/java/accord/local/CommandStores.java
+++ b/accord-core/src/main/java/accord/local/CommandStores.java
@@ -27,12 +27,11 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.function.BiConsumer;
-import java.util.function.BiFunction;
import java.util.function.Consumer;
-import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
+import java.util.stream.Stream;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@@ -65,16 +64,14 @@ import accord.topology.Topology;
import accord.utils.IndexedQuadConsumer;
import accord.utils.IndexedRangeQuadConsumer;
import accord.utils.Invariants;
-import accord.utils.MapReduce;
-import accord.utils.MapReduceConsume;
import accord.utils.RandomSource;
import accord.utils.Reduce;
import accord.utils.SearchableRangeList;
import accord.utils.LargeBitSet;
-import accord.utils.TriFunction;
import accord.utils.UnhandledEnum;
import accord.utils.async.AsyncChain;
import accord.utils.async.AsyncChains;
+import accord.utils.async.AsyncResult;
import accord.utils.async.AsyncResults;
import accord.utils.async.Cancellable;
import org.agrona.collections.Hashing;
@@ -82,6 +79,9 @@ import org.agrona.collections.Int2IntHashMap;
import org.agrona.collections.Int2ObjectHashMap;
import static accord.api.ConfigurationService.EpochReady.done;
+import static accord.api.DataStore.FetchKind.Sync;
+import static accord.local.CommandStores.BootstrapRangeAction.NO_BOOTSTRAP;
+import static accord.local.CommandStores.BootstrapRangeAction.SAFE_BOOTSTRAP;
import static accord.primitives.Routables.Slice.Minimal;
import static accord.utils.Invariants.illegalState;
import static java.util.stream.Collectors.toList;
@@ -680,14 +680,6 @@ public abstract class CommandStores implements AsyncExecutorFactory
return current.local;
}
- public void forEach(BiConsumer<CommandStore, RangesForEpoch> forEach)
- {
- for (ShardHolder shard : current.shards)
- {
- forEach.accept(shard.store, shard.ranges);
- }
- }
-
public DataStore dataStore()
{
return supplier.store;
@@ -705,9 +697,31 @@ public abstract class CommandStores implements AsyncExecutorFactory
}
}
- protected boolean shouldBootstrap(Node node, Topology local, Topology newLocalTopology, Range add)
+ public enum BootstrapRangeAction
{
- return newLocalTopology.epoch() != 1;
+ NO_BOOTSTRAP, SAFE_BOOTSTRAP, UNSAFE_BOOTSTRAP
+ }
+
+ protected BootstrapRangeAction shouldBootstrap(Node node, Topology prevGlobal, Topology newLocalTopology, Range add)
+ {
+ if (newLocalTopology.epoch() == 1 || !prevGlobal.ranges().contains(add))
+ return NO_BOOTSTRAP;
+
+ return SAFE_BOOTSTRAP;
+ }
+
+ public AsyncResult<Void> rebootstrap(Node node)
+ {
+ List<AsyncResult<EpochReady>> results = new ArrayList<>();
+ Snapshot snapshot = current;
+ for (ShardHolder shard : snapshot.shards)
+ results.add(shard.store.startUnsafeBootstrap(node, shard.ranges.all(), snapshot.global.epoch(), Sync));
+ return AsyncResults.allOf(results).flatMap(list -> {
+ return AsyncChains.reduce(list.stream()
+ .flatMap(b -> Stream.of(b.reads.chain(), b.coordinate.chain()))
+ .collect(Collectors.toList()),
+ Reduce.toNull()).beginAsResult();
+ });
}
private synchronized TopologyUpdate updateTopology(Node node, Snapshot prev, Topology newTopology, boolean startSync)
@@ -767,11 +781,12 @@ public abstract class CommandStores implements AsyncExecutorFactory
ShardHolder shard = new ShardHolder(supplier.create(nextId++, updateHolder));
shard.ranges = rangesForEpoch;
- Map<Boolean, Ranges> partitioned = addRanges.partitioningBy(range -> shouldBootstrap(node, prev.global, newLocalTopology, range));
- if (partitioned.containsKey(false))
- bootstrapUpdates.add(shard.store.initialise(epoch, partitioned.get(false)));
- if (partitioned.containsKey(true))
- bootstrapUpdates.add(shard.store.bootstrapper(node, partitioned.get(true), newLocalTopology.epoch()));
+ Map<BootstrapRangeAction, Ranges> partitioned = addRanges.partitioningBy(range -> shouldBootstrap(node, prev.global, newLocalTopology, range));
+ for (Map.Entry<BootstrapRangeAction, Ranges> entry : partitioned.entrySet())
+ {
+ BootstrapRangeAction action = entry.getKey();
+ bootstrapUpdates.add(shard.store.bootstrapper(node, entry.getValue(), newLocalTopology.epoch(), action));
+ }
result.add(shard);
}
}
@@ -816,95 +831,62 @@ public abstract class CommandStores implements AsyncExecutorFactory
return false;
}
- public <R> R unsafeFoldLeft(R initial, BiFunction<R, CommandStore, R> f)
- {
- Snapshot snapshot = current;
- for (ShardHolder shard : snapshot.shards)
- initial = f.apply(initial, shard.store);
- return initial;
- }
-
- public AsyncChain<Void> forEach(PreLoadContext context, Consumer<SafeCommandStore> forEach)
- {
- List<AsyncChain<Void>> list = new ArrayList<>();
- Snapshot snapshot = current;
- for (ShardHolder shard : snapshot.shards)
- list.add(shard.store.chain(context, forEach));
- return AsyncChains.reduce(list, Reduce.toNull(), null);
- }
-
- public void forEachCommandStore(Consumer<CommandStore> forEach)
+ public void forAllUnsafe(Consumer<CommandStore> forEach)
{
Snapshot snapshot = current;
for (ShardHolder shard : snapshot.shards)
forEach.accept(shard.store);
}
- public AsyncChain<Void> ifLocal(PreLoadContext context, RoutingKey key, long minEpoch, long maxEpoch, Consumer<SafeCommandStore> forEach)
+ public AsyncChain<Void> forAll(String reason, Consumer<SafeCommandStore> forEach)
{
- return forEach(context, RoutingKeys.of(key), minEpoch, maxEpoch, forEach, false);
- }
-
- public AsyncChain<Void> forEach(PreLoadContext context, RoutingKey key, long minEpoch, long maxEpoch, Consumer<SafeCommandStore> forEach)
- {
- return forEach(context, RoutingKeys.of(key), minEpoch, maxEpoch, forEach, true);
- }
-
- public AsyncChain<Void> forEach(PreLoadContext context, Unseekables<?> keys, long minEpoch, long maxEpoch, Consumer<SafeCommandStore> forEach)
- {
- return forEach(context, keys, minEpoch, maxEpoch, forEach, true);
- }
-
- private AsyncChain<Void> forEach(PreLoadContext context, Unseekables<?> keys, long minEpoch, long maxEpoch, Consumer<SafeCommandStore> forEach, boolean matchesMultiple)
- {
- return this.mapReduce(context, keys, minEpoch, maxEpoch, new MapReduce<>()
+ return mapReduce(snapshot -> Stream.of(snapshot.shards).map(shard -> shard.store).iterator(), new MapReduceCommandStores<>(RoutingKeys.EMPTY)
{
+ @Override public Void reduce(Void o1, Void o2) { return null; }
+ @Override public TxnId primaryTxnId() { return null; }
+ @Override public String reason() { return reason; }
@Override
- public Void apply(SafeCommandStore in)
+ protected Void applyInternal(SafeCommandStore safeStore)
{
- forEach.accept(in);
+ forEach.accept(safeStore);
return null;
}
+ });
+ }
- @Override
- public Void reduce(Void o1, Void o2)
- {
- if (!matchesMultiple && minEpoch == maxEpoch)
- throw new IllegalStateException();
-
- return null;
- }
+ public AsyncChain<Void> forEach(String reason, Participants<?> participants, long minEpoch, long maxEpoch, Consumer<SafeCommandStore> forEach)
+ {
+ return forEach(reason, null, participants, minEpoch, maxEpoch, forEach);
+ }
- @Override
- public String toString()
- {
- return forEach.getClass().getName();
- }
- });
+ public AsyncChain<Void> forEach(String reason, TxnId txnId, Participants<?> participants, long minEpoch, long maxEpoch, Consumer<SafeCommandStore> forEach)
+ {
+ return forEach(reason, txnId, participants, LoadKeys.SYNC, LoadKeysFor.READ_WRITE, minEpoch, maxEpoch, forEach);
}
- public AsyncChain<Void> forEach(PreLoadContext context, StoreSelector selector, Consumer<SafeCommandStore> forEach)
+ public AsyncChain<Void> forEach(String reason, TxnId txnId, Participants<?> participants, LoadKeys loadKeys, LoadKeysFor loadKeysFor, long minEpoch, long maxEpoch, Consumer<SafeCommandStore> forEach)
{
- return this.mapReduce(context, selector, new MapReduce<>()
- {
+ return mapReduce(StoreFinder.selector(participants, minEpoch, maxEpoch), new MapReduceCommandStores<Participants<?>, Void>(participants)
+ {
+ @Override public Unseekables<?> keys() { return participants; }
+ @Override public LoadKeys loadKeys() { return loadKeys;}
+ @Override public LoadKeysFor loadKeysFor() { return loadKeysFor; }
+ @Override public Void reduce(Void o1, Void o2) { return null; }
+ @Override public TxnId primaryTxnId() { return txnId; }
+ @Override public String reason() { return reason; }
@Override
- public Void apply(SafeCommandStore in)
+ protected Void applyInternal(SafeCommandStore safeStore)
{
- forEach.accept(in);
+ forEach.accept(safeStore);
return null;
}
- @Override
- public Void reduce(Void o1, Void o2) { return null; }
- @Override public String toString() { return forEach.getClass().getName(); }
});
}
- /**
- * See {@link #mapReduceConsume(PreLoadContext, Unseekables, long, long, MapReduceConsume)}
- */
- public <O> Cancellable mapReduceConsume(PreLoadContext context, RoutingKey key, long minEpoch, long maxEpoch, MapReduceConsume<? super SafeCommandStore, O> mapReduceConsume)
+ public <O> Cancellable mapReduceConsume(long minEpoch, long maxEpoch, MapReduceConsumeCommandStores<?, O> mapReduceConsume)
{
- return mapReduceConsume(context, RoutingKeys.of(key), minEpoch, maxEpoch, mapReduceConsume);
+ AsyncChain<O> reduced = mapReduce(StoreFinder.selector(mapReduceConsume.scope, minEpoch, maxEpoch), mapReduceConsume);
+ return reduced.begin(mapReduceConsume);
}
/**
@@ -913,123 +895,38 @@ public abstract class CommandStores implements AsyncExecutorFactory
*
* Note that {@code reduce} and {@code accept} are invoked by only one thread, and never concurrently with {@code apply},
* so they do not require mutual exclusion.
- *
- * Implementations are expected to invoke {@link #mapReduceConsume(PreLoadContext, Unseekables, long, long, MapReduceConsume)}
*/
- protected <O> Cancellable mapReduceConsume(PreLoadContext context, Unseekables<?> keys, long minEpoch, long maxEpoch, MapReduceConsume<? super SafeCommandStore, O> mapReduceConsume)
+ public <O> Cancellable mapReduceConsume(StoreSelector selector, MapReduceConsumeCommandStores<?, O> mapReduceConsume)
{
- AsyncChain<O> reduced = mapReduce(context, keys, minEpoch, maxEpoch, mapReduceConsume);
+ AsyncChain<O> reduced = mapReduce(selector, mapReduceConsume);
return reduced.begin(mapReduceConsume);
}
- protected <O> Cancellable mapReduceConsume(PreLoadContext context, StoreSelector selector, MapReduceConsume<? super SafeCommandStore, O> mapReduceConsume)
+ public <O> Cancellable mapReduceConsume(IntStream commandStoreIds, MapReduceConsumeCommandStores<?, O> mapReduceConsume)
{
- AsyncChain<O> reduced = mapReduce(context, selector, mapReduceConsume);
+ AsyncChain<O> reduced = mapReduce(commandStoreIds, mapReduceConsume);
return reduced.begin(mapReduceConsume);
}
- public <O> Cancellable mapReduceConsume(PreLoadContext context, IntStream commandStoreIds, MapReduceConsume<? super SafeCommandStore, O> mapReduceConsume)
+ public <O> AsyncChain<O> mapReduce(IntStream commandStoreIds, MapReduceCommandStores<?, O> mapReduce)
{
- AsyncChain<O> reduced = mapReduce(context, commandStoreIds, mapReduceConsume);
- return reduced.begin(mapReduceConsume);
+ return mapReduce(snapshot -> commandStoreIds.mapToObj(snapshot::byId).iterator(), mapReduce);
}
- public <O> AsyncChain<O> mapReduce(PreLoadContext context, Unseekables<?> unseekables, long minEpoch, long maxEpoch, MapReduce<? super SafeCommandStore, O> mapReduce)
- {
- // TODO (desired): we shouldn't need to allocate a new lambda here
- return mapReduce(CommandStore::chain, context, mapReduce, mapReduce, unseekables, minEpoch, maxEpoch);
- }
-
- public <O> Cancellable mapReduceConsume(Unseekables<?> unseekables, long minEpoch, long maxEpoch, Function<? super CommandStore, AsyncChain<O>> map, Reduce<O, O> reduce, BiConsumer<? super O, Throwable> consume)
- {
- AsyncChain<O> reduced = mapReduce(unseekables, minEpoch, maxEpoch, map, reduce);
- return reduced.begin(consume);
- }
-
- public <O> AsyncChain<O> mapReduce(Unseekables<?> unseekables, long minEpoch, long maxEpoch, Function<? super CommandStore, AsyncChain<O>> map, Reduce<O, O> reducer)
- {
- return mapReduce((commandStore, i, f) -> f.apply(commandStore), null, map, reducer, unseekables, minEpoch, maxEpoch);
- }
-
- public <O, P1, P2> AsyncChain<O> mapReduce(TriFunction<CommandStore, P1, P2, AsyncChain<O>> applyMap, P1 p1, P2 p2, Reduce<O, O> reducer, Unseekables<?> unseekables, long minEpoch, long maxEpoch)
- {
- return mapReduce(StoreFinder.selector(unseekables, minEpoch, maxEpoch), applyMap, p1, p2, reducer);
- }
-
- public <O> AsyncChain<O> mapReduce(PreLoadContext context, StoreSelector selector, MapReduce<? super SafeCommandStore, O> mapReduce)
- {
- return mapReduce(selector, CommandStore::chain, context, mapReduce, mapReduce);
- }
-
- public <O, P1, P2> AsyncChain<O> mapReduce(StoreSelector selector, TriFunction<CommandStore, P1, P2, AsyncChain<O>> applyMap, P1 p1, P2 p2, Reduce<O, O> reducer)
+ public <O> AsyncChain<O> mapReduce(StoreSelector selector, MapReduceCommandStores<?, O> mapReduceConsume)
{
Snapshot snapshot = current;
Iterator<CommandStore> stores = selector.select(snapshot);
AsyncChain<O> chain = null;
while (stores.hasNext())
{
- AsyncChain<O> next = applyMap.apply(stores.next(), p1, p2);
+ AsyncChain<O> next = mapReduceConsume.applyAsync(stores.next());
if (next != null)
- chain = chain != null ? AsyncChains.reduce(chain, next, reducer) : next;
+ chain = chain != null ? AsyncChains.reduce(chain, next, mapReduceConsume) : next;
}
return chain == null ? AsyncChains.success(null) : chain;
}
- protected <O> AsyncChain<O> mapReduce(PreLoadContext context, IntStream commandStoreIds, MapReduce<? super SafeCommandStore, O> mapReduce)
- {
- return mapReduce(context, snapshot -> commandStoreIds.mapToObj(snapshot::byId).iterator(), mapReduce);
- }
-
- public <O> Cancellable mapReduceConsume(PreLoadContext context, MapReduceConsume<? super SafeCommandStore, O> mapReduceConsume)
- {
- AsyncChain<O> reduced = mapReduce(context, mapReduceConsume);
- return reduced.begin(mapReduceConsume);
- }
-
- protected <O> AsyncChain<O> mapReduce(PreLoadContext context, MapReduce<? super SafeCommandStore, O> mapReduce)
- {
- AsyncChain<O> chain = null;
- for (ShardHolder shardHolder : current.shards)
- {
- CommandStore commandStore = shardHolder.store;
- AsyncChain<O> next = commandStore.chain(context, mapReduce);
- chain = chain != null ? AsyncChains.reduce(chain, next, mapReduce) : next;
- }
- return chain == null ? AsyncChains.success(null) : chain;
- }
-
-
- public <O> AsyncChain<List<O>> map(PreLoadContext context, Function<? super SafeCommandStore, O> mapper)
- {
- ShardHolder[] shards = current.shards;
- List<AsyncChain<O>> results = new ArrayList<>(shards.length);
-
- for (ShardHolder shard : shards)
- results.add(shard.store.chain(context, mapper));
-
- return AsyncChains.allOf(results);
- }
-
- protected <O> AsyncChain<List<O>> map(PreLoadContext context, IntStream commandStoreIds, Function<? super SafeCommandStore, O> map)
- {
- // TODO (low priority, efficiency): avoid using an array, or use a scratch buffer
- int[] ids = commandStoreIds.toArray();
- if (ids.length == 1)
- {
- return forId(ids[0]).chain(context, map.andThen(Collections::singletonList));
- }
-
- List<AsyncChain<O>> list = new ArrayList<>(ids.length);
- for (int id : ids)
- {
- CommandStore commandStore = forId(id);
- AsyncChain<O> next = commandStore.chain(context, map);
- list.add(next);
- }
-
- return AsyncChains.allOf(list);
- }
-
/**
* Initialize topology from snapshot on boot.
*/
@@ -1053,6 +950,56 @@ public abstract class CommandStores implements AsyncExecutorFactory
loadSnapshot(new Snapshot(shards, update.global.forNode(supplier.node.id()).trim(), update.global));
}
+ public synchronized void resetTopology(Journal.TopologyUpdate update)
+ {
+ // TODO: assert
+ Snapshot current = this.current;
+ Invariants.require(update.global.epoch() == current.local.epoch());
+ ShardHolder[] shards = new ShardHolder[current.commandStores.size()];
+ int maxId = -1;
+ for (Map.Entry<Integer, RangesForEpoch> e : update.commandStores.entrySet())
+ {
+ int storeId = e.getKey();
+ RangesForEpoch ranges = e.getValue();
+ Invariants.require(ranges != null);
+ ShardHolder shard = new ShardHolder(current.byId(storeId), ranges);
+ EpochUpdateHolder holder = shard.store.epochUpdateHolder;
+ ranges.forEach(new BiConsumer<>()
+ {
+ RangesForEpoch accumulator = null;
+ Ranges prev = null;
+ public void accept(Long epoch, Ranges ranges)
+ {
+ if (accumulator == null)
+ accumulator = new RangesForEpoch(epoch, ranges);
+ else
+ accumulator = accumulator.withRanges(epoch, ranges);
+
+ Ranges additions = Ranges.EMPTY;
+ Ranges removals = Ranges.EMPTY;
+ if (prev != null)
+ {
+ additions = ranges.without(prev);
+ removals = prev.without(ranges);
+ }
+
+ if (!additions.isEmpty())
+ holder.add(epoch, accumulator, additions);
+ if (!removals.isEmpty())
+ holder.remove(epoch, accumulator, removals);
+ shard.store.unsafeUpdateRangesForEpoch();
+ prev = ranges;
+ }
+ });
+
+ shards[storeId] = shard;
+ maxId = Math.max(maxId, storeId);
+ }
+
+ nextId = maxId + 1;
+ loadSnapshot(new Snapshot(shards, current.local, current.global));
+ }
+
public synchronized Supplier<EpochReady> updateTopology(Node node, Topology newTopology, boolean startSync)
{
TopologyUpdate update = updateTopology(node, current, newTopology, startSync);
diff --git a/accord-core/src/main/java/accord/local/Commands.java b/accord-core/src/main/java/accord/local/Commands.java
index 8e059011..832f2e13 100644
--- a/accord-core/src/main/java/accord/local/Commands.java
+++ b/accord-core/src/main/java/accord/local/Commands.java
@@ -94,7 +94,7 @@ import static accord.local.RedundantStatus.Property.LOCALLY_APPLIED;
import static accord.local.RedundantStatus.Property.LOCALLY_DEFUNCT;
import static accord.local.RedundantStatus.Property.LOCALLY_REDUNDANT;
import static accord.local.RedundantStatus.Property.LOCALLY_SYNCED;
-import static accord.local.RedundantStatus.Property.PRE_BOOTSTRAP_OR_STALE;
+import static accord.local.RedundantStatus.Property.UNREADY;
import static accord.local.RedundantStatus.Property.SHARD_APPLIED;
import static accord.local.RedundantStatus.Property.WAS_OWNED;
import static accord.local.StoreParticipants.Filter.LOAD;
@@ -200,7 +200,7 @@ public class Commands
}
if (command.known().deps().hasProposedOrDecidedDeps()) participants = command.participants().supplement(participants);
- else participants = participants.filter(UPDATE, safeStore, txnId, null);
+ else participants = participants.filter(UPDATE, safeStore, txnId, command.executeAtIfKnown()); // executeAt may be known if PreCommitted (without Definition)
Validated validated = validate(safeStore, txnId, ballot, newSaveStatus, command, participants, participants.route(), txn, deps);
Invariants.require(validated.compareTo(INSUFFICIENT) > 0);
@@ -455,7 +455,7 @@ public class Commands
// information to execute in the eventual execution epoch (that they didn't know they needed when they were made stable)
participants = participants.supplement(route);
- participants = participants.filter(UPDATE, safeStore, txnId, null);
+ participants = participants.filter(UPDATE, safeStore, txnId, txnId);
Validated validated = validate(safeStore, txnId, null, SaveStatus.Stable, command, participants, route, txn, deps);
Invariants.require(validated.compareTo(INSUFFICIENT) > 0);
@@ -1097,6 +1097,7 @@ public class Commands
private static boolean validateSafeToCleanup(RedundantBefore redundantBefore, Command command, @Nonnull StoreParticipants participants, boolean force)
{
+ if (participants.stillTouches().isEmpty()) return true;
if (command.hasBeen(Applied)) return true;
if (!command.hasBeen(PreCommitted)) return true;
if (participants.route() == null) return true; // TODO (expected): tighten this e.g. with && participants.owns.isEmpty()
@@ -1110,7 +1111,8 @@ public class Commands
return true;
}
- if (status.all(SHARD_APPLIED, LOCALLY_REDUNDANT) || status.all(LOCALLY_DEFUNCT))
+ // TODO (required): should be an additional property so we can track correctly on merge...?
+ if (status.all(SHARD_APPLIED, LOCALLY_REDUNDANT))
return true;
if (force && participants.waitsOn() != null && participants.stillWaitsOn().isEmpty())
@@ -1119,8 +1121,10 @@ public class Commands
return false;
}
+ static int counter = 0;
public static boolean maybeCleanup(SafeCommandStore safeStore, SafeCommand safeCommand, Command command, @Nonnull StoreParticipants newParticipants)
{
+ ++counter;
StoreParticipants cleanupParticipants = newParticipants.filter(LOAD, safeStore, command.txnId(), command.executeAtIfKnown());
Cleanup cleanup = shouldCleanup(FULL, safeStore, command, cleanupParticipants);
if (cleanup == NO)
@@ -1362,7 +1366,7 @@ public class Commands
boolean remove = status.all(LOCALLY_REDUNDANT);
// TODO (required): consider this logic again, incl. whether it is even needed
if (remove && waitingSafe.txnId().isSyncPoint() && depId.isSyncPoint())
- remove = status.all(LOCALLY_SYNCED) || status.all(PRE_BOOTSTRAP_OR_STALE);
+ remove = status.all(LOCALLY_SYNCED) || status.all(UNREADY); // TODO (required): should be additional property for correct merge?
if (!remove)
return ifNotRedundant;
@@ -1493,6 +1497,8 @@ public class Commands
case UPDATE_TXN_KEEP_DEPS:
return upd.partialDeps();
case UPDATE_TXN_MERGE_DEPS:
+ if (newDeps == null)
+ return upd.partialDeps();
return upd.partialDeps().with(newDeps.intersecting(participants.stillTouches()));
case UPDATE_TXN_AND_DEPS:
return prepareNewDeps(participants, newDeps);
@@ -1582,7 +1588,7 @@ public class Commands
if (txnId.isSyncPoint() && expectKnown.is(DepsKnown))
{
Ranges missing = Ranges.ofSortedAndDeoverlapped(safeStore.redundantBefore().foldl(fullRoute, (Bounds b, List<Range> v, TxnId id) -> {
- if (b.endEpoch < Long.MAX_VALUE && !b.isLocallyRetiredOrPreBootstrap(id))
+ if (b.endEpoch < Long.MAX_VALUE && !b.isLocallyRetiredOrUnready(id))
v.add(b.range);
return v;
}, new ArrayList<>(), txnId).toArray(Range[]::new)).intersecting(fullRoute, Minimal).without(participants.stillWaitsOn());
@@ -1601,7 +1607,7 @@ public class Commands
return adding == null ? required.isEmpty() : adding.covers(required);
}
- private static <V> boolean containsAll(Deps adding, Participants<?> required)
+ private static boolean containsAll(Deps adding, Participants<?> required)
{
return adding == null ? required.isEmpty() : adding.covers(required);
}
diff --git a/accord-core/src/main/java/accord/utils/MapReduceConsume.java b/accord-core/src/main/java/accord/local/LogUnavailableException.java
similarity index 55%
copy from accord-core/src/main/java/accord/utils/MapReduceConsume.java
copy to accord-core/src/main/java/accord/local/LogUnavailableException.java
index 4b93737b..e7cf3c36 100644
--- a/accord-core/src/main/java/accord/utils/MapReduceConsume.java
+++ b/accord-core/src/main/java/accord/local/LogUnavailableException.java
@@ -16,22 +16,24 @@
* limitations under the License.
*/
-package accord.utils;
+package accord.local;
-import java.util.function.BiConsumer;
-import java.util.function.Consumer;
+import accord.utils.WrappableException;
-public interface MapReduceConsume<I, O> extends MapReduce<I, O>, BiConsumer<O, Throwable>
+public class LogUnavailableException extends RuntimeException implements WrappableException<LogUnavailableException>
{
- @Override
- void accept(O result, Throwable failure);
+ public LogUnavailableException()
+ {
+ }
- static <I> MapReduceConsume<I, Void> forEach(Consumer<I> forEach, BiConsumer<Object, Throwable> consume)
+ private LogUnavailableException(Throwable cause)
+ {
+ super(cause);
+ }
+
+ @Override
+ public LogUnavailableException wrap()
{
- return new MapReduceConsume<I, Void>() {
- @Override public void accept(Void result, Throwable failure) { consume.accept(result, failure); }
- @Override public Void apply(I in) { forEach.accept(in); return null; }
- @Override public Void reduce(Void o1, Void o2) {return null; }
- };
+ return new LogUnavailableException(this);
}
}
diff --git a/accord-core/src/main/java/accord/local/MapReduceCommandStores.java b/accord-core/src/main/java/accord/local/MapReduceCommandStores.java
new file mode 100644
index 00000000..089666d9
--- /dev/null
+++ b/accord-core/src/main/java/accord/local/MapReduceCommandStores.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package accord.local;
+
+import accord.primitives.Participants;
+import accord.utils.MapReduce;
+import accord.utils.async.AsyncChain;
+
+public abstract class MapReduceCommandStores<P extends Participants<?>, O> implements PreLoadContext, MapReduce<SafeCommandStore, O>
+{
+ public final P scope;
+
+ protected MapReduceCommandStores(P scope)
+ {
+ this.scope = scope;
+ }
+
+ public final O apply(SafeCommandStore safeStore)
+ {
+ if (refuses(safeStore))
+ return refuseInternal(safeStore);
+ return applyInternal(safeStore);
+ }
+
+ public final AsyncChain<O> applyAsync(CommandStore commandStore)
+ {
+ return applyAsyncInternal(commandStore);
+ }
+
+ AsyncChain<O> applyAsyncInternal(CommandStore commandStore)
+ {
+ return commandStore.chain(this, this);
+ }
+
+ protected boolean supportsPartialRefusal()
+ {
+ return false;
+ }
+
+ private boolean refuses(SafeCommandStore safeStore)
+ {
+ if (supportsPartialRefusal()) return safeStore.refusesAllOwnedOf(scope);
+ else return safeStore.refusesAnyOf(scope);
+ }
+
+ protected O refuseInternal(SafeCommandStore safeStore)
+ {
+ throw new LogUnavailableException();
+ }
+
+ protected abstract O applyInternal(SafeCommandStore safeStore);
+}
diff --git a/accord-core/src/main/java/accord/local/MapReduceConsumeCommandStores.java b/accord-core/src/main/java/accord/local/MapReduceConsumeCommandStores.java
new file mode 100644
index 00000000..7c8472c6
--- /dev/null
+++ b/accord-core/src/main/java/accord/local/MapReduceConsumeCommandStores.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package accord.local;
+
+import java.util.function.Function;
+import javax.annotation.Nullable;
+
+import accord.primitives.Participants;
+import accord.primitives.TxnId;
+import accord.utils.MapReduceConsume;
+import accord.utils.async.AsyncChain;
+
+public abstract class MapReduceConsumeCommandStores<P extends Participants<?>, O> extends MapReduceCommandStores<P, O> implements MapReduceConsume<SafeCommandStore, O>
+{
+ protected MapReduceConsumeCommandStores(P scope)
+ {
+ super(scope);
+ }
+
+ public MapReduceConsumeCommandStores<P, O> overrideWithSynchronousApply(Function<? super CommandStore, AsyncChain<O>> apply)
+ {
+ return new Delegate<>(this)
+ {
+ @Override
+ AsyncChain<O> applyAsyncInternal(CommandStore commandStore)
+ {
+ return apply.apply(commandStore);
+ }
+
+ @Override
+ protected O applyInternal(SafeCommandStore safeStore)
+ {
+ throw new IllegalStateException();
+ }
+ };
+ }
+
+ public static class Delegate<P extends Participants<?>, O> extends MapReduceConsumeCommandStores<P, O>
+ {
+ final MapReduceConsumeCommandStores<P, O> delegate;
+
+ public Delegate(MapReduceConsumeCommandStores<P, O> delegate)
+ {
+ super(delegate.scope);
+ this.delegate = delegate;
+ }
+
+ @Override
+ protected O applyInternal(SafeCommandStore safeStore)
+ {
+ return delegate.applyInternal(safeStore);
+ }
+
+ @Nullable
+ @Override
+ public TxnId primaryTxnId()
+ {
+ return delegate.primaryTxnId();
+ }
+
+ @Override
+ public String reason()
+ {
+ return delegate.reason();
+ }
+
+ @Override
+ public void accept(O result, Throwable failure)
+ {
+ delegate.accept(result, failure);
+ }
+
+ @Override
+ public O reduce(O o1, O o2)
+ {
+ return delegate.reduce(o1, o2);
+ }
+ }
+}
diff --git a/accord-core/src/main/java/accord/local/Node.java b/accord-core/src/main/java/accord/local/Node.java
index 61d66b4b..3cfc3844 100644
--- a/accord-core/src/main/java/accord/local/Node.java
+++ b/accord-core/src/main/java/accord/local/Node.java
@@ -27,9 +27,9 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicLongFieldUpdater;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.BiConsumer;
-import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.Supplier;
+import java.util.stream.Stream;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@@ -68,29 +68,26 @@ import accord.messages.Callback;
import accord.messages.Reply;
import accord.messages.ReplyContext;
import accord.messages.Request;
-import accord.messages.TxnRequest;
import accord.primitives.Ballot;
import accord.primitives.EpochSupplier;
import accord.primitives.FullRoute;
import accord.primitives.Ranges;
import accord.primitives.Routable.Domain;
import accord.primitives.Routables;
+import accord.primitives.RoutingKeys;
import accord.primitives.Seekables;
import accord.primitives.Timestamp;
import accord.primitives.Txn;
import accord.primitives.TxnId;
import accord.primitives.TxnId.Cardinality;
-import accord.primitives.Unseekables;
import accord.topology.Shard;
import accord.topology.Topology;
import accord.topology.TopologyManager;
import accord.utils.DeterministicSet;
import accord.utils.Invariants;
-import accord.utils.MapReduceConsume;
import accord.utils.PersistentField;
import accord.utils.PersistentField.Persister;
import accord.utils.RandomSource;
-import accord.utils.Reduce;
import accord.utils.SortedList;
import accord.utils.SortedListMap;
import accord.utils.async.AsyncChain;
@@ -224,7 +221,7 @@ public class Node implements ConfigurationService.Listener, NodeCommandStoreServ
this.commandStores = factory.create(this, agent, dataSupplier.get(), random.fork(), journal, shardDistributor, progressLogFactory.apply(this), localListenersFactory.apply(this));
this.durabilityService = new DurabilityService(this);
// TODO (desired): make frequency configurable
- scheduler.recurring(() -> commandStores.forEachCommandStore(store -> store.progressLog.maybeNotify()), 1, SECONDS);
+ scheduler.recurring(() -> commandStores.forAllUnsafe(store -> store.progressLog.maybeNotify()), 1, SECONDS);
scheduler.recurring(timeouts::maybeNotify, 100, MILLISECONDS);
configService.registerListener(this);
}
@@ -556,51 +553,6 @@ public class Node implements ConfigurationService.Listener, NodeCommandStoreServ
return time.elapsed(timeUnit);
}
- public AsyncChain<Void> forEachLocal(PreLoadContext context, Unseekables<?> unseekables, long minEpoch, long maxEpoch, Consumer<SafeCommandStore> forEach)
- {
- return commandStores.forEach(context, unseekables, minEpoch, maxEpoch, forEach);
- }
-
- public AsyncChain<Void> forEachLocal(PreLoadContext context, StoreSelector selector, Consumer<SafeCommandStore> forEach)
- {
- return commandStores.forEach(context, selector, forEach);
- }
-
- public <T> Cancellable mapReduceConsumeLocal(TxnRequest<?> request, long minEpoch, long maxEpoch, MapReduceConsume<SafeCommandStore, T> mapReduceConsume)
- {
- return commandStores.mapReduceConsume(request, request.scope(), minEpoch, maxEpoch, mapReduceConsume);
- }
-
- public <T> Cancellable mapReduceConsumeLocal(Unseekables<?> keys, long minEpoch, long maxEpoch, Function<? super CommandStore, AsyncChain<T>> map, Reduce<T, T> reduce, BiConsumer<? super T, Throwable> consume)
- {
- return commandStores.mapReduceConsume(keys, minEpoch, maxEpoch, map, reduce, consume);
- }
-
- public <T> Cancellable mapReduceConsumeLocal(PreLoadContext context, RoutingKey key, long atEpoch, MapReduceConsume<SafeCommandStore, T> mapReduceConsume)
- {
- return mapReduceConsumeLocal(context, key, atEpoch, atEpoch, mapReduceConsume);
- }
-
- public <T> Cancellable mapReduceConsumeLocal(PreLoadContext context, RoutingKey key, long minEpoch, long maxEpoch, MapReduceConsume<SafeCommandStore, T> mapReduceConsume)
- {
- return commandStores.mapReduceConsume(context, key, minEpoch, maxEpoch, mapReduceConsume);
- }
-
- public <T> Cancellable mapReduceConsumeLocal(PreLoadContext context, Unseekables<?> keys, long minEpoch, long maxEpoch, MapReduceConsume<SafeCommandStore, T> mapReduceConsume)
- {
- return commandStores.mapReduceConsume(context, keys, minEpoch, maxEpoch, mapReduceConsume);
- }
-
- public <T> Cancellable mapReduceConsumeLocal(PreLoadContext context, StoreSelector selector, MapReduceConsume<SafeCommandStore, T> mapReduceConsume)
- {
- return commandStores.mapReduceConsume(context, selector, mapReduceConsume);
- }
-
- public <T> Cancellable mapReduceConsumeAllLocal(PreLoadContext context, MapReduceConsume<SafeCommandStore, T> mapReduceConsume)
- {
- return commandStores.mapReduceConsume(context, mapReduceConsume);
- }
-
// send to every node besides ourselves
public void send(Topology topology, Request send)
{
@@ -882,7 +834,19 @@ public class Node implements ConfigurationService.Listener, NodeCommandStoreServ
public AsyncChain<Void> updateMinHlc(long minHlc)
{
- return commandStores().forEach((PreLoadContext.Empty)() -> "Update Accord minHlc", safeStore -> safeStore.commandStore().updateMinHlc(minHlc));
+ // TODO (required): command stores that are not ready due to bootstrap need to refresh their min HLC on bootstrap completion
+ StoreSelector selector = snapshot -> Stream.of(snapshot.shards).map(sh -> sh.store).iterator();
+ return commandStores().mapReduce(selector, new MapReduceCommandStores<>(RoutingKeys.EMPTY)
+ {
+ @Override public Void reduce(Void o1, Void o2) { return null; }
+ @Override public TxnId primaryTxnId() { return null; }
+ @Override public String reason() { return "Update Min HLC"; }
+ @Override protected Void applyInternal(SafeCommandStore safeStore)
+ {
+ safeStore.commandStore().updateMinHlc(minHlc);
+ return null;
+ }
+ });
}
public Scheduler scheduler()
diff --git a/accord-core/src/main/java/accord/local/RedundantBefore.java b/accord-core/src/main/java/accord/local/RedundantBefore.java
index 078d5895..d153b9ed 100644
--- a/accord-core/src/main/java/accord/local/RedundantBefore.java
+++ b/accord-core/src/main/java/accord/local/RedundantBefore.java
@@ -58,7 +58,6 @@ import static accord.api.ProtocolModifiers.Toggles.requiresUniqueHlcs;
import static accord.local.RedundantStatus.Coverage.SOME;
import static accord.local.RedundantStatus.ONLY_LE_MASK;
import static accord.local.RedundantStatus.NOT_OWNED_ONLY;
-import static accord.local.RedundantStatus.PRE_BOOTSTRAP_OR_STALE_ONLY;
import static accord.local.RedundantStatus.Property.GC_BEFORE;
import static accord.local.RedundantStatus.Property.LOCALLY_APPLIED;
import static accord.local.RedundantStatus.Property.LOCALLY_DEFUNCT;
@@ -67,21 +66,20 @@ import static accord.local.RedundantStatus.Property.LOCALLY_DURABLE_TO_DATA_STOR
import static accord.local.RedundantStatus.Property.LOCALLY_REDUNDANT;
import static accord.local.RedundantStatus.Property.LOCALLY_SYNCED;
import static accord.local.RedundantStatus.Property.LOCALLY_WITNESSED;
-import static accord.local.RedundantStatus.Property.PRE_BOOTSTRAP;
-import static accord.local.RedundantStatus.Property.PRE_BOOTSTRAP_OR_STALE;
+import static accord.local.RedundantStatus.Property.UNREADY;
import static accord.local.RedundantStatus.Property.SHARD_APPLIED;
import static accord.local.RedundantStatus.WAS_OWNED_SYNCED;
import static accord.local.RedundantStatus.WAS_OWNED_ONLY;
import static accord.local.RedundantStatus.WAS_OWNED_RETIRED;
import static accord.local.RedundantStatus.addHistory;
import static accord.local.RedundantStatus.any;
-import static accord.local.RedundantStatus.mask;
+import static accord.local.RedundantStatus.shiftedMask;
import static accord.local.RedundantStatus.matchesMask;
import static accord.local.RedundantStatus.toAll;
import static accord.primitives.Routables.Slice.Minimal;
import static accord.primitives.Timestamp.Flag.SHARD_BOUND;
import static accord.utils.ArrayBuffers.cachedAny;
-import static accord.utils.ArrayBuffers.cachedShorts;
+import static accord.utils.ArrayBuffers.cachedInts;
import static accord.utils.Invariants.illegalState;
import static accord.utils.Invariants.require;
import static accord.utils.Invariants.requireStrictlyOrdered;
@@ -105,16 +103,15 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Bounds>
{
// start inclusive, end exclusive
public final long startEpoch, endEpoch;
- // TODO (required): this should probably be preBootstrapOrStale OR WE SHOULD REVISIT (e.g. HAS_CONSENSUS_LOG)
- public final TxnId bootstrappedAt;
+ public final TxnId readyAt;
public final TxnId gcBefore;
public final TxnId locallyAppliedBefore;
- public QuickBounds(long startEpoch, long endEpoch, TxnId bootstrappedAt, TxnId gcBefore, TxnId locallyAppliedBefore)
+ public QuickBounds(long startEpoch, long endEpoch, TxnId readyAt, TxnId gcBefore, TxnId locallyAppliedBefore)
{
this.startEpoch = startEpoch;
this.endEpoch = endEpoch;
- this.bootstrappedAt = bootstrappedAt;
+ this.readyAt = readyAt;
this.gcBefore = gcBefore;
this.locallyAppliedBefore = locallyAppliedBefore;
}
@@ -123,44 +120,44 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Bounds>
{
if (startEpoch == this.startEpoch && endEpoch == this.endEpoch)
return this;
- return new QuickBounds(startEpoch, endEpoch, bootstrappedAt, gcBefore, locallyAppliedBefore);
+ return new QuickBounds(startEpoch, endEpoch, readyAt, gcBefore, locallyAppliedBefore);
}
- public QuickBounds withGcBeforeBeforeAtLeast(TxnId newGcBefore)
+ public QuickBounds withReadyAtLeast(TxnId newReadyAt)
{
- if (newGcBefore.equals(this.gcBefore))
+ if (newReadyAt.equals(readyAt))
return this;
- return new QuickBounds(startEpoch, endEpoch, bootstrappedAt, newGcBefore, locallyAppliedBefore);
+ return new QuickBounds(startEpoch, endEpoch, newReadyAt, gcBefore, locallyAppliedBefore);
}
- public QuickBounds withBootstrappedAtLeast(TxnId newBootstrappedAt)
+ public QuickBounds withLocallyAppliedAtLeast(TxnId newLocallyAppliedBefore)
{
- if (newBootstrappedAt.equals(bootstrappedAt))
+ if (newLocallyAppliedBefore.equals(locallyAppliedBefore))
return this;
- return new QuickBounds(startEpoch, endEpoch, newBootstrappedAt, gcBefore, locallyAppliedBefore);
+ return new QuickBounds(startEpoch, endEpoch, readyAt, gcBefore, newLocallyAppliedBefore);
}
- public QuickBounds withLocallyAppliedAtLeast(TxnId newLocallyAppliedBefore)
+ public QuickBounds withGcBeforeBeforeAtLeast(TxnId newGcBefore)
{
- if (newLocallyAppliedBefore.equals(locallyAppliedBefore))
+ if (newGcBefore.equals(this.gcBefore))
return this;
- return new QuickBounds(startEpoch, endEpoch, bootstrappedAt, gcBefore, newLocallyAppliedBefore);
+ return new QuickBounds(startEpoch, endEpoch, readyAt, newGcBefore, locallyAppliedBefore);
}
}
public static class Bounds extends QuickBounds
{
- public static final Bounds NONE = new Bounds(null, Long.MIN_VALUE, Long.MAX_VALUE, TxnId.NO_TXNIDS, new short[0], null);
+ public static final Bounds NONE = new Bounds(null, Long.MIN_VALUE, Long.MAX_VALUE, TxnId.NO_TXNIDS, new int[0], null);
// TODO (desired): we don't need to maintain this now, can migrate to ReducingRangeMap.foldWithBounds
public final Range range;
- // TODO (expected): we need to eventually support GCing PRE_BOOTSTRAP bounds
+ // TODO (expected): we need to eventually support GCing UNREADY bounds
// once we know storage layer has fully expunged earlier TxnId
// OR we may be able to safely overwrite them with some better invariants and adequate testing
public final TxnId[] bounds;
// two entries per bound, first for equality (LE) matches, second for inequality (LT) matches
- private final short[] statuses;
+ private final int[] statuses;
private transient final long maxBoundEpoch, maxBoundHlc;
public transient final TxnId depBound;
@@ -172,19 +169,19 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Bounds>
* See also {@link SafeCommandStore#safeToReadAt()}.
*/
public final @Nullable Timestamp staleUntilAtLeast;
- private transient final RedundantStatus noMatch;
private transient RedundantStatus last = RedundantStatus.NONE;
- public Bounds(Range range, long startEpoch, long endEpoch, TxnId[] bounds, short[] statuses, @Nullable Timestamp staleUntilAtLeast)
+ public Bounds(Range range, long startEpoch, long endEpoch, TxnId[] bounds, int[] statuses, @Nullable Timestamp staleUntilAtLeast)
{
- super(startEpoch, endEpoch, maxBound(bounds, statuses, PRE_BOOTSTRAP), maxBound(bounds, statuses, GC_BEFORE),
+ super(startEpoch, endEpoch,
+ maxBound(bounds, statuses, UNREADY),
+ maxBound(bounds, statuses, GC_BEFORE),
maxBound(bounds, statuses, LOCALLY_APPLIED));
Invariants.require(statuses.length == bounds.length * 2);
this.range = range;
this.bounds = bounds;
this.statuses = statuses;
this.staleUntilAtLeast = staleUntilAtLeast;
- this.noMatch = staleUntilAtLeast == null ? RedundantStatus.NONE : PRE_BOOTSTRAP_OR_STALE_ONLY;
this.maxBoundEpoch = bounds.length == 0 ? 0 : bounds[0].epoch();
this.maxBoundHlc = bounds.length == 0 ? 0 : bounds[0].hlc();
this.depBound = depBound(bounds, statuses);
@@ -193,9 +190,9 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Bounds>
require(isShardBound(gcBefore) || isMinBound(gcBefore));
}
- public final int status(int index)
+ public final long status(int index)
{
- return statuses[index] & 0xFFFF;
+ return statuses[index] & 0xFFFFFFFFL;
}
public static Bounds create(Range range, TxnId bound, SomeStatus status, @Nullable Timestamp staleUntilAtLeast)
@@ -205,12 +202,12 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Bounds>
public static Bounds create(Range range, long startEpoch, long endEpoch, TxnId bound, SomeStatus status, @Nullable Timestamp staleUntilAtLeast)
{
- return new Bounds(range, startEpoch, endEpoch, new TxnId[] { bound }, new short[] { (short) (status.encoded & ONLY_LE_MASK), (short)status.encoded }, staleUntilAtLeast);
+ return new Bounds(range, startEpoch, endEpoch, new TxnId[] { bound }, new int[] { (int) (status.encoded & ONLY_LE_MASK), (int)status.encoded }, staleUntilAtLeast);
}
- private static TxnId depBound(TxnId[] bounds, short[] statuses)
+ private static TxnId depBound(TxnId[] bounds, int[] statuses)
{
- TxnId depBound = maxBound(bounds, statuses, mask(SHARD_APPLIED, SOME) | mask(LOCALLY_APPLIED, SOME));
+ TxnId depBound = maxBound(bounds, statuses, shiftedMask(SHARD_APPLIED, SOME) | shiftedMask(LOCALLY_APPLIED, SOME));
if (depBound.equals(TxnId.NONE))
return null;
return depBound.addFlag(SHARD_BOUND);
@@ -259,19 +256,19 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Bounds>
long startEpoch = Long.max(cur.startEpoch, add.startEpoch);
long endEpoch = Long.min(cur.endEpoch, add.endEpoch);
TxnId[] mergedBounds;
- short[] mergedStatuses;
+ int[] mergedStatuses;
{
Object[] boundBuf = null;
- short[] statusBuf = null;
+ int[] statusBuf = null;
int mergedCount = 0;
- int prevLtStatus = staleUntilAtLeast == null ? 0 : PRE_BOOTSTRAP_OR_STALE_ONLY.encodedPart(SOME);
- int prevExistingLtStatus = prevLtStatus; // we don't apply PRE_BOOTSTRAP_MERGE_MASK as this already applied
+ long prevLtStatus = 0;
+ long prevExistingLtStatus = prevLtStatus; // we don't apply UNREADY_MERGE_MASK as this already applied
int i = 0, j = 0;
while (i < cur.bounds.length || j < add.bounds.length)
{
int c = i == cur.bounds.length ? -1 : j == add.bounds.length ? 1 : cur.bounds[i].compareTo(add.bounds[j]);
TxnId nextBound;
- int leStatus, ltStatus;
+ long leStatus, ltStatus;
if (c > 0)
{
nextBound = cur.bounds[i];
@@ -295,18 +292,18 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Bounds>
++j;
}
- // we keep the start/end bound of an equal pre-bootstrap run, so that we correctly apply Property.mergeWithPreBootstrap
+ // we keep the start/end bound of an equal pre-bootstrap run, so that we correctly apply Property.mergeWithUnready
if (leStatus == ltStatus && ltStatus == prevLtStatus)
{
- if (!any(prevLtStatus, PRE_BOOTSTRAP_OR_STALE))
+ if (!any(prevLtStatus, UNREADY))
continue;
if (mergedCount >= 2)
{
- short[] prev = statusBuf != null ? statusBuf : cur.statuses;
- int prev2LtStatus = 0xFFFF & prev[mergedCount*2 - 3];
- int prevLeStatus = 0xFFFF & prev[mergedCount*2 - 2];
- Invariants.require(prevLtStatus == (0xFFFF & prev[mergedCount*2 - 1]));
+ int[] prev = statusBuf != null ? statusBuf : cur.statuses;
+ long prev2LtStatus = 0xFFFFFFFFL & prev[mergedCount*2 - 3];
+ long prevLeStatus = 0xFFFFFFFFL & prev[mergedCount*2 - 2];
+ Invariants.require(prevLtStatus == (0xFFFFFFFFL & prev[mergedCount*2 - 1]));
if (prevLtStatus == prev2LtStatus && prevLeStatus == prev2LtStatus)
--mergedCount;
}
@@ -323,13 +320,13 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Bounds>
}
boundBuf = cachedAny().get(cur.bounds.length + add.bounds.length);
- statusBuf = cachedShorts().getShorts((cur.bounds.length + add.bounds.length) * 2);
+ statusBuf = cachedInts().getInts((cur.bounds.length + add.bounds.length) * 2);
System.arraycopy(cur.bounds, 0, boundBuf, 0, mergedCount);
System.arraycopy(cur.statuses, 0, statusBuf, 0, mergedCount*2);
}
boundBuf[mergedCount] = nextBound;
- statusBuf[mergedCount*2] = (short) leStatus;
- statusBuf[mergedCount*2 + 1] = (short) ltStatus;
+ statusBuf[mergedCount*2] = (int) leStatus;
+ statusBuf[mergedCount*2 + 1] = (int) ltStatus;
++mergedCount;
prevLtStatus = ltStatus;
}
@@ -342,22 +339,22 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Bounds>
else
{
mergedBounds = new TxnId[mergedCount];
- mergedStatuses = new short[mergedCount*2];
+ mergedStatuses = new int[mergedCount*2];
System.arraycopy(boundBuf, 0, mergedBounds, 0, mergedCount);
System.arraycopy(statusBuf, 0, mergedStatuses, 0, mergedCount*2);
cachedAny().forceDiscard(boundBuf, mergedCount);
- cachedShorts().forceDiscard(statusBuf);
+ cachedInts().forceDiscard(statusBuf);
}
}
return new Bounds(range, startEpoch, endEpoch, mergedBounds, mergedStatuses, staleUntilAtLeast);
}
- private static Timestamp maybeClearStaleUntilAtLeast(Timestamp staleUntilAtLeast, TxnId[] bounds, short[] statuses)
+ private static Timestamp maybeClearStaleUntilAtLeast(Timestamp staleUntilAtLeast, TxnId[] bounds, int[] statuses)
{
for (int i = 0 ; staleUntilAtLeast != null && i < bounds.length && bounds[i].compareTo(staleUntilAtLeast) > 0 ; ++i)
{
- if (any(statuses[i], PRE_BOOTSTRAP))
+ if (any(statuses[i], UNREADY))
staleUntilAtLeast = null;
}
return staleUntilAtLeast;
@@ -366,7 +363,7 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Bounds>
public Bounds with(TxnId newBound, SomeStatus addStatus)
{
// TODO (desired): introduce special-cased faster merge for adding a single value
- return merge(range, this, new Bounds(range, Long.MIN_VALUE, Long.MAX_VALUE, new TxnId[] { newBound }, new short[] { (short)addStatus.encoded }, null));
+ return merge(range, this, new Bounds(range, Long.MIN_VALUE, Long.MAX_VALUE, new TxnId[] { newBound }, new int[] { (int)addStatus.encoded }, null));
}
@VisibleForImplementation
@@ -391,10 +388,7 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Bounds>
if (a.compareLessEqual != b.compareLessEqual)
return is(txnId, a) && is(txnId, b);
- int propertyMask = mask(a, SOME) | mask(b, SOME);
- if (matchesMask(noMatch.encoded, propertyMask))
- return true;
-
+ long propertyMask = shiftedMask(a, SOME) | shiftedMask(b, SOME);
if (noBoundMatches(txnId))
return false;
@@ -406,9 +400,9 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Bounds>
{
Invariants.require(test != GC_BEFORE);
if (noBoundMatches(txnId))
- return noMatch.any(test);
+ return false;
- if (test.mergeWithPreBootstrapOrStale)
+ if (test.mergeWithUnready)
return txnId.compareTo(bound(test)) <= test.compareLessEqual;
int i = findStatusIndexInternal(txnId);
@@ -418,16 +412,16 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Bounds>
TxnId bound(Property property)
{
Invariants.require(property != GC_BEFORE);
- Invariants.require(property.mergeWithPreBootstrapOrStale);
+ Invariants.require(property.mergeWithUnready);
return maxBound(property);
}
TxnId bound(Property a, Property b)
{
Invariants.require(a != GC_BEFORE);
- Invariants.require(a.mergeWithPreBootstrapOrStale);
+ Invariants.require(a.mergeWithUnready);
Invariants.require(b != GC_BEFORE);
- Invariants.require(b.mergeWithPreBootstrapOrStale);
+ Invariants.require(b.mergeWithUnready);
return maxBoundBoth(a, b);
}
@@ -464,15 +458,15 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Bounds>
public TxnId maxBoundBoth(Property a, Property b)
{
- return maxBound(bounds, statuses, mask(a, SOME) | mask(b, SOME));
+ return maxBound(bounds, statuses, shiftedMask(a, SOME) | shiftedMask(b, SOME));
}
- static TxnId maxBound(TxnId[] bounds, short[] statuses, Property property)
+ static TxnId maxBound(TxnId[] bounds, int[] statuses, Property property)
{
- return maxBound(bounds, statuses, mask(property, SOME));
+ return maxBound(bounds, statuses, shiftedMask(property, SOME));
}
- static TxnId maxBound(TxnId[] bounds, short[] statuses, int propertyMask)
+ static TxnId maxBound(TxnId[] bounds, int[] statuses, long propertyMask)
{
// we ignore LT/LE, as this should be applied by the caller as part of comparing maxBound
for (int i = 1; i < statuses.length ; i += 2)
@@ -508,7 +502,7 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Bounds>
if (bounds == null)
return safeToRead;
- if (bootstrapAt.compareTo(bounds.maxBootstrappedAt()) < 0 || (bounds.staleUntilAtLeast != null && bootstrapAt.compareTo(bounds.staleUntilAtLeast) < 0))
+ if (bootstrapAt.compareTo(bounds.maxReadyAt()) < 0 || (bounds.staleUntilAtLeast != null && bootstrapAt.compareTo(bounds.staleUntilAtLeast) < 0))
return safeToRead.without(Ranges.of(bounds.range));
return safeToRead;
@@ -530,43 +524,42 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Bounds>
return TxnId.nonNullOrMax(max, get.apply(bounds));
}
- static Participants<?> withoutStaleOrPreBootstrap(Bounds bounds, @Nonnull Participants<?> execute, TxnId txnId, @Nullable EpochSupplier executeAt)
+ static Participants<?> withoutLostAtExecutionOrUnready(Bounds bounds, @Nonnull Participants<?> execute, TxnId txnId, EpochSupplier executeAt)
{
if (bounds == null)
return execute;
- Invariants.require(txnId.isSyncPoint() || (executeAt == null ? !bounds.outOfBounds(txnId) : !bounds.outOfBounds(txnId, executeAt)));
- if (bounds.is(txnId, PRE_BOOTSTRAP_OR_STALE))
+ if (bounds.endEpoch <= executeAt.epoch() || bounds.is(txnId, UNREADY))
return execute.without(Ranges.of(bounds.range));
return execute;
}
- static Participants<?> withoutStaleOrPreBootstrapOrLocallyRetired(Bounds bounds, @Nonnull Participants<?> execute, TxnId txnId)
+ static Participants<?> withoutUnreadyOrLocallyRetired(Bounds bounds, @Nonnull Participants<?> execute, TxnId txnId)
{
if (bounds == null)
return execute;
- if (bounds.is(txnId, PRE_BOOTSTRAP_OR_STALE) || bounds.isLocallyRetired())
+ if (bounds.is(txnId, UNREADY) || bounds.isLocallyRetired())
return execute.without(Ranges.of(bounds.range));
return execute;
}
- static Participants<?> withoutRedundantAnd_StaleOrPreBootstrap(Bounds bounds, @Nonnull Participants<?> execute, TxnId txnId, @Nullable Timestamp executeAt)
+ static Participants<?> withoutRedundantAnd_Unready(Bounds bounds, @Nonnull Participants<?> execute, TxnId txnId, @Nullable Timestamp executeAt)
{
if (bounds == null)
return execute;
boolean outOfBounds = executeAt == null ? bounds.outOfBounds(txnId) : bounds.outOfBounds(txnId, executeAt);
- Invariants.expect(!outOfBounds, "Trying to apply withoutRedundantAnd_StaleOrPreBootstrap to %s for a range we don't own (%s), suggesting we computed ownership without an up-to-date epoch", txnId, bounds);
- if (outOfBounds || bounds.is(txnId, SHARD_APPLIED, PRE_BOOTSTRAP_OR_STALE))
+ Invariants.expect(!outOfBounds, "Trying to apply withoutRedundantAnd_Unready to %s for a range we don't own (%s), suggesting we computed ownership without an up-to-date epoch", txnId, bounds);
+ if (outOfBounds || bounds.is(txnId, SHARD_APPLIED, UNREADY))
return execute.without(Ranges.of(bounds.range));
return execute;
}
- static Participants<?> withoutRedundantAnd_StaleOrPreBootstrapOrRetiredOrNotOwned(Bounds bounds, @Nonnull Participants<?> execute, TxnId txnId, @Nullable Timestamp executeAtIfKnown)
+ static Participants<?> withoutRedundantAnd_UnreadyOrRetiredOrNotOwned(Bounds bounds, @Nonnull Participants<?> execute, TxnId txnId, @Nullable Timestamp executeAtIfKnown)
{
if (bounds == null)
return execute;
@@ -575,7 +568,7 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Bounds>
if (bounds.is(txnId, SHARD_APPLIED)
&& ((bounds.endEpoch <= txnId.epoch() && bounds.isLocallyRetired())
|| (executeAtIfKnown != null && executeAtIfKnown.epoch() < bounds.startEpoch)
- || bounds.is(txnId, PRE_BOOTSTRAP_OR_STALE)))
+ || bounds.is(txnId, UNREADY)))
return execute.without(Ranges.of(bounds.range));
return execute;
@@ -647,15 +640,15 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Bounds>
return notRetired.without(Ranges.of(bounds.range));
}
- static Ranges withoutPreBootstrap(Bounds bounds, @Nonnull Ranges notPreBootstrap, TxnId txnId, Object ignore)
+ static Ranges withoutUnreadyOrLost(Bounds bounds, @Nonnull Ranges notUnready, TxnId txnId, Object ignore)
{
if (bounds == null)
- return notPreBootstrap;
+ return notUnready;
- if (bounds.is(txnId, PRE_BOOTSTRAP))
- return notPreBootstrap.without(Ranges.of(bounds.range));
+ if (bounds.is(txnId, UNREADY))
+ return notUnready.without(Ranges.of(bounds.range));
- return notPreBootstrap;
+ return notUnready;
}
RedundantStatus get(TxnId txnId, @Nullable Timestamp applyAtIfKnown)
@@ -673,9 +666,9 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Bounds>
{
int i = findStatusIndex(txnId);
if (i < 0)
- return noMatch;
+ return RedundantStatus.NONE;
- int status = statuses[i] & 0xFFFF;
+ long status = status(i);
if (any(status, GC_BEFORE))
{
if (requiresUniqueHlcs() && txnId.isWrite())
@@ -684,7 +677,7 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Bounds>
{
// if this is partial input (e.g. compaction) we don't want to infer ERASE
// we anyway only EXPUNGE on summary information before this point
- status &= ~(1 << GC_BEFORE.shift());
+ status &= ~(1L << GC_BEFORE.shift());
}
else if (bounds[i/2].hlc() <= applyAtIfKnown.uniqueHlc())
{
@@ -692,7 +685,7 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Bounds>
i/= 2;
while (--i >= 0 && bounds[i].hlc() <= uniqueHlc) {}
if (i < 0 || !any(statuses[i*2+1], GC_BEFORE))
- status &= ~(1 << GC_BEFORE.shift());
+ status &= ~(1L << GC_BEFORE.shift());
}
}
}
@@ -742,10 +735,9 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Bounds>
return maxBound(LOCALLY_APPLIED);
}
- // TODO (expected): check call-sites to see if can use something weaker such as redundantBefore
- public final TxnId maxBootstrappedAt()
+ public final TxnId maxReadyAt()
{
- return bootstrappedAt;
+ return readyAt;
}
private boolean outOfBounds(EpochSupplier lb, EpochSupplier ub)
@@ -770,9 +762,14 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Bounds>
return endEpoch <= maxBound(LOCALLY_SYNCED).epoch();
}
- public boolean isLocallyRetiredOrPreBootstrap(TxnId txnId)
+ public boolean hasLostOwnership()
+ {
+ return endEpoch < Long.MAX_VALUE;
+ }
+
+ public boolean isLocallyRetiredOrUnready(TxnId txnId)
{
- return isLocallyRetired() || is(txnId, PRE_BOOTSTRAP_OR_STALE);
+ return isLocallyRetired() || is(txnId, UNREADY);
}
private boolean outOfBounds(Timestamp lb)
@@ -818,41 +815,40 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Bounds>
public static RedundantBefore EMPTY = new RedundantBefore();
- private final Ranges staleRanges, locallyRetiredRanges;
- private final TxnId maxBootstrap, maxShardAppliedBefore, maxGcBefore;
+ private final Ranges staleRanges, lostRanges;
+ private final TxnId maxStale, maxShardAppliedBefore, maxGcBefore;
private final TxnId minShardAndLocallyAppliedBefore, minGcBefore;
private final long minGcHlcBefore;
- private final long maxStartEpoch, minLocallyRetiredEpoch;
+ private final long maxStartEpoch, minEndEpoch;
private RedundantBefore()
{
- staleRanges = locallyRetiredRanges = Ranges.EMPTY;
- maxBootstrap = maxShardAppliedBefore = maxGcBefore = TxnId.NONE;
+ staleRanges = lostRanges = Ranges.EMPTY;
+ maxStale = maxShardAppliedBefore = maxGcBefore = TxnId.NONE;
minShardAndLocallyAppliedBefore = minGcBefore = TxnId.MAX;
minGcHlcBefore = 0L;
maxStartEpoch = 0;
- minLocallyRetiredEpoch = Long.MAX_VALUE;
+ minEndEpoch = Long.MAX_VALUE;
}
RedundantBefore(boolean inclusiveEnds, RoutingKey[] starts, Bounds[] values)
{
super(inclusiveEnds, starts, values);
staleRanges = extractRanges(values, b -> b.staleUntilAtLeast != null);
- locallyRetiredRanges = extractRanges(values, Bounds::isLocallyRetired);
- TxnId maxBootstrap = TxnId.NONE, maxGcBefore = TxnId.NONE, maxShardAppliedBefore = TxnId.NONE;
+ lostRanges = extractRanges(values, Bounds::hasLostOwnership);
+ TxnId maxUnready = TxnId.NONE, maxGcBefore = TxnId.NONE, maxShardAppliedBefore = TxnId.NONE;
TxnId minShardAndLocallyRedundantBefore = TxnId.MAX, minGcBefore = TxnId.MAX;
long minGcHlcBefore = Long.MAX_VALUE;
- long minLocallyRetiredEpoch = Long.MAX_VALUE, maxStartEpoch = 0;
- boolean hasLocallyRetired = !locallyRetiredRanges.isEmpty();
+ long maxStartEpoch = 0, minEndEpoch = Long.MAX_VALUE;
for (Bounds bounds : values)
{
if (bounds == null)
continue;
{
- TxnId bootstrappedAt = bounds.maxBound(PRE_BOOTSTRAP_OR_STALE);
- if (bootstrappedAt.compareTo(maxBootstrap) > 0)
- maxBootstrap = bootstrappedAt;
+ TxnId unready = bounds.maxBound(UNREADY);
+ if (unready.compareTo(maxUnready) > 0)
+ maxUnready = unready;
}
{
TxnId gcBefore = bounds.maxBound(GC_BEFORE);
@@ -872,17 +868,17 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Bounds>
}
if (bounds.startEpoch > maxStartEpoch)
maxStartEpoch = bounds.startEpoch;
- if (hasLocallyRetired && bounds.isLocallyRetired() && bounds.endEpoch < minLocallyRetiredEpoch)
- minLocallyRetiredEpoch = bounds.endEpoch;
+ if (bounds.endEpoch < minEndEpoch)
+ minEndEpoch = bounds.endEpoch;
}
- this.maxBootstrap = maxBootstrap;
+ this.maxStale = maxUnready;
this.maxShardAppliedBefore = maxShardAppliedBefore;
this.maxGcBefore = maxGcBefore;
this.minShardAndLocallyAppliedBefore = minShardAndLocallyRedundantBefore;
this.minGcBefore = minGcBefore;
this.minGcHlcBefore = minGcHlcBefore;
this.maxStartEpoch = maxStartEpoch;
- this.minLocallyRetiredEpoch = minLocallyRetiredEpoch;
+ this.minEndEpoch = minEndEpoch;
checkParanoid(starts, values);
}
@@ -928,7 +924,7 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Bounds>
if (ranges.isEmpty())
return EMPTY;
- Bounds bounds = new Bounds(null, startEpoch, endEpoch, new TxnId[] { bound }, new short[] { (short) (status.encoded & ONLY_LE_MASK), (short)status.encoded }, staleUntilAtLeast);
+ Bounds bounds = new Bounds(null, startEpoch, endEpoch, new TxnId[] { bound }, new int[] { (int) (status.encoded & ONLY_LE_MASK), (int)status.encoded }, staleUntilAtLeast);
Builder builder = new Builder(ranges.get(0).endInclusive(), ranges.size() * 2);
for (int i = 0 ; i < ranges.size() ; ++i)
{
@@ -964,9 +960,9 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Bounds>
* RedundantStatus.REDUNDANT overrides PRE_BOOTSTRAP; to avoid complicating that state machine,
* for cases where we care independently about the overall pre-bootstrap state we have a separate mechanism
*/
- public Coverage locallyDefunct(TxnId txnId, Participants<?> participants)
+ public boolean isLocallyDefunct(TxnId txnId, Participants<?> participants, Coverage coverage)
{
- return status(txnId, null, participants).get(LOCALLY_DEFUNCT);
+ return status(txnId, null, participants).is(LOCALLY_DEFUNCT, coverage);
}
public <T extends Deps> RangeDeps.BuilderByRange collectDeps(Routables<?> participants, RangeDeps.BuilderByRange builder, EpochSupplier minEpoch, EpochSupplier executeAt)
@@ -1015,6 +1011,11 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Bounds>
return foldl(ranges, Bounds::withoutAnyRetired, ranges, r -> false);
}
+ public Ranges removeLocallyRetired(Ranges ranges)
+ {
+ return ranges.without(lostRanges);
+ }
+
public TxnId minShardAndLocallyAppliedBefore()
{
return minShardAndLocallyAppliedBefore;
@@ -1035,16 +1036,6 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Bounds>
return maxGcBefore;
}
- /**
- * Subtract any ranges we consider stale or pre-bootstrap
- */
- public Ranges removePreBootstrap(TxnId txnId, Ranges ranges)
- {
- if (maxBootstrap.compareTo(txnId) <= 0)
- return ranges;
- return foldl(ranges, Bounds::withoutPreBootstrap, ranges, txnId, null);
- }
-
/**
* Subtract anything we don't need to coordinate (because they are known to be shard durable),
* and we don't execute locally, i.e. are pre-bootstrap or stale (or for RX are on ranges that are already retired)
@@ -1055,58 +1046,56 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Bounds>
{
if (txnId.isSyncPoint())
{
- if (!mayFilterStaleOrPreBootstrapOrRetiredOrNotOwned(txnId, executeAt, participants))
+ if (!mayFilterUnreadyOrNotOwned(txnId, executeAt, participants))
return participants;
- return foldl(participants, Bounds::withoutRedundantAnd_StaleOrPreBootstrapOrRetiredOrNotOwned, participants, txnId, executeAt);
+ return foldl(participants, Bounds::withoutRedundantAnd_UnreadyOrRetiredOrNotOwned, participants, txnId, executeAt);
}
else
{
- if (!mayFilterStaleOrPreBootstrap(txnId, participants))
+ if (!mayFilterStale(txnId, participants))
return participants;
- return foldl(participants, Bounds::withoutRedundantAnd_StaleOrPreBootstrap, participants, txnId, executeAt);
+ return foldl(participants, Bounds::withoutRedundantAnd_Unready, participants, txnId, executeAt);
}
}
/**
* Subtract anything we won't execute locally, i.e. are pre-bootstrap or stale (or for RX are on ranges that are already retired)
+ * but also anything that executes in an epoch we don't own
*/
- public Participants<?> expectToExecute(TxnId txnId, @Nullable Timestamp executeAt, Participants<?> participants)
+ public Participants<?> expectToExecute(TxnId txnId, @Nonnull Timestamp executeAt, Participants<?> participants)
{
- if (!mayFilterStaleOrPreBootstrap(txnId, participants))
- return participants;
-
- return foldl(participants, Bounds::withoutStaleOrPreBootstrap, participants, txnId, executeAt);
+ Invariants.require(executeAt != null);
+ return foldl(participants, Bounds::withoutLostAtExecutionOrUnready, (Participants)participants, txnId, executeAt);
}
/**
* Subtract anything we won't execute locally, i.e. are pre-bootstrap or stale or retired
*/
- public Participants<?> expectExclusiveSyncPointToWaitOn(TxnId txnId, @Nullable Timestamp executeAt, Participants<?> participants)
+ public Participants<?> expectExclusiveSyncPointToWaitOn(TxnId txnId, @Nonnull Timestamp executeAt, Participants<?> participants)
{
+ Invariants.require(executeAt != null);
Invariants.require(txnId.isSyncPoint());
- if (!mayFilterStaleOrPreBootstrapOrRetiredOrNotOwned(txnId, executeAt, participants))
- return participants;
-
- return foldl(participants, Bounds::withoutStaleOrPreBootstrapOrLocallyRetired, participants, txnId);
+ return foldl(participants, Bounds::withoutUnreadyOrLocallyRetired, participants, txnId);
}
public boolean mayFilter(TxnId txnId, @Nullable Timestamp executeAtIfKnown, Participants<?> participants)
{
- return mayFilterStaleOrPreBootstrapOrRetiredOrNotOwned(txnId, executeAtIfKnown, participants);
+ return mayFilterUnreadyOrNotOwned(txnId, executeAtIfKnown, participants);
}
- private boolean mayFilterStaleOrPreBootstrapOrRetiredOrNotOwned(TxnId txnId, @Nullable Timestamp executeAt, Participants<?> participants)
+ private boolean mayFilterUnreadyOrNotOwned(TxnId txnId, @Nullable Timestamp executeAt, Participants<?> participants)
{
- return (minLocallyRetiredEpoch <= txnId.epoch() && locallyRetiredRanges.intersects(participants))
+ long maxEpoch = (executeAt == null ? txnId : executeAt).epoch();
+ return (minEndEpoch <= maxEpoch && lostRanges.intersects(participants))
|| (executeAt != null && executeAt.epoch() < maxStartEpoch)
- || mayFilterStaleOrPreBootstrap(txnId, participants);
+ || mayFilterStale(txnId, participants);
}
- private boolean mayFilterStaleOrPreBootstrap(TxnId txnId, Participants<?> participants)
+ private boolean mayFilterStale(TxnId txnId, Participants<?> participants)
{
- return maxBootstrap.compareTo(txnId) > 0 || (staleRanges != null && staleRanges.intersects(participants));
+ return maxStale.compareTo(txnId) > 0 || (staleRanges != null && staleRanges.intersects(participants));
}
/**
@@ -1241,7 +1230,7 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Bounds>
RangeDeps rangeDeps = builder.directRangeDeps;
foldl(participants, (e, s, d, b) -> {
- int bootstrapIdx = d.txnIdsWithFlags().find(e.maxBootstrappedAt());
+ int bootstrapIdx = d.txnIdsWithFlags().find(e.maxReadyAt());
if (bootstrapIdx < 0) bootstrapIdx = -1 - bootstrapIdx;
s.bootstrapIdx = bootstrapIdx;
@@ -1296,7 +1285,7 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Bounds>
{
return "gc:" + toString(GC_BEFORE)
+ "\nlocal:" + toString(LOCALLY_DURABLE_TO_DATA_STORE, LOCALLY_DURABLE_TO_COMMAND_STORE)
- + "\nbootstrap:" + toString(PRE_BOOTSTRAP);
+ + "\nbootstrap:" + toString(UNREADY);
}
private String toString(Property p1)
diff --git a/accord-core/src/main/java/accord/local/RedundantStatus.java b/accord-core/src/main/java/accord/local/RedundantStatus.java
index c752d0d7..de15cbb1 100644
--- a/accord-core/src/main/java/accord/local/RedundantStatus.java
+++ b/accord-core/src/main/java/accord/local/RedundantStatus.java
@@ -20,7 +20,6 @@ package accord.local;
import accord.utils.Invariants;
import accord.utils.TinyEnumSet;
-import accord.utils.UnhandledEnum;
import static accord.local.RedundantStatus.Cmp.LT;
import static accord.local.RedundantStatus.Cmp.LE;
@@ -32,14 +31,13 @@ import static accord.local.RedundantStatus.Property.LOCALLY_DURABLE_TO_COMMAND_S
import static accord.local.RedundantStatus.Property.LOCALLY_DURABLE_TO_DATA_STORE;
import static accord.local.RedundantStatus.Property.LOCALLY_SYNCED;
import static accord.local.RedundantStatus.Property.LOCALLY_WITNESSED;
+import static accord.local.RedundantStatus.Property.LOG_UNAVAILABLE;
import static accord.local.RedundantStatus.Property.QUORUM_APPLIED;
import static accord.local.RedundantStatus.Property.NOT_OWNED;
-import static accord.local.RedundantStatus.Property.PRE_BOOTSTRAP;
-import static accord.local.RedundantStatus.Property.PRE_BOOTSTRAP_OR_STALE;
+import static accord.local.RedundantStatus.Property.UNREADY;
import static accord.local.RedundantStatus.Property.REVERSE_PROPERTIES;
import static accord.local.RedundantStatus.Property.SHARD_APPLIED;
import static accord.local.RedundantStatus.Property.WAS_OWNED;
-import static accord.utils.Invariants.illegalState;
// TODO (testing): validate that we never lose a status previously held,
// i.e. once any particular property holds for a TxnId, it should continue to hold in perpetuity
@@ -47,13 +45,13 @@ public class RedundantStatus
{
public enum Coverage
{
- NONE(0),
- SOME(1),
- ALL(0x10001);
+ NONE(0L),
+ SOME(1L),
+ ALL(0x100000001L);
- final int mask;
+ final long mask;
- Coverage(int mask)
+ Coverage(long mask)
{
this.mask = mask;
}
@@ -69,6 +67,8 @@ public class RedundantStatus
LE, LT
}
+ // note: while many of these properties are implied by other ones, the reason we track them as separate bits
+ // is so that the implied properties are correctly tracked across shards when we merge.
public enum Property
{
// applied or pre-bootstrap or stale or was owned
@@ -76,29 +76,31 @@ public class RedundantStatus
LOCALLY_REDUNDANT (false, true, LT),
// pre-bootstrap or stale or was owned
- // (WAS_OWNED | PRE_BOOTSTRAP_OR_STALE)
+ // (WAS_OWNED | UNREADY)
LOCALLY_DEFUNCT (false, true, LT, LOCALLY_REDUNDANT),
/**
+ *
* We can bootstrap ranges at different times, and have a transaction that participates in both ranges -
* in this case one of the portions of the transaction may be totally unordered with respect to other transactions
- * in that range because both occur prior to the bootstrappedAt point, so their (local) dependencies are entirely erased.
+ * in that range because both occur prior to the readyAt point, so their (local) dependencies are entirely erased.
* We can also re-bootstrap the same range because bootstrap failed, and leave dangling transactions to execute
* which then execute in an unordered fashion.
*
* See also {@link SafeCommandStore#safeToReadAt()}.
- * TODO (expected): do we need to distinguish this case from DEFUNCT?
*/
- PRE_BOOTSTRAP_OR_STALE ( true, true, LT, LOCALLY_DEFUNCT),
- PRE_BOOTSTRAP ( true, true, LT, PRE_BOOTSTRAP_OR_STALE),
+ UNREADY(true, true, LT, LOCALLY_DEFUNCT),
+
+ UNUSED(true, true, LT),
/**
- * Ranges can be marked as LOCALLY_LOST if the node has lost was forced by operator to give up knowledge
- * of transactional history for a specific range.
+ * A point before which we do not know OUR OWN log
*/
- LOCALLY_LOST (false, false, LT, PRE_BOOTSTRAP_OR_STALE),
+ LOG_UNAVAILABLE(true, false, LT, UNREADY),
+ // have applied a VisibilitySyncPoint locally, syncing deps for active transactions
LOCALLY_WITNESSED (false, true, LE),
+
// we've applied a sync point locally covering the transaction, but the transaction itself may not have applied
LOCALLY_SYNCED (false, true, LE, LOCALLY_REDUNDANT),
LOCALLY_APPLIED ( true, false, LE, LOCALLY_SYNCED),
@@ -127,7 +129,10 @@ public class RedundantStatus
*/
SHARD_APPLIED (false, true, LE, QUORUM_APPLIED),
+ // TODO (expected): this is effectively defunct (== GC_BEFORE). Presumably intended to apply prior to LOCALLY_DURABLE_..
+ // to permit TRUNCATE_WITH_OUTCOME to be applied by compaction more eagerly
TRUNCATE_BEFORE (false, true, LT, SHARD_APPLIED, LOCALLY_SYNCED),
+
// TODO (desired): separate GC_BEFORE with HLC_BOUND and without
GC_BEFORE (false, true, LT, TRUNCATE_BEFORE),
@@ -136,7 +141,6 @@ public class RedundantStatus
NOT_OWNED (false, false, LT),
;
-
static final Property[] PROPERTIES = values();
static final Property[] REVERSE_PROPERTIES = values();
static
@@ -151,15 +155,15 @@ public class RedundantStatus
}
final boolean overrideWasOwned;
- final boolean mergeWithPreBootstrapOrStale;
+ final boolean mergeWithUnready;
final Cmp cmp;
final int compareLessEqual;
final Property[] implies;
- Property(boolean overrideWasOwned, boolean mergeWithPreBootstrapOrStale, Cmp cmp, Property ... implies)
+ Property(boolean overrideWasOwned, boolean mergeWithUnready, Cmp cmp, Property ... implies)
{
this.overrideWasOwned = overrideWasOwned;
- this.mergeWithPreBootstrapOrStale = mergeWithPreBootstrapOrStale;
+ this.mergeWithUnready = mergeWithUnready;
this.cmp = cmp;
this.compareLessEqual = cmp == Cmp.LT ? -1 : 0;
this.implies = implies;
@@ -175,11 +179,12 @@ public class RedundantStatus
{
public static final SomeStatus NONE = new SomeStatus(0);
- public static final SomeStatus PRE_BOOTSTRAP_ONLY = oneSlow(PRE_BOOTSTRAP);
+ public static final SomeStatus UNREADY_ONLY = oneSlow(UNREADY);
public static final SomeStatus LOCALLY_WITNESSED_ONLY = oneSlow(LOCALLY_WITNESSED);
public static final SomeStatus LOCALLY_APPLIED_ONLY = oneSlow(LOCALLY_APPLIED);
public static final SomeStatus QUORUM_APPLIED_ONLY = oneSlow(QUORUM_APPLIED);
public static final SomeStatus SHARD_APPLIED_ONLY = oneSlow(SHARD_APPLIED);
+ public static final SomeStatus LOG_UNAVAILABLE_ONLY = oneSlow(LOG_UNAVAILABLE);
public static final SomeStatus LOCALLY_DURABLE_TO_DATA_STORE_ONLY = oneSlow(LOCALLY_DURABLE_TO_DATA_STORE);
public static final SomeStatus LOCALLY_DURABLE_TO_COMMAND_STORE_ONLY = oneSlow(LOCALLY_DURABLE_TO_COMMAND_STORE);
public static final SomeStatus GC_BEFORE_AND_LOCALLY_DURABLE = multi(GC_BEFORE, LOCALLY_DURABLE_TO_DATA_STORE, LOCALLY_DURABLE_TO_COMMAND_STORE);
@@ -187,7 +192,6 @@ public class RedundantStatus
final int encoded;
public SomeStatus(int encoded)
{
- Invariants.require((encoded & ~0xFFFF) == 0);
this.encoded = encoded;
}
@@ -199,54 +203,54 @@ public class RedundantStatus
private static final Coverage[] COVERAGE_TO_STRING = new Coverage[] { ALL, SOME };
public static final RedundantStatus NONE = new RedundantStatus(0);
- public static final RedundantStatus PRE_BOOTSTRAP_OR_STALE_ONLY = toAll(oneSlow(PRE_BOOTSTRAP_OR_STALE));
+ public static final RedundantStatus UNREADY_ONLY = toAll(oneSlow(UNREADY));
public static final RedundantStatus NOT_OWNED_ONLY = toAll(oneSlow(NOT_OWNED));
public static final RedundantStatus WAS_OWNED_ONLY = toAll(oneSlow(WAS_OWNED));
public static final RedundantStatus WAS_OWNED_SYNCED = toAll(multi(WAS_OWNED, LOCALLY_SYNCED));
public static final RedundantStatus WAS_OWNED_RETIRED = toAll(multi(WAS_OWNED, GC_BEFORE));
- final int encoded;
- RedundantStatus(int encoded)
+ final long encoded;
+ RedundantStatus(long encoded)
{
this.encoded = encoded;
}
public RedundantStatus mergeShards(RedundantStatus that)
{
- int merged = mergeShards(this.encoded, that.encoded);
+ long merged = mergeShards(this.encoded, that.encoded);
return selectOrCreate(merged, this, that);
}
public RedundantStatus add(RedundantStatus that)
{
- int bits = this.encoded | that.encoded;
+ long bits = this.encoded | that.encoded;
return selectOrCreate(bits, this, that);
}
public RedundantStatus subtract(RedundantStatus that)
{
- int bits = this.encoded & ~that.encoded;
+ long bits = this.encoded & ~that.encoded;
return selectOrCreate(bits, this, that);
}
static RedundantStatus addHistory(RedundantStatus add, RedundantStatus history)
{
- int addEncoded = add.encoded;
- if (history.any(PRE_BOOTSTRAP_OR_STALE))
- addEncoded &= PRE_BOOTSTRAP_MERGE_MASK;
- int encoded = addEncoded | history.encoded;
+ long addEncoded = add.encoded;
+ if (history.any(UNREADY))
+ addEncoded &= UNREADY_MERGE_MASK;
+ long encoded = addEncoded | history.encoded;
return selectOrCreate(encoded, history, add);
}
- static int addHistory(int add, int history)
+ static long addHistory(long add, long history)
{
- if (decode(history, PRE_BOOTSTRAP_OR_STALE) != Coverage.NONE)
- add &= PRE_BOOTSTRAP_MERGE_MASK;
+ if (!none(history, UNREADY))
+ add &= UNREADY_MERGE_MASK;
return add | history;
}
- static RedundantStatus selectOrCreate(int encoded, RedundantStatus a, RedundantStatus b)
+ static RedundantStatus selectOrCreate(long encoded, RedundantStatus a, RedundantStatus b)
{
if (encoded == a.encoded) return a;
if (encoded == b.encoded) return b;
@@ -255,36 +259,14 @@ public class RedundantStatus
static SomeStatus selectOrCreate(int encoded, SomeStatus a, SomeStatus b)
{
- Invariants.require(0 == (encoded & ~0xFFFF));
if (encoded == a.encoded) return a;
if (encoded == b.encoded) return b;
return new SomeStatus(encoded);
}
- public int encodedPart(Coverage coverage)
- {
- switch (coverage)
- {
- default: throw new UnhandledEnum(coverage);
- case NONE: return 0;
- case SOME: return (encoded & 0xffff);
- case ALL: return (encoded >>> 16) & 0xffff;
- }
- }
-
- public Coverage get(Property property)
- {
- return get(encoded, property);
- }
-
- public static Coverage get(short encoded, Property property)
+ public boolean is(Property property, Coverage coverage)
{
- return get(encoded & 0xFF, property);
- }
-
- public static Coverage get(int encoded, Property property)
- {
- return decode(encoded, property);
+ return is(encoded, property, coverage);
}
public boolean all(Property property)
@@ -297,22 +279,22 @@ public class RedundantStatus
return all(encoded, a, b);
}
- public static boolean all(short encoded, Property property)
+ public static boolean all(int encoded, Property property)
{
- return all(encoded & 0xFF, property);
+ return all(encoded & 0xFFFFFFFFL, property);
}
- public static boolean all(int encoded, Property property)
+ public static boolean all(long encoded, Property property)
{
- return get(encoded, property) == ALL;
+ return is(encoded, property, ALL);
}
- public static boolean all(short encoded, Property a, Property b)
+ public static boolean all(int encoded, Property a, Property b)
{
- return all(encoded & 0xFF, a, b);
+ throw new IllegalArgumentException("Impossible to match ALL on int encoded");
}
- public static boolean all(int encoded, Property a, Property b)
+ public static boolean all(long encoded, Property a, Property b)
{
return all(encoded, a) && all(encoded, b);
}
@@ -322,14 +304,14 @@ public class RedundantStatus
return none(encoded, property);
}
- public static boolean none(short encoded, Property property)
+ public static boolean none(int encoded, Property property)
{
- return none(encoded & 0xFF, property);
+ return none(encoded & 0xFFFFFFFFL, property);
}
- public static boolean none(int encoded, Property property)
+ public static boolean none(long encoded, Property property)
{
- return get(encoded, property) == Coverage.NONE;
+ return !is(encoded, property, SOME);
}
public boolean any(Property property)
@@ -337,76 +319,71 @@ public class RedundantStatus
return any(encoded, property);
}
- public static boolean any(short encoded, Property property)
+ public static boolean any(int encoded, Property property)
{
- return any(encoded & 0xFFFF, property);
+ return any(encoded & 0xFFFFFFFFL, property);
}
- public static boolean any(int encoded, Property property)
+ public static boolean any(long encoded, Property property)
{
- return get(encoded, property) != Coverage.NONE;
+ return is(encoded, property, SOME);
}
- public static boolean matchesMask(short encoded, int propertyMask)
+ // propertyMask in this case must be SOME, not ALL
+ public static boolean matchesMask(int encoded, long propertyMask)
{
- return matchesMask(encoded & 0xFFFF, propertyMask);
+ Invariants.require((propertyMask & ~0xFFFFFFFL) == 0);
+ return matchesMask(encoded & 0xFFFFFFFFL, propertyMask);
}
- public static boolean matchesMask(int encoded, int propertyMask)
+ public static boolean matchesMask(long encoded, long propertyMask)
{
return (encoded & propertyMask) == propertyMask;
}
- public static int mask(Property property, Coverage coverage)
+ public static long shiftedMask(Property property, Coverage coverage)
{
- int mask = coverage.mask;
+ long mask = coverage.mask;
return mask << property.shift();
}
- public static Coverage decode(long encoded, Property property)
+ public static boolean is(long encoded, Property property, Coverage coverage)
{
- int coverage = (int)((encoded >>> property.shift()) & ALL.mask);
- switch (coverage)
- {
- default: throw illegalState("Invalid Coverage value encoded for " + property + ": " + coverage);
- case 0: return Coverage.NONE;
- case 1: return SOME;
- case 0x10001: return ALL;
- }
+ return matchesMask(encoded, shiftedMask(property, coverage));
}
- private static final int ALL_BITS = 0xFFFF0000;
- private static final int ANY_BITS = 0x0000FFFF;
- private static final int WAS_OWNED_OVERRIDE_MASK;
- static final int PRE_BOOTSTRAP_MERGE_MASK;
- static final int ONLY_LE_MASK;
- private static final int WAS_ALL_OWNED_MASK = ALL.mask << WAS_OWNED.shift();
- private static final int NOT_OWNED_MASK = ALL.mask << NOT_OWNED.shift();
+ private static final long ALL_BITS = 0xFFFFFFFF00000000L;
+ private static final long ANY_BITS = 0x00000000FFFFFFFFL;
+ private static final long WAS_OWNED_OVERRIDE_MASK;
+ static final long UNREADY_MERGE_MASK;
+ static final long ONLY_LE_MASK;
+ private static final long WAS_ALL_OWNED_MASK = ALL.mask << WAS_OWNED.shift();
+ private static final long NOT_OWNED_MASK = ALL.mask << NOT_OWNED.shift();
static
{
- int wasOwnedMask = 0;
- int preBootstrapMask = 0, leMask = 0;
+ long wasOwnedMask = 0;
+ long unreadyMask = 0, leMask = 0;
for (Property property : Property.values())
{
if (!property.overrideWasOwned)
wasOwnedMask |= encodeAll(property);
- if (property.mergeWithPreBootstrapOrStale)
- preBootstrapMask |= encodeAny(property);
+ if (property.mergeWithUnready)
+ unreadyMask |= encodeAny(property);
if (property.cmp == LE)
leMask |= encodeAny(property);
}
WAS_OWNED_OVERRIDE_MASK = wasOwnedMask;
- PRE_BOOTSTRAP_MERGE_MASK = preBootstrapMask;
+ UNREADY_MERGE_MASK = unreadyMask;
ONLY_LE_MASK = leMask;
}
- public static int mergeShards(int a, int b)
+ public static long mergeShards(long a, long b)
{
- int either = a | b;
+ long either = a | b;
Invariants.require((either & NOT_OWNED_MASK) == 0);
- int all = (a & b) & ALL_BITS;
- int any = either & ANY_BITS;
- int result = all | any;
+ long all = (a & b) & ALL_BITS;
+ long any = either & ANY_BITS;
+ long result = all | any;
if ((either & WAS_ALL_OWNED_MASK) == WAS_ALL_OWNED_MASK)
{
if ((a & WAS_ALL_OWNED_MASK) != WAS_ALL_OWNED_MASK)
@@ -443,15 +420,15 @@ public class RedundantStatus
return 1 << property.shift();
}
- static int encodeAll(Property property)
+ static long encodeAll(Property property)
{
return ALL.mask << property.shift();
}
- static int toAll(int encodedKeyStatus)
+ static long toAll(long encodedKeyStatus)
{
- Invariants.require(0 == (encodedKeyStatus & ~0xFFFF));
- return encodedKeyStatus | (encodedKeyStatus<<16);
+ Invariants.require(0 == (encodedKeyStatus & ~0xFFFFFFFFL));
+ return encodedKeyStatus | (encodedKeyStatus<<32);
}
static RedundantStatus toAll(SomeStatus key)
@@ -478,7 +455,7 @@ public class RedundantStatus
{
implied |= TinyEnumSet.encode(property.implies);
}
- else if (get(property) == coverage)
+ else if (is(property, coverage))
{
if (!firstProperty) builder.append(",");
firstProperty = false;
diff --git a/accord-core/src/main/java/accord/local/SafeCommandStore.java b/accord-core/src/main/java/accord/local/SafeCommandStore.java
index 403b760b..6f1ee63a 100644
--- a/accord-core/src/main/java/accord/local/SafeCommandStore.java
+++ b/accord-core/src/main/java/accord/local/SafeCommandStore.java
@@ -56,6 +56,7 @@ import accord.utils.async.AsyncChains;
import static accord.local.LoadKeys.INCR;
import static accord.local.LoadKeys.NONE;
import static accord.local.LoadKeysFor.WRITE;
+import static accord.local.RedundantStatus.Property.LOCALLY_APPLIED;
import static accord.local.RedundantStatus.SomeStatus.LOCALLY_WITNESSED_ONLY;
import static accord.local.RedundantStatus.Property.LOCALLY_REDUNDANT;
import static accord.local.RedundantStatus.Property.SHARD_APPLIED;
@@ -93,6 +94,23 @@ public abstract class SafeCommandStore implements RangesForEpochSupplier, Redund
Invariants.require(reentrancyCounter >= 0);
}
+ public final boolean refusesAnyOf(Participants<?> participants)
+ {
+ Ranges refuses = commandStore().refuses;
+ return refuses != null && participants.intersects(refuses);
+ }
+
+ public final boolean refusesAllOwnedOf(Participants<?> participants)
+ {
+ Ranges refuses = commandStore().refuses;
+ if (refuses == null)
+ return false;
+
+ // TODO (required): memoize this, and expose it as a standard method as we want it elsewhere
+ Ranges notRetired = redundantBefore().removeLocallyRetired(ranges().all());
+ return refuses.containsAll(participants.slice(notRetired, Minimal));
+ }
+
/**
* If the transaction exists (with some associated data) in the CommandStore, return it. Otherwise return null.
*
@@ -328,7 +346,7 @@ public abstract class SafeCommandStore implements RangesForEpochSupplier, Redund
if (!txnId.isVisible())
return;
- commandStore().updateMaxConflicts(prev, updated);
+ commandStore().updateMaxConflicts(prev, updated, force);
}
/**
@@ -447,7 +465,7 @@ public abstract class SafeCommandStore implements RangesForEpochSupplier, Redund
TxnId maxTxnId = txnIdsForKey.get(txnIdsForKey.size() - 1);
// TODO (desired): convert to O(n) merge
RedundantStatus status = redundantBefore.status(maxTxnId, null, key);
- if (!status.all(SHARD_APPLIED) || !status.all(LOCALLY_REDUNDANT))
+ if (!status.all(SHARD_APPLIED, LOCALLY_APPLIED) || !status.all(LOCALLY_REDUNDANT)) // TODO (required): should be a new property for correct merge?
select.set(i);
}
if (select.getSetBitCount() != keys.size())
@@ -515,7 +533,7 @@ public abstract class SafeCommandStore implements RangesForEpochSupplier, Redund
CommandStore commandStore = safeStore.commandStore();
Ranges touches = syncCommand.participants().touches().toRanges();
- Ranges waitingOn = commandStore.isWaitingOnSync(syncId, touches);
+ Ranges waitingOn = commandStore.isWaitingOnVisibility(syncId, touches);
if (waitingOn.isEmpty())
return;
@@ -531,11 +549,11 @@ public abstract class SafeCommandStore implements RangesForEpochSupplier, Redund
}));
});
- AsyncChains.chain(() -> commandStore.markSyncing(syncId, waitingOn))
+ AsyncChains.chain(() -> commandStore.markingVisible(syncId, waitingOn))
.flatMap(ignore -> AsyncChains.reduce(async, Reduce.toNull(), null))
.begin((success, fail) -> {
- if (fail == null) commandStore.execute((PreLoadContext.Empty)() -> "Mark Synced", safeStore0 -> commandStore.markSynced(safeStore0, syncId, waitingOn));
- else commandStore.execute((PreLoadContext.Empty)() -> "Unmark Syncing", safeStore0 -> commandStore.unmarkSyncing(syncId, waitingOn));
+ if (fail == null) commandStore.execute((PreLoadContext.Empty)() -> "Mark Synced", safeStore0 -> commandStore.markVisible(safeStore0, syncId, waitingOn));
+ else commandStore.execute((PreLoadContext.Empty)() -> "Unmark Syncing", safeStore0 -> commandStore.cancelMarkingVisible(syncId, waitingOn));
});
}
diff --git a/accord-core/src/main/java/accord/local/cfk/CommandsForKey.java b/accord-core/src/main/java/accord/local/cfk/CommandsForKey.java
index dcf8e6af..c1b6305c 100644
--- a/accord-core/src/main/java/accord/local/cfk/CommandsForKey.java
+++ b/accord-core/src/main/java/accord/local/cfk/CommandsForKey.java
@@ -88,7 +88,7 @@ import static accord.local.cfk.CommandsForKey.InternalStatus.TRANSITIVE;
import static accord.local.cfk.CommandsForKey.InternalStatus.TRANSITIVE_VISIBLE;
import static accord.local.cfk.CommandsForKey.InternalStatus.from;
import static accord.local.cfk.CommandsForKey.InternalStatus.prunedFrom;
-import static accord.local.cfk.PostProcess.notifyManagedPreBootstrap;
+import static accord.local.cfk.PostProcess.notifyManagedUnready;
import static accord.local.cfk.Pruning.find;
import static accord.local.cfk.Pruning.isAnyPredecessorWaitingOnPruned;
import static accord.local.cfk.Pruning.isWaitingOnPruned;
@@ -98,7 +98,7 @@ import static accord.local.cfk.Pruning.prunedBeforeId;
import static accord.local.cfk.UpdateUnmanagedMode.UPDATE;
import static accord.local.cfk.Updating.insertOrUpdate;
import static accord.local.CommandSummaries.ComputeIsDep.IGNORE;
-import static accord.local.cfk.Updating.maybeUpdateMaxAppliedPreBootstrapWriteById;
+import static accord.local.cfk.Updating.maybeUpdateMaxAppliedUnreadyWriteById;
import static accord.primitives.Known.KnownExecuteAt.ApplyAtKnown;
import static accord.primitives.Routable.Domain.Key;
import static accord.primitives.Status.Durability.AllQuorums;
@@ -301,7 +301,7 @@ public class CommandsForKey extends CommandsForKeyUpdate
public static boolean executes(QuickBounds bounds, TxnId txnId, Timestamp executeAt)
{
return executesIgnoringBootstrap(bounds, txnId, executeAt)
- && bounds.bootstrappedAt.compareTo(txnId) < 0;
+ && bounds.readyAt.compareTo(txnId) < 0;
}
public static boolean executesIgnoringBootstrap(QuickBounds bounds, TxnId txnId, Timestamp executeAt)
@@ -341,7 +341,7 @@ public class CommandsForKey extends CommandsForKeyUpdate
interface Updater<O>
{
O update(RoutingKey key, QuickBounds bounds, boolean isNewBoundsInfo,
- TxnInfo[] byId, int minUndecidedById, int maxAppliedPreBootstrapWriteById,
+ TxnInfo[] byId, int minUndecidedById, int maxAppliedUnreadyWriteById,
TxnInfo[] committedByExecuteAt, int maxAppliedWriteByExecuteAt,
long maxUniqueHlc, Object[] loadingPruned, int newPrunedBeforeById,
Unmanaged[] unmanageds, boolean expectUpToDate);
@@ -1063,17 +1063,17 @@ public class CommandsForKey extends CommandsForKeyUpdate
// all transactions, sorted by TxnId
final TxnInfo[] byId;
final int minUndecidedById;
- final int maxAppliedPreBootstrapWriteById;
+ final int maxAppliedUnreadyWriteById;
// managed commands that are committed, stable or applied; sorted by executeAt
// TODO (required): validate that it is always a prefix that is Applied (i.e. never a gap)
// TODO (desired): filter transactions whose execution we don't manage
final TxnInfo[] committedByExecuteAt;
- final int maxAppliedWriteByExecuteAt; // applied OR prebootstrap
+ final int maxAppliedWriteByExecuteAt; // applied OR unready
// a btree keyed by TxnId we have encountered since pruning that occur before prunedBefore;
// mapping to those TxnId that had witnessed this potentially-pruned TxnId.
- // note that this may include TxnId < bootstrappedAt
+ // note that this may include TxnId < readyAt
final Object[] loadingPruned;
// this points to the primary prunedBefore APPLIED write, but note that if this CFK spans multiple epochs
// we retain the latest APPLIED write per epoch so that any dependencies we compute that might include
@@ -1083,18 +1083,18 @@ public class CommandsForKey extends CommandsForKeyUpdate
final Unmanaged[] unmanageds;
final long maxUniqueHlc;
- CommandsForKey(RoutingKey key, QuickBounds bounds, boolean isNewBoundsInfo, TxnInfo[] byId, int minUndecidedById, int maxAppliedPreBootstrapWriteById, TxnInfo[] committedByExecuteAt, int maxAppliedWriteByExecuteAt, long maxUniqueHlc, Object[] loadingPruned, int prunedBeforeById, Unmanaged[] unmanageds, boolean expectUpToDate)
+ CommandsForKey(RoutingKey key, QuickBounds bounds, boolean isNewBoundsInfo, TxnInfo[] byId, int minUndecidedById, int maxAppliedUnreadyWriteById, TxnInfo[] committedByExecuteAt, int maxAppliedWriteByExecuteAt, long maxUniqueHlc, Object[] loadingPruned, int prunedBeforeById, Unmanaged[] unmanageds, boolean expectUpToDate)
{
- this(key, bounds, byId, minUndecidedById, maxAppliedPreBootstrapWriteById, committedByExecuteAt, maxAppliedWriteByExecuteAt, maxUniqueHlc, loadingPruned, prunedBeforeById, unmanageds, expectUpToDate);
+ this(key, bounds, byId, minUndecidedById, maxAppliedUnreadyWriteById, committedByExecuteAt, maxAppliedWriteByExecuteAt, maxUniqueHlc, loadingPruned, prunedBeforeById, unmanageds, expectUpToDate);
}
- CommandsForKey(RoutingKey key, QuickBounds bounds, TxnInfo[] byId, int minUndecidedById, int maxAppliedPreBootstrapWriteById, TxnInfo[] committedByExecuteAt, int maxAppliedWriteByExecuteAt, long maxUniqueHlc, Object[] loadingPruned, int prunedBeforeById, Unmanaged[] unmanageds, boolean expectUpToDate)
+ CommandsForKey(RoutingKey key, QuickBounds bounds, TxnInfo[] byId, int minUndecidedById, int maxAppliedUnreadyWriteById, TxnInfo[] committedByExecuteAt, int maxAppliedWriteByExecuteAt, long maxUniqueHlc, Object[] loadingPruned, int prunedBeforeById, Unmanaged[] unmanageds, boolean expectUpToDate)
{
this.key = key;
this.bounds = bounds;
this.byId = byId;
this.minUndecidedById = minUndecidedById;
- this.maxAppliedPreBootstrapWriteById = maxAppliedPreBootstrapWriteById;
+ this.maxAppliedUnreadyWriteById = maxAppliedUnreadyWriteById;
this.committedByExecuteAt = committedByExecuteAt;
this.maxAppliedWriteByExecuteAt = maxAppliedWriteByExecuteAt;
this.maxUniqueHlc = maxUniqueHlc;
@@ -1110,7 +1110,7 @@ public class CommandsForKey extends CommandsForKeyUpdate
this.bounds = copy.bounds;
this.byId = copy.byId;
this.minUndecidedById = copy.minUndecidedById;
- this.maxAppliedPreBootstrapWriteById = copy.maxAppliedPreBootstrapWriteById;
+ this.maxAppliedUnreadyWriteById = copy.maxAppliedUnreadyWriteById;
this.committedByExecuteAt = copy.committedByExecuteAt;
this.maxAppliedWriteByExecuteAt = copy.maxAppliedWriteByExecuteAt;
this.maxUniqueHlc = copy.maxUniqueHlc;
@@ -1131,7 +1131,7 @@ public class CommandsForKey extends CommandsForKeyUpdate
this.bounds = NO_BOUNDS_INFO;
this.byId = NO_INFOS;
this.committedByExecuteAt = NO_INFOS;
- this.minUndecidedById = this.maxAppliedPreBootstrapWriteById = this.maxAppliedWriteByExecuteAt = -1;
+ this.minUndecidedById = this.maxAppliedUnreadyWriteById = this.maxAppliedWriteByExecuteAt = -1;
this.maxUniqueHlc = maxUniqueHlc;
this.loadingPruned = LoadingPruned.empty();
this.prunedBeforeById = -1;
@@ -1211,20 +1211,20 @@ public class CommandsForKey extends CommandsForKeyUpdate
public TxnId redundantOrBootstrappedBefore()
{
- return TxnId.nonNullOrMax(redundantBefore(), bounds.bootstrappedAt);
+ return TxnId.nonNullOrMax(redundantBefore(), bounds.readyAt);
}
- public TxnId bootstrappedAt()
+ public TxnId readyAt()
{
- return bootstrappedAt(bounds);
+ return readyAt(bounds);
}
- static TxnId bootstrappedAt(QuickBounds bounds)
+ static TxnId readyAt(QuickBounds bounds)
{
- TxnId bootstrappedAt = bounds.bootstrappedAt;
- if (bootstrappedAt.compareTo(bounds.gcBefore) <= 0)
- bootstrappedAt = null;
- return bootstrappedAt;
+ TxnId readyAt = bounds.readyAt;
+ if (readyAt.compareTo(bounds.gcBefore) <= 0)
+ readyAt = null;
+ return readyAt;
}
public TxnId redundantBefore()
@@ -1249,24 +1249,24 @@ public class CommandsForKey extends CommandsForKeyUpdate
return bounds.locallyAppliedBefore;
}
- public boolean isPostBootstrapAndOwned(TxnId txnId)
+ public boolean isReadyAndOwned(TxnId txnId)
{
- return isPostBootstrapAndOwned(txnId, bounds);
+ return isReadyAndOwned(txnId, bounds);
}
- private static boolean isPostBootstrapAndOwned(TxnId txnId, QuickBounds bounds)
+ private static boolean isReadyAndOwned(TxnId txnId, QuickBounds bounds)
{
- return txnId.compareTo(bounds.bootstrappedAt) >= 0 && txnId.epoch() >= bounds.startEpoch;
+ return txnId.compareTo(bounds.readyAt) >= 0 && txnId.epoch() >= bounds.startEpoch;
}
- public boolean isPreBootstrap(TxnId txnId)
+ public boolean isUnready(TxnId txnId)
{
- return isPreBootstrap(txnId, bounds);
+ return isUnready(txnId, bounds);
}
- static boolean isPreBootstrap(TxnId txnId, QuickBounds bounds)
+ static boolean isUnready(TxnId txnId, QuickBounds bounds)
{
- return txnId.compareTo(bounds.bootstrappedAt) < 0 || txnId.epoch() < bounds.startEpoch;
+ return txnId.compareTo(bounds.readyAt) < 0 || txnId.epoch() < bounds.startEpoch;
}
private TxnId nextWaitingToApply(Kinds kinds, @Nullable Timestamp untilExecuteAt)
@@ -1647,7 +1647,7 @@ public class CommandsForKey extends CommandsForKeyUpdate
if (maxUniqueHlc <= bounds.gcBefore.hlc() && bounds.gcBefore.is(HLC_BOUND))
return this;
- return new CommandsForKey(key, bounds, byId, minUndecidedById, maxAppliedPreBootstrapWriteById, committedByExecuteAt, maxAppliedWriteByExecuteAt, minUniqueHlc, loadingPruned, prunedBeforeById, unmanageds, true);
+ return new CommandsForKey(key, bounds, byId, minUndecidedById, maxAppliedUnreadyWriteById, committedByExecuteAt, maxAppliedWriteByExecuteAt, minUniqueHlc, loadingPruned, prunedBeforeById, unmanageds, true);
}
public CommandsForKey setDurable(TxnId txnId, Durability durability)
@@ -1661,7 +1661,7 @@ public class CommandsForKey extends CommandsForKeyUpdate
else return this;
// return the exact same data as we have updated in place, but change detection relies on identity
- return new CommandsForKey(key, bounds, byId, minUndecidedById, maxAppliedPreBootstrapWriteById, committedByExecuteAt, maxAppliedWriteByExecuteAt, maxUniqueHlc, loadingPruned, prunedBeforeById, unmanageds, true);
+ return new CommandsForKey(key, bounds, byId, minUndecidedById, maxAppliedUnreadyWriteById, committedByExecuteAt, maxAppliedWriteByExecuteAt, maxUniqueHlc, loadingPruned, prunedBeforeById, unmanageds, true);
}
CommandsForKeyUpdate maybePrunedCallback(SafeCommandStore safeStore, Command update, @Nullable LoadingPruned loading)
@@ -1794,11 +1794,11 @@ public class CommandsForKey extends CommandsForKeyUpdate
// TODO (required): additional linearizability violation detection, based on expectation of presence in missing set
- CommandsForKeyUpdate update(TxnInfo[] newById, int newMinUndecidedById, int newMaxAppliedPreBootstrapWriteById, TxnInfo[] newCommittedByExecuteAt, int newMaxAppliedWriteByExecuteAt, long maxUniqueHlc, Object[] newLoadingPruned, int newPrunedBeforeById, @Nullable TxnInfo curInfo, @Nonnull TxnInfo newInfo)
+ CommandsForKeyUpdate update(TxnInfo[] newById, int newMinUndecidedById, int newMaxAppliedUnreadyWriteById, TxnInfo[] newCommittedByExecuteAt, int newMaxAppliedWriteByExecuteAt, long maxUniqueHlc, Object[] newLoadingPruned, int newPrunedBeforeById, @Nullable TxnInfo curInfo, @Nonnull TxnInfo newInfo)
{
Invariants.require(prunedBeforeById < 0 || newById[newPrunedBeforeById].equals(byId[prunedBeforeById]));
return updateAndNotifyUnmanageds(key, bounds, false,
- newById, newMinUndecidedById, newMaxAppliedPreBootstrapWriteById,
+ newById, newMinUndecidedById, newMaxAppliedUnreadyWriteById,
newCommittedByExecuteAt, newMaxAppliedWriteByExecuteAt,
maxUniqueHlc, newLoadingPruned, newPrunedBeforeById,
unmanageds, true, curInfo, newInfo);
@@ -1820,7 +1820,7 @@ public class CommandsForKey extends CommandsForKeyUpdate
{
int countCommitted = 0;
int minUndecidedById = -1;
- int maxAppliedPreBootstrapWriteById = -1;
+ int maxAppliedUnreadyWriteById = -1;
for (int i = 0; i < byId.length ; ++i)
{
TxnInfo txn = byId[i];
@@ -1833,7 +1833,7 @@ public class CommandsForKey extends CommandsForKeyUpdate
}
else
{
- maxAppliedPreBootstrapWriteById = maybeUpdateMaxAppliedPreBootstrapWriteById(bounds, byId, maxAppliedPreBootstrapWriteById, i);
+ maxAppliedUnreadyWriteById = maybeUpdateMaxAppliedUnreadyWriteById(bounds, byId, maxAppliedUnreadyWriteById, i);
}
}
TxnInfo[] committedByExecuteAt = new TxnInfo[countCommitted];
@@ -1848,13 +1848,13 @@ public class CommandsForKey extends CommandsForKeyUpdate
while (--maxAppliedWriteByExecuteAt >= 0)
{
TxnInfo txn = committedByExecuteAt[maxAppliedWriteByExecuteAt];
- if (txn.is(Write) && (txn.is(APPLIED) || isPreBootstrap(txn, bounds)))
+ if (txn.is(Write) && (txn.is(APPLIED) || isUnready(txn, bounds)))
break;
}
if (maxAppliedWriteByExecuteAt >= 0) maxUniqueHlc = Math.max(maxUniqueHlc, committedByExecuteAt[maxAppliedWriteByExecuteAt].executeAt.hlc());
else maxUniqueHlc = Math.max(maxUniqueHlc, bounds.gcBefore.hlc() - 1); // only guaranteed to witness those with strictly lower hlc; might be some txn agreed with higher flag bits
- return updater.update(key, bounds, isNewBoundsInfo, byId, minUndecidedById, maxAppliedPreBootstrapWriteById, committedByExecuteAt, maxAppliedWriteByExecuteAt, maxUniqueHlc, loadingPruned, newPrunedBeforeById, unmanageds, expectUpToDate);
+ return updater.update(key, bounds, isNewBoundsInfo, byId, minUndecidedById, maxAppliedUnreadyWriteById, committedByExecuteAt, maxAppliedWriteByExecuteAt, maxUniqueHlc, loadingPruned, newPrunedBeforeById, unmanageds, expectUpToDate);
}
static CommandsForKeyUpdate reconstructAndUpdateUnmanaged(RoutingKey key, QuickBounds bounds, boolean isNewBoundsInfo, TxnInfo[] byId, long maxUniqueHlc, Object[] loadingPruned, int newPrunedBeforeById, Unmanaged[] unmanageds, boolean expectUpToDate)
@@ -1862,17 +1862,17 @@ public class CommandsForKey extends CommandsForKeyUpdate
return reconstruct(key, bounds, isNewBoundsInfo, byId, maxUniqueHlc, loadingPruned, newPrunedBeforeById, unmanageds, expectUpToDate, CommandsForKey::updateAndNotifyUnmanageds);
}
- static CommandsForKeyUpdate updateAndNotifyUnmanageds(RoutingKey key, QuickBounds bounds, boolean isNewBoundsInfo, TxnInfo[] byId, int minUndecidedById, int maxAppliedPreBootstrapWriteById, TxnInfo[] committedByExecuteAt, int maxAppliedWriteByExecuteAt, long maxUniqueHlc, Object[] loadingPruned, int newPrunedBeforeById, Unmanaged[] unmanageds, boolean expectUpToDate)
+ static CommandsForKeyUpdate updateAndNotifyUnmanageds(RoutingKey key, QuickBounds bounds, boolean isNewBoundsInfo, TxnInfo[] byId, int minUndecidedById, int maxAppliedUnreadyWriteById, TxnInfo[] committedByExecuteAt, int maxAppliedWriteByExecuteAt, long maxUniqueHlc, Object[] loadingPruned, int newPrunedBeforeById, Unmanaged[] unmanageds, boolean expectUpToDate)
{
- return updateAndNotifyUnmanageds(key, bounds, isNewBoundsInfo, byId, minUndecidedById, maxAppliedPreBootstrapWriteById, committedByExecuteAt, maxAppliedWriteByExecuteAt, maxUniqueHlc, loadingPruned, newPrunedBeforeById, unmanageds, expectUpToDate, null, null);
+ return updateAndNotifyUnmanageds(key, bounds, isNewBoundsInfo, byId, minUndecidedById, maxAppliedUnreadyWriteById, committedByExecuteAt, maxAppliedWriteByExecuteAt, maxUniqueHlc, loadingPruned, newPrunedBeforeById, unmanageds, expectUpToDate, null, null);
}
- static CommandsForKeyUpdate updateAndNotifyUnmanageds(RoutingKey key, QuickBounds bounds, boolean isNewBoundsInfo, TxnInfo[] byId, int minUndecidedById, int maxAppliedPreBootstrapWriteById, TxnInfo[] committedByExecuteAt, int maxAppliedWriteByExecuteAt, long maxUniqueHlc, Object[] loadingPruned, int newPrunedBeforeById, Unmanaged[] unmanageds, boolean expectUpToDate, @Nullable TxnInfo curInfo, @Nullable TxnInfo newInfo)
+ static CommandsForKeyUpdate updateAndNotifyUnmanageds(RoutingKey key, QuickBounds bounds, boolean isNewBoundsInfo, TxnInfo[] byId, int minUndecidedById, int maxAppliedUnreadyWriteById, TxnInfo[] committedByExecuteAt, int maxAppliedWriteByExecuteAt, long maxUniqueHlc, Object[] loadingPruned, int newPrunedBeforeById, Unmanaged[] unmanageds, boolean expectUpToDate, @Nullable TxnInfo curInfo, @Nullable TxnInfo newInfo)
{
NotifyUnmanagedResult notifyUnmanaged = PostProcess.notifyUnmanaged(unmanageds, byId, minUndecidedById, committedByExecuteAt, maxAppliedWriteByExecuteAt, loadingPruned, bounds, isNewBoundsInfo, curInfo, newInfo);
if (notifyUnmanaged != null)
unmanageds = notifyUnmanaged.newUnmanaged;
- CommandsForKey newCfk = new CommandsForKey(key, bounds, byId, minUndecidedById, maxAppliedPreBootstrapWriteById, committedByExecuteAt, maxAppliedWriteByExecuteAt, maxUniqueHlc, loadingPruned, newPrunedBeforeById, unmanageds, expectUpToDate);
+ CommandsForKey newCfk = new CommandsForKey(key, bounds, byId, minUndecidedById, maxAppliedUnreadyWriteById, committedByExecuteAt, maxAppliedWriteByExecuteAt, maxUniqueHlc, loadingPruned, newPrunedBeforeById, unmanageds, expectUpToDate);
CommandsForKeyUpdate result = newCfk;
if (notifyUnmanaged != null)
result = new CommandsForKeyUpdateWithPostProcess(newCfk, notifyUnmanaged.postProcess);
@@ -1883,12 +1883,12 @@ public class CommandsForKey extends CommandsForKeyUpdate
CommandsForKey update(Unmanaged[] newUnmanageds)
{
- return new CommandsForKey(key, bounds, byId, minUndecidedById, maxAppliedPreBootstrapWriteById, committedByExecuteAt, maxAppliedWriteByExecuteAt, maxUniqueHlc, loadingPruned, prunedBeforeById, newUnmanageds, true);
+ return new CommandsForKey(key, bounds, byId, minUndecidedById, maxAppliedUnreadyWriteById, committedByExecuteAt, maxAppliedWriteByExecuteAt, maxUniqueHlc, loadingPruned, prunedBeforeById, newUnmanageds, true);
}
CommandsForKey update(Object[] newLoadingPruned)
{
- return new CommandsForKey(key, bounds, byId, minUndecidedById, maxAppliedPreBootstrapWriteById, committedByExecuteAt, maxAppliedWriteByExecuteAt, maxUniqueHlc, newLoadingPruned, prunedBeforeById, unmanageds, true);
+ return new CommandsForKey(key, bounds, byId, minUndecidedById, maxAppliedUnreadyWriteById, committedByExecuteAt, maxAppliedWriteByExecuteAt, maxUniqueHlc, newLoadingPruned, prunedBeforeById, unmanageds, true);
}
CommandsForKeyUpdate registerUnmanaged(SafeCommandStore safeStore, SafeCommand safeCommand, UpdateUnmanagedMode mode)
@@ -1979,7 +1979,7 @@ public class CommandsForKey extends CommandsForKeyUpdate
long unappliedCounters = 0L;
TxnId minUndecided = minUndecided();
if (minUndecided == null)
- minUndecided = bootstrappedAt(bounds); // we don't count txns before this as waiting to execute
+ minUndecided = readyAt(bounds); // we don't count txns before this as waiting to execute
for (int i = maxAppliedWriteByExecuteAt + 1; i < mayExecuteToIndex ; ++i)
{
@@ -2049,7 +2049,7 @@ public class CommandsForKey extends CommandsForKeyUpdate
{
TxnId txnId = txn.plainTxnId();
long minUniqueHlc = 0;
- if (txnId.isWrite() && (maxAppliedPreBootstrapWriteById < 0 || byId[maxAppliedPreBootstrapWriteById].executeAt.compareTo(txn.executeAt) < 0))
+ if (txnId.isWrite() && (maxAppliedUnreadyWriteById < 0 || byId[maxAppliedUnreadyWriteById].executeAt.compareTo(txn.executeAt) < 0))
minUniqueHlc = maxUniqueHlc + 1;
notifySink.notWaiting(safeStore, txnId, key, minUniqueHlc);
// TODO (expected): avoid invoking this here; we may do redundant work if we have local dependencies we're already waiting on
@@ -2098,10 +2098,10 @@ public class CommandsForKey extends CommandsForKeyUpdate
// we can't let HLC epoch go backwards as this breaks assumptions around maxUniqueHlc tracking
if (newBounds.gcBefore.hlc() < bounds.gcBefore.hlc())
{
- if (newBounds.endEpoch != bounds.endEpoch || !newBounds.bootstrappedAt.equals(bounds.bootstrappedAt) || !newBounds.locallyAppliedBefore.equals(bounds.locallyAppliedBefore))
+ if (newBounds.endEpoch != bounds.endEpoch || !newBounds.readyAt.equals(bounds.readyAt) || !newBounds.locallyAppliedBefore.equals(bounds.locallyAppliedBefore))
{
newBounds = bounds.withEpochs(bounds.startEpoch, newBounds.endEpoch)
- .withBootstrappedAtLeast(newBounds.bootstrappedAt)
+ .withReadyAtLeast(newBounds.readyAt)
.withLocallyAppliedAtLeast(bounds.locallyAppliedBefore);
}
else
@@ -2110,7 +2110,7 @@ public class CommandsForKey extends CommandsForKeyUpdate
}
}
else if (newBounds.gcBefore.equals(bounds.gcBefore)
- && newBounds.bootstrappedAt.equals(bounds.bootstrappedAt)
+ && newBounds.readyAt.equals(bounds.readyAt)
&& newBounds.locallyAppliedBefore.equals(bounds.locallyAppliedBefore)
&& newBounds.endEpoch == bounds.endEpoch)
{
@@ -2135,13 +2135,13 @@ public class CommandsForKey extends CommandsForKeyUpdate
int newPrunedBeforeById = prunedBeforeId(newById, prunedBefore(), redundantBefore(newBounds));
Invariants.paranoid(!expectUpToDate || (newPrunedBeforeById < 0 ? prunedBeforeById < 0 || byId[prunedBeforeById].compareTo(newBounds.gcBefore) < 0 : newById[newPrunedBeforeById].equals(byId[prunedBeforeById])));
- return notifyManagedPreBootstrap(this, newBounds, reconstructAndUpdateUnmanaged(key, newBounds, true, newById, maxUniqueHlc, newLoadingPruned, newPrunedBeforeById, unmanageds, expectUpToDate));
+ return notifyManagedUnready(this, newBounds, reconstructAndUpdateUnmanaged(key, newBounds, true, newById, maxUniqueHlc, newLoadingPruned, newPrunedBeforeById, unmanageds, expectUpToDate));
}
/**
* Permits out-of-band truncation of CommandsForKey (i.e. on another thread touching only storage) so that we do not
* notify any commands. Correctness largely relies on the fact that the full withRedundantBeforeAtLeast will be invoked
- * on load and not no-op due to e.g. newSafelyPrunedBefore or newBootstrappedAt being non-null.
+ * on load and not no-op due to e.g. newSafelyPrunedBefore or newReadyAt being non-null.
*/
@VisibleForImplementation
public CommandsForKey withRedundantBeforeAtLeast(TxnId newRedundantBefore, boolean expectUpToDate)
@@ -2225,35 +2225,35 @@ public class CommandsForKey extends CommandsForKeyUpdate
return maxAppliedWriteByExecuteAt < 0 ? NO_INFO : committedByExecuteAt[maxAppliedWriteByExecuteAt];
}
- static int maxContiguousManagedAppliedIndex(TxnInfo[] committedByExecuteAt, int maxAppliedWriteByExecuteAt, TxnId bootstrappedAt)
+ static int maxContiguousManagedAppliedIndex(TxnInfo[] committedByExecuteAt, int maxAppliedWriteByExecuteAt, TxnId readyAt)
{
int i = maxAppliedWriteByExecuteAt + 1;
while (i < committedByExecuteAt.length)
{
TxnInfo txn = committedByExecuteAt[i];
// TODO (expected): should we count any final run of !managesExecution()? i.e. if we have Y(es)N(o)YNYNNN, should we not stop after only YNYNY?
- if (!txn.is(APPLIED) && managesExecution(txn) && (bootstrappedAt == null || bootstrappedAt.compareTo(txn) <= 0))
+ if (!txn.is(APPLIED) && managesExecution(txn) && (readyAt == null || readyAt.compareTo(txn) <= 0))
break;
++i;
}
return i - 1;
}
- static TxnInfo maxContiguousManagedApplied(TxnInfo[] committedByExecuteAt, int maxAppliedWriteByExecuteAt, TxnId bootstrappedAt)
+ static TxnInfo maxContiguousManagedApplied(TxnInfo[] committedByExecuteAt, int maxAppliedWriteByExecuteAt, TxnId readyAt)
{
- int i = maxContiguousManagedAppliedIndex(committedByExecuteAt, maxAppliedWriteByExecuteAt, bootstrappedAt);
+ int i = maxContiguousManagedAppliedIndex(committedByExecuteAt, maxAppliedWriteByExecuteAt, readyAt);
return i < 0 ? null : committedByExecuteAt[i];
}
/**
* Treat anything pre bootstrap or redundant as applied. i.e.,
*
- * max(max(bootstrappedAt, redundantBefore), maxContiguousManagedApplied().executeAt)
+ * max(max(readyAt, redundantBefore), maxContiguousManagedApplied().executeAt)
*/
- static Timestamp maxContiguousManagedAppliedExecuteAt(TxnInfo[] committedByExecuteAt, int maxAppliedWriteByExecuteAt, TxnId bootstrappedAt, TxnId redundantBefore)
+ static Timestamp maxContiguousManagedAppliedExecuteAt(TxnInfo[] committedByExecuteAt, int maxAppliedWriteByExecuteAt, TxnId readyAt, TxnId redundantBefore)
{
- Timestamp maxBound = TxnId.nonNullOrMax(redundantBefore, bootstrappedAt);
- TxnInfo maxInfo = maxContiguousManagedApplied(committedByExecuteAt, maxAppliedWriteByExecuteAt, bootstrappedAt);
+ Timestamp maxBound = TxnId.nonNullOrMax(redundantBefore, readyAt);
+ TxnInfo maxInfo = maxContiguousManagedApplied(committedByExecuteAt, maxAppliedWriteByExecuteAt, readyAt);
if (maxInfo == null || maxInfo.executeAt.compareTo(maxBound) < 0)
return maxBound;
return maxInfo.executeAt;
@@ -2272,7 +2272,7 @@ public class CommandsForKey extends CommandsForKeyUpdate
break;
}
- if (isPreBootstrap(txn))
+ if (isUnready(txn))
break;
}
}
@@ -2286,7 +2286,7 @@ public class CommandsForKey extends CommandsForKeyUpdate
Invariants.require(prunedBeforeById == -1 || (prunedBefore().is(APPLIED) && prunedBefore().is(Write)));
Invariants.require(minUndecidedById < 0 || (byId[minUndecidedById].status().compareTo(InternalStatus.COMMITTED) < 0 && mayExecute(byId[minUndecidedById])));
Invariants.require(maxAppliedWrite() == null || maxUniqueHlc >= maxAppliedWrite().executeAt.hlc());
- Invariants.require(maxAppliedPreBootstrapWriteById < 0 || (maxAppliedPreBootstrapWriteById < byId.length && byId[maxAppliedPreBootstrapWriteById].isWrite() && !byId[maxAppliedPreBootstrapWriteById].mayExecute()));
+ Invariants.require(maxAppliedUnreadyWriteById < 0 || (maxAppliedUnreadyWriteById < byId.length && byId[maxAppliedUnreadyWriteById].isWrite() && !byId[maxAppliedUnreadyWriteById].mayExecute()));
if (maxAppliedWriteByExecuteAt >= 0)
{
@@ -2306,8 +2306,8 @@ public class CommandsForKey extends CommandsForKeyUpdate
}
for (TxnInfo txn : committedByExecuteAt) Invariants.require(txn.mayExecute());
- if (minUndecidedById >= 0) for (int i = 0 ; i < minUndecidedById ; ++i) Invariants.require(byId[i].status().compareTo(InternalStatus.COMMITTED) >= 0 || !mayExecute(byId[i]) || isPreBootstrap(byId[i]));
- else for (TxnInfo txn : byId) Invariants.require(txn.status().compareTo(InternalStatus.COMMITTED) >= 0 || !mayExecute(txn) || isPreBootstrap(txn));
+ if (minUndecidedById >= 0) for (int i = 0 ; i < minUndecidedById ; ++i) Invariants.require(byId[i].status().compareTo(InternalStatus.COMMITTED) >= 0 || !mayExecute(byId[i]) || isUnready(byId[i]));
+ else for (TxnInfo txn : byId) Invariants.require(txn.status().compareTo(InternalStatus.COMMITTED) >= 0 || !mayExecute(txn) || isUnready(txn));
if (maxAppliedWriteByExecuteAt >= 0)
{
@@ -2358,12 +2358,12 @@ public class CommandsForKey extends CommandsForKeyUpdate
if (maxDecidedBefore < decidedBefore)
decidedBefore = maxDecidedBefore;
}
- int appliedBefore = 1 + maxContiguousManagedAppliedIndex(committedByExecuteAt, maxAppliedWriteByExecuteAt, bootstrappedAt());
- if (bootstrappedAt() != null)
+ int appliedBefore = 1 + maxContiguousManagedAppliedIndex(committedByExecuteAt, maxAppliedWriteByExecuteAt, readyAt());
+ if (readyAt() != null)
{
for (int i = 0 ; i < appliedBefore ; ++i)
{
- if (committedByExecuteAt[i].compareTo(bootstrappedAt()) < 0) continue;
+ if (committedByExecuteAt[i].compareTo(readyAt()) < 0) continue;
if (!committedByExecuteAt[i].is(APPLIED))
{
appliedBefore = i;
diff --git a/accord-core/src/main/java/accord/local/cfk/PostProcess.java b/accord-core/src/main/java/accord/local/cfk/PostProcess.java
index 87a88990..42a33034 100644
--- a/accord-core/src/main/java/accord/local/cfk/PostProcess.java
+++ b/accord-core/src/main/java/accord/local/cfk/PostProcess.java
@@ -150,15 +150,15 @@ abstract class PostProcess
* In practice this means that transactions which include a bootstrap range and a range not covered by bootstrap
* will not wait for the bootstrapping key.
*/
- static CommandsForKeyUpdate notifyManagedPreBootstrap(CommandsForKey prev, QuickBounds newBounds, CommandsForKeyUpdate update)
+ static CommandsForKeyUpdate notifyManagedUnready(CommandsForKey prev, QuickBounds newBounds, CommandsForKeyUpdate update)
{
Timestamp maxApplied = null;
TxnId[] notify = NO_TXNIDS;
int notifyCount = 0;
- // <= because maxAppliedWrite is actually maxAppliedOrPreBootstrapWrite
+ // <= because maxAppliedWrite is actually maxAppliedOrUnreadyWrite
for (TxnInfo txn : prev.committedByExecuteAt)
{
- if (txn.compareTo(newBounds.bootstrappedAt) >= 0)
+ if (txn.compareTo(newBounds.readyAt) >= 0)
continue;
if (txn.is(STABLE))
@@ -259,8 +259,8 @@ abstract class PostProcess
{
// TODO (expected): can we relax this to shardRedundantBefore?
TxnId redundantBefore = bounds.gcBefore;
- TxnId bootstrappedAt = bounds.bootstrappedAt;
- if (bootstrappedAt.compareTo(redundantBefore) <= 0) bootstrappedAt = null;
+ TxnId readyAt = bounds.readyAt;
+ if (readyAt.compareTo(redundantBefore) <= 0) readyAt = null;
PostProcess notifier = null;
{
@@ -268,8 +268,8 @@ abstract class PostProcess
Timestamp minUndecided = minUndecidedById < 0 ? Timestamp.MAX : byId[minUndecidedById];
if (!BTree.isEmpty(loadingPruned))
{
- TxnId minLoadingPruned = bootstrappedAt == null ? BTree.findByIndex(loadingPruned, 0)
- : BTree.ceil(loadingPruned, TxnId::compareTo, bootstrappedAt);
+ TxnId minLoadingPruned = readyAt == null ? BTree.findByIndex(loadingPruned, 0)
+ : BTree.ceil(loadingPruned, TxnId::compareTo, readyAt);
minUndecided = Timestamp.nonNullOrMin(minUndecided, minLoadingPruned);
}
@@ -288,7 +288,7 @@ abstract class PostProcess
{
if (newInfo != null && newInfo.is(APPLIED))
{
- TxnInfo maxContiguousApplied = CommandsForKey.maxContiguousManagedApplied(committedByExecuteAt, maxAppliedWriteByExecuteAt, bootstrappedAt);
+ TxnInfo maxContiguousApplied = CommandsForKey.maxContiguousManagedApplied(committedByExecuteAt, maxAppliedWriteByExecuteAt, readyAt);
if (maxContiguousApplied != null && maxContiguousApplied.compareExecuteAt(newInfo) >= 0)
{
Timestamp applyTo = maxContiguousApplied.executeAt;
@@ -323,7 +323,7 @@ abstract class PostProcess
{ // process unmanaged waiting on applies that may now not occur
int start = firstApply;
int end = start;
- int j = 1 + maxContiguousManagedAppliedIndex(committedByExecuteAt, maxAppliedWriteByExecuteAt, bootstrappedAt);
+ int j = 1 + maxContiguousManagedAppliedIndex(committedByExecuteAt, maxAppliedWriteByExecuteAt, readyAt);
boolean hasUnapplied = false;
while (end < unmanageds.length && j < committedByExecuteAt.length)
{
@@ -367,29 +367,29 @@ abstract class PostProcess
rescheduleOrNotifyIf = curInfo.executeAt::equals;
}
- if (isNewBoundsInfo && bootstrappedAt != null)
+ if (isNewBoundsInfo && readyAt != null)
{
- Timestamp maxPreBootstrap;
+ Timestamp maxUnready;
{
- Timestamp tmp = bootstrappedAt;
+ Timestamp tmp = readyAt;
for (TxnInfo txn : byId)
{
- if (txn.compareTo(bootstrappedAt) > 0)
+ if (txn.compareTo(readyAt) > 0)
break;
// while we can in principle exclude all transactions with a lower txnId regardless of their executeAt
// for consistent handling with other transactions we don't leap ahead by executeAt as this permits
// us to also exclude transactions with a higher txnId which is not consistent with other validity checks
// which don't have this additional context
- if (txn.executeAt.compareTo(bootstrappedAt) > 0)
+ if (txn.executeAt.compareTo(readyAt) > 0)
continue;
tmp = Timestamp.nonNullOrMax(tmp, txn.executeAt);
}
- maxPreBootstrap = tmp;
+ maxUnready = tmp;
}
if (rescheduleOrNotifyIf == null)
- rescheduleOrNotifyIf = test -> test.compareTo(maxPreBootstrap) <= 0;
+ rescheduleOrNotifyIf = test -> test.compareTo(maxUnready) <= 0;
else
- rescheduleOrNotifyIf = test -> curInfo.executeAt.equals(test) || test.compareTo(maxPreBootstrap) <= 0;
+ rescheduleOrNotifyIf = test -> curInfo.executeAt.equals(test) || test.compareTo(maxUnready) <= 0;
}
if (rescheduleOrNotifyIf != null)
@@ -415,7 +415,7 @@ abstract class PostProcess
if (predecessor >= 0)
{
- int maxContiguousApplied = maxContiguousManagedAppliedIndex(committedByExecuteAt, maxAppliedWriteByExecuteAt, bootstrappedAt);
+ int maxContiguousApplied = maxContiguousManagedAppliedIndex(committedByExecuteAt, maxAppliedWriteByExecuteAt, readyAt);
if (maxContiguousApplied >= predecessor)
predecessor = -1;
}
diff --git a/accord-core/src/main/java/accord/local/cfk/Pruning.java b/accord-core/src/main/java/accord/local/cfk/Pruning.java
index 00b553a9..0fea7b27 100644
--- a/accord-core/src/main/java/accord/local/cfk/Pruning.java
+++ b/accord-core/src/main/java/accord/local/cfk/Pruning.java
@@ -42,14 +42,14 @@ import static accord.local.cfk.CommandsForKey.InternalStatus.COMMITTED;
import static accord.local.cfk.CommandsForKey.InternalStatus.PRUNED;
import static accord.local.cfk.CommandsForKey.InternalStatus.STABLE;
import static accord.local.cfk.CommandsForKey.appliedBefore;
-import static accord.local.cfk.CommandsForKey.bootstrappedAt;
+import static accord.local.cfk.CommandsForKey.readyAt;
import static accord.local.cfk.CommandsForKey.insertPos;
import static accord.local.cfk.CommandsForKey.managesExecution;
import static accord.local.cfk.CommandsForKey.mayExecute;
import static accord.local.cfk.CommandsForKey.redundantBefore;
import static accord.local.cfk.CommandsForKey.reportLinearizabilityViolations;
import static accord.local.cfk.Updating.nextUndecided;
-import static accord.local.cfk.Updating.recomputeMaxAppliedPreBootstrapWriteById;
+import static accord.local.cfk.Updating.recomputeMaxAppliedUnreadyWriteById;
import static accord.local.cfk.Utils.removeRedundantMissing;
import static accord.primitives.Timestamp.Flag.UNSTABLE;
import static accord.primitives.Txn.Kind.Write;
@@ -191,7 +191,7 @@ public class Pruning
if (BTree.isEmpty(loadingPruned))
return false;
- int startIndex = BTree.ceilIndex(loadingPruned, Timestamp::compareTo, bounds.bootstrappedAt);
+ int startIndex = BTree.ceilIndex(loadingPruned, Timestamp::compareTo, bounds.readyAt);
int endIndex = BTree.ceilIndex(loadingPruned, Timestamp::compareTo, waitingExecuteAt);
// TODO (desired): this is O(n.lg n), whereas we could import the accumulate function and perform in O(max(m, lg n))
for (int i = startIndex; i < endIndex; ++i)
@@ -266,10 +266,10 @@ public class Pruning
newById[count++] = txn;
}
int newMinUndecidedById = nextUndecided(newById, 0, cfk);
- int newMaxAppliedPreBootstrapWriteById = recomputeMaxAppliedPreBootstrapWriteById(cfk.bounds, newById, cfk.maxAppliedPreBootstrapWriteById);
+ int newMaxAppliedUnreadyWriteById = recomputeMaxAppliedUnreadyWriteById(cfk.bounds, newById, cfk.maxAppliedUnreadyWriteById);
int newPrunedBeforeId = cfk.prunedBeforeById - prunedCount;
return new CommandsForKey(cfk.key, cfk.bounds, false, newById,
- newMinUndecidedById, newMaxAppliedPreBootstrapWriteById,
+ newMinUndecidedById, newMaxAppliedUnreadyWriteById,
cfk.committedByExecuteAt, cfk.maxAppliedWriteByExecuteAt,
cfk.maxUniqueHlc, cfk.loadingPruned, newPrunedBeforeId,
cfk.unmanageds, true);
@@ -487,9 +487,9 @@ public class Pruning
cachedAny().forceDiscard(removedExecuteAts, removedExecuteAtCount);
int newMaxAppliedWriteByExecuteAt = cfk.maxAppliedWriteByExecuteAt - removedCommittedCount;
- int newMaxAppliedPreBootstrapWriteById = recomputeMaxAppliedPreBootstrapWriteById(cfk.bounds, newById, cfk.maxAppliedPreBootstrapWriteById);
+ int newMaxAppliedUnreadyWriteById = recomputeMaxAppliedUnreadyWriteById(cfk.bounds, newById, cfk.maxAppliedUnreadyWriteById);
Invariants.require(newById[retainCount] == newPrunedBefore);
- return new CommandsForKey(cfk.key, cfk.bounds, newById, minUndecidedById, newMaxAppliedPreBootstrapWriteById, newCommittedByExecuteAt, newMaxAppliedWriteByExecuteAt, cfk.maxUniqueHlc, cfk.loadingPruned, retainCount, cfk.unmanageds, true);
+ return new CommandsForKey(cfk.key, cfk.bounds, newById, minUndecidedById, newMaxAppliedUnreadyWriteById, newCommittedByExecuteAt, newMaxAppliedWriteByExecuteAt, cfk.maxUniqueHlc, cfk.loadingPruned, retainCount, cfk.unmanageds, true);
}
/**
@@ -535,11 +535,11 @@ public class Pruning
{
TxnId newAppliedBefore = appliedBefore(newBounds);
TxnId newRedundantBefore = redundantBefore(newBounds);
- TxnId newBootstrappedAt = bootstrappedAt(newBounds);
+ TxnId newReadyAt = readyAt(newBounds);
TxnId prevRedundantBefore = redundantBefore(prevBounds);
- TxnId prevBootstrappedAt = bootstrappedAt(prevBounds);
+ TxnId prevReadyAt = readyAt(prevBounds);
Invariants.requireArgument(newRedundantBefore.compareTo(prevRedundantBefore) >= 0, "Expect new RedundantBefore.Entry locallyAppliedOrInvalidatedBefore to be ahead of existing one");
- Invariants.requireArgument(prevBootstrappedAt == null || newRedundantBefore.compareTo(prevBootstrappedAt) >= 0 || (newBootstrappedAt != null && newBootstrappedAt.compareTo(prevBootstrappedAt) >= 0), "Expect new RedundantBefore.Entry bootstrappedAt to be ahead of existing one");
+ Invariants.requireArgument(prevReadyAt == null || newRedundantBefore.compareTo(prevReadyAt) >= 0 || (newReadyAt != null && newReadyAt.compareTo(prevReadyAt) >= 0), "Expect new RedundantBefore.Entry readyAt to be ahead of existing one");
TxnInfo[] newById = byId;
int pos = insertPos(byId, newRedundantBefore);
@@ -549,7 +549,7 @@ public class Pruning
{
if (Invariants.isParanoid() && expectUpToDate && testParanoia(LINEAR, NONE, LOW))
{
- int startPos = prevBootstrappedAt == null ? 0 : insertPos(byId, prevBootstrappedAt);
+ int startPos = prevReadyAt == null ? 0 : insertPos(byId, prevReadyAt);
for (int i = startPos ; i < pos ; ++i)
Invariants.require((byId[i].isNot(COMMITTED) && byId[i].isNot(STABLE)) || !byId[i].mayExecute() || !reportLinearizabilityViolations(), "%s redundant; expected to be applied, undecided or to execute in a future epoch", byId[i]);
}
@@ -610,7 +610,7 @@ public class Pruning
if (newBounds.startEpoch != prevBounds.startEpoch
|| newBounds.endEpoch != prevBounds.endEpoch
- || !newBounds.bootstrappedAt.equals(prevBounds.bootstrappedAt))
+ || !newBounds.readyAt.equals(prevBounds.readyAt))
{
for (int i = 0 ; i < newById.length ; ++i)
{
diff --git a/accord-core/src/main/java/accord/local/cfk/Serialize.java b/accord-core/src/main/java/accord/local/cfk/Serialize.java
index c9e3744d..659aeb2e 100644
--- a/accord-core/src/main/java/accord/local/cfk/Serialize.java
+++ b/accord-core/src/main/java/accord/local/cfk/Serialize.java
@@ -168,9 +168,9 @@ public class Serialize
nodeIds[0] = cfk.redundantBefore().node.id;
nodeIdCount = 1;
{
- TxnId bootstrappedAt = cfk.bootstrappedAt();
- if (bootstrappedAt != null)
- nodeIds[nodeIdCount++] = bootstrappedAt.node.id;
+ TxnId readyAt = cfk.readyAt();
+ if (readyAt != null)
+ nodeIds[nodeIdCount++] = readyAt.node.id;
}
for (int i = 0 ; i < commandCount ; ++i)
{
@@ -326,7 +326,7 @@ public class Serialize
| (executeAtCount > 0 ? HAS_EXECUTE_AT_HEADER_BIT : 0)
| (ballotCount > 0 ? HAS_BALLOT_HEADER_BIT : 0)
| (overrideCount > 0 ? HAS_STATUS_OVERRIDES_HEADER_BIT : 0)
- | (cfk.bootstrappedAt() != null ? HAS_BOOTSTRAPPED_AT_HEADER_BIT : 0)
+ | (cfk.readyAt() != null ? HAS_BOOTSTRAPPED_AT_HEADER_BIT : 0)
| (hasBoundsFlags(cfk) ? HAS_BOUNDS_FLAGS_HEADER_BIT : 0)
| (cfk.hasMaxUniqueHlc() ? HAS_MAX_HLC_HEADER_BIT : 0)
;
@@ -368,7 +368,7 @@ public class Serialize
{
Timestamp redundantBefore = cfk.redundantBefore();
- TxnId bootstrappedAt = cfk.bootstrappedAt();
+ TxnId readyAt = cfk.readyAt();
prevEpoch = redundantBefore.epoch();
prevHlc = redundantBefore.hlc();
{
@@ -384,13 +384,13 @@ public class Serialize
if (0 != (globalFlags & HAS_BOUNDS_FLAGS_HEADER_BIT))
totalBytes += sizeOfUnsignedVInt(redundantBefore.flags());
totalBytes += sizeOfUnsignedVInt(Arrays.binarySearch(nodeIds, 0, nodeIdCount, redundantBefore.node.id));
- if (bootstrappedAt != null)
+ if (readyAt != null)
{
- totalBytes += sizeOfUnsignedVInt(bootstrappedAt.epoch() - prevEpoch);
- totalBytes += sizeOfVInt(bootstrappedAt.hlc() - prevHlc);
+ totalBytes += sizeOfUnsignedVInt(readyAt.epoch() - prevEpoch);
+ totalBytes += sizeOfVInt(readyAt.hlc() - prevHlc);
if (0 != (globalFlags & HAS_BOUNDS_FLAGS_HEADER_BIT))
- totalBytes += sizeOfUnsignedVInt(bootstrappedAt.flags());
- totalBytes += sizeOfUnsignedVInt(Arrays.binarySearch(nodeIds, 0, nodeIdCount, bootstrappedAt.node.id));
+ totalBytes += sizeOfUnsignedVInt(readyAt.flags());
+ totalBytes += sizeOfUnsignedVInt(Arrays.binarySearch(nodeIds, 0, nodeIdCount, readyAt.node.id));
}
if (0 != (globalFlags & HAS_MAX_HLC_HEADER_BIT))
totalBytes += sizeOfVInt(cfk.maxUniqueHlc - prevHlc);
@@ -468,7 +468,7 @@ public class Serialize
{
TxnId redundantBefore = cfk.redundantBefore();
- Timestamp bootstrappedAt = cfk.bootstrappedAt();
+ Timestamp readyAt = cfk.readyAt();
QuickBounds bounds = cfk.bounds();
long start = bounds.startEpoch;
long end = bounds.endEpoch;
@@ -480,13 +480,13 @@ public class Serialize
if (0 != (globalFlags & HAS_BOUNDS_FLAGS_HEADER_BIT))
VIntCoding.writeUnsignedVInt32(redundantBefore.flags(), out);
VIntCoding.writeUnsignedVInt32(Arrays.binarySearch(nodeIds, 0, nodeIdCount, redundantBefore.node.id), out);
- if (bootstrappedAt != null)
+ if (readyAt != null)
{
- VIntCoding.writeUnsignedVInt(bootstrappedAt.epoch() - prevEpoch, out);
- VIntCoding.writeVInt(bootstrappedAt.hlc() - prevHlc, out);
+ VIntCoding.writeUnsignedVInt(readyAt.epoch() - prevEpoch, out);
+ VIntCoding.writeVInt(readyAt.hlc() - prevHlc, out);
if (0 != (globalFlags & HAS_BOUNDS_FLAGS_HEADER_BIT))
- VIntCoding.writeUnsignedVInt(bootstrappedAt.flags(), out);
- VIntCoding.writeUnsignedVInt(Arrays.binarySearch(nodeIds, 0, nodeIdCount, bootstrappedAt.node.id), out);
+ VIntCoding.writeUnsignedVInt(readyAt.flags(), out);
+ VIntCoding.writeUnsignedVInt(Arrays.binarySearch(nodeIds, 0, nodeIdCount, readyAt.node.id), out);
}
if (0 != (globalFlags & HAS_MAX_HLC_HEADER_BIT))
VIntCoding.writeVInt(cfk.maxUniqueHlc - prevHlc, out);
@@ -819,7 +819,7 @@ public class Serialize
long hlc = prevHlc + VIntCoding.readVInt(in);
int flags = ((globalFlags & HAS_BOUNDS_FLAGS_HEADER_BIT) == 0) ? RX_FLAGS : VIntCoding.readUnsignedVInt32(in);
Id node = (Id)nodeIds[VIntCoding.readUnsignedVInt32(in)];
- bounds = bounds.withBootstrappedAtLeast(TxnId.fromValues(epoch, hlc, flags, node));
+ bounds = bounds.withReadyAtLeast(TxnId.fromValues(epoch, hlc, flags, node));
}
if (0 != (globalFlags & HAS_MAX_HLC_HEADER_BIT))
maxUniqueHlc = bounds.gcBefore.hlc() + VIntCoding.readVInt(in);
@@ -1143,9 +1143,9 @@ public class Serialize
private static boolean hasBoundsFlags(CommandsForKey cfk)
{
- Timestamp bootstrappedAt = cfk.bootstrappedAt();
+ Timestamp readyAt = cfk.readyAt();
Timestamp redundantBefore = cfk.redundantBefore();
- return (bootstrappedAt != null && bootstrappedAt.flags() != RX_FLAGS) || (redundantBefore.flags() != RX_FLAGS);
+ return (readyAt != null && readyAt.flags() != RX_FLAGS) || (redundantBefore.flags() != RX_FLAGS);
}
private static final int RX_FLAGS = new TxnId(0, 0, TrackStable.bit(), ExclusiveSyncPoint, Range, Id.NONE).flags();
diff --git a/accord-core/src/main/java/accord/local/cfk/Updating.java b/accord-core/src/main/java/accord/local/cfk/Updating.java
index 31d29804..9c4255d9 100644
--- a/accord-core/src/main/java/accord/local/cfk/Updating.java
+++ b/accord-core/src/main/java/accord/local/cfk/Updating.java
@@ -165,11 +165,11 @@ class Updating
// we don't insert anything before prunedBeforeById (we LoadPruned instead), so we can simply bump it by 0 or 1
int newPrunedBeforeById = advanceByIdIndex(cfk.prunedBeforeById, insertPos, updatePos);
Invariants.paranoid(cfk.prunedBeforeById < 0 || newById[newPrunedBeforeById].equals(byId[cfk.prunedBeforeById]));
- int newMaxAppliedPreBootstrapWriteById = updateMaxAppliedPreBootstrapWriteById(newInfo, insertPos, curInfo == null, cfk, byId, additions, additionCount);
+ int newMaxAppliedUnreadyWriteById = updateMaxAppliedUnreadyWriteById(newInfo, insertPos, curInfo == null, cfk, byId, additions, additionCount);
long newMaxHlc = updateMaxUniqueHlc(cfk, newInfo, command);
cachedTxnIds().forceDiscard(additions, additionAndPrunedCount);
- return LoadPruned.load(prunedIds, cfk.update(newById, newMinUndecidedById, newMaxAppliedPreBootstrapWriteById, newCommittedByExecuteAt, newMaxAppliedWriteByExecuteAt, newMaxHlc, newLoadingPruned, newPrunedBeforeById, curInfo, newInfo));
+ return LoadPruned.load(prunedIds, cfk.update(newById, newMinUndecidedById, newMaxAppliedUnreadyWriteById, newCommittedByExecuteAt, newMaxAppliedWriteByExecuteAt, newMaxHlc, newLoadingPruned, newPrunedBeforeById, curInfo, newInfo));
}
static int advanceByIdIndex(int curIndex, TxnInfo[] byId, int insertPos, int updatePos, TxnId[] additions, int additionCount)
@@ -233,14 +233,14 @@ class Updating
for (int i = 0 ; i < additionsBeforeOldUndecided ; ++i)
{
TxnId addition = additions[i];
- if (!cfk.mayExecute(addition) || cfk.isPreBootstrap(addition))
+ if (!cfk.mayExecute(addition) || cfk.isUnready(addition))
continue;
min = addition;
break;
}
- if ((min == null || min.compareTo(newInfo) > 0) && newInfo.compareTo(COMMITTED) < 0 && cfk.mayExecute(newInfo) && !cfk.isPreBootstrap(newInfo))
+ if ((min == null || min.compareTo(newInfo) > 0) && newInfo.compareTo(COMMITTED) < 0 && cfk.mayExecute(newInfo) && !cfk.isUnready(newInfo))
min = newInfo;
if (min != null && cur != null && cur.compareTo(min) < 0)
@@ -261,31 +261,31 @@ class Updating
return newIndex;
}
- static int updateMaxAppliedPreBootstrapWriteById(TxnInfo newInfo, int insertPos, boolean inserted, CommandsForKey cfk, TxnInfo[] byId, TxnId[] additions, int additionCount)
+ static int updateMaxAppliedUnreadyWriteById(TxnInfo newInfo, int insertPos, boolean inserted, CommandsForKey cfk, TxnInfo[] byId, TxnId[] additions, int additionCount)
{
- int curIndex = cfk.maxAppliedPreBootstrapWriteById;
- if (!newInfo.mayExecute() && newInfo.is(APPLIED) && newInfo.is(Write) && cfk.isPreBootstrap(newInfo) && newInfo.executeAt.compareTo(cfk.bounds.bootstrappedAt) > 0
+ int curIndex = cfk.maxAppliedUnreadyWriteById;
+ if (!newInfo.mayExecute() && newInfo.is(APPLIED) && newInfo.is(Write) && cfk.isUnready(newInfo) && newInfo.executeAt.compareTo(cfk.bounds.readyAt) > 0
&& (curIndex < 0 || newInfo.executeAt.compareTo(byId[curIndex].executeAt) > 0))
return advanceByIdIndexByAdditionsOnly(insertPos, newInfo, additions, additionCount);
return advanceByIdIndex(curIndex, byId, insertPos, inserted, additions, additionCount);
}
- static int recomputeMaxAppliedPreBootstrapWriteById(QuickBounds bounds, TxnInfo[] newById, int maxIndex)
+ static int recomputeMaxAppliedUnreadyWriteById(QuickBounds bounds, TxnInfo[] newById, int maxIndex)
{
maxIndex = Math.min(maxIndex, newById.length);
- int newMaxAppliedPreBootstrapWriteById = -1;
+ int newMaxAppliedUnreadyWriteById = -1;
for (int i = 0 ; i < maxIndex ; ++i)
- newMaxAppliedPreBootstrapWriteById = maybeUpdateMaxAppliedPreBootstrapWriteById(bounds, newById, newMaxAppliedPreBootstrapWriteById, i);
- return newMaxAppliedPreBootstrapWriteById;
+ newMaxAppliedUnreadyWriteById = maybeUpdateMaxAppliedUnreadyWriteById(bounds, newById, newMaxAppliedUnreadyWriteById, i);
+ return newMaxAppliedUnreadyWriteById;
}
- static int maybeUpdateMaxAppliedPreBootstrapWriteById(QuickBounds bounds, TxnInfo[] byId, int newMaxAppliedPreBootstrapWriteById, int i)
+ static int maybeUpdateMaxAppliedUnreadyWriteById(QuickBounds bounds, TxnInfo[] byId, int newMaxAppliedUnreadyWriteById, int i)
{
TxnInfo txn = byId[i];
- if (!txn.mayExecute() && txn.is(APPLIED) && txn.isWrite() && CommandsForKey.isPreBootstrap(txn, bounds) && txn.executeAt.compareTo(bounds.bootstrappedAt) > 0
- && (newMaxAppliedPreBootstrapWriteById < 0 || txn.executeAt.compareTo(byId[newMaxAppliedPreBootstrapWriteById].executeAt) > 0))
+ if (!txn.mayExecute() && txn.is(APPLIED) && txn.isWrite() && CommandsForKey.isUnready(txn, bounds) && txn.executeAt.compareTo(bounds.readyAt) > 0
+ && (newMaxAppliedUnreadyWriteById < 0 || txn.executeAt.compareTo(byId[newMaxAppliedUnreadyWriteById].executeAt) > 0))
return i;
- return newMaxAppliedPreBootstrapWriteById;
+ return newMaxAppliedUnreadyWriteById;
}
static Object computeInfoAndAdditions(CommandsForKey cfk, int insertPos, int updatePos, TxnId plainTxnId, InternalStatus newStatus, boolean mayExecute, Command command)
@@ -495,10 +495,10 @@ class Updating
int newPrunedBeforeById = advanceByIdIndex(cfk.prunedBeforeById, pos, curInfo == null);
Invariants.paranoid(cfk.prunedBeforeById < 0 || newById[newPrunedBeforeById].equals(byId[cfk.prunedBeforeById]));
- int newMaxAppliedPreBootstrapWriteById = updateMaxAppliedPreBootstrapWriteById(newInfo, pos, curInfo == null, cfk, byId, NO_INFOS, 0);
+ int newMaxAppliedUnreadyWriteById = updateMaxAppliedUnreadyWriteById(newInfo, pos, curInfo == null, cfk, byId, NO_INFOS, 0);
long newMaxHlc = updateMaxUniqueHlc(cfk, newInfo, command);
- return cfk.update(newById, newMinUndecidedById, newMaxAppliedPreBootstrapWriteById, newCommittedByExecuteAt, newMaxAppliedWriteByExecuteAt, newMaxHlc, newLoadingPruned, newPrunedBeforeById, curInfo, newInfo);
+ return cfk.update(newById, newMinUndecidedById, newMaxAppliedUnreadyWriteById, newCommittedByExecuteAt, newMaxAppliedWriteByExecuteAt, newMaxHlc, newLoadingPruned, newPrunedBeforeById, curInfo, newInfo);
}
/**
@@ -800,7 +800,7 @@ class Updating
return newMaxAppliedWriteByExecuteAt;
}
}
- else if (newInfo.is(APPLIED) || (cfk.isPreBootstrap(newInfo) && pos - 1 == maxAppliedWriteByExecuteAt))
+ else if (newInfo.is(APPLIED) || (cfk.isUnready(newInfo) && pos - 1 == maxAppliedWriteByExecuteAt))
{
return maybeAdvanceMaxAppliedAndCheckForLinearizabilityViolations(cfk, pos, newInfo.kind(), newInfo);
}
@@ -1060,7 +1060,7 @@ class Updating
if (newById == null) newById = byId;
TxnInfo[] newCommittedByExecuteAt = cfk.committedByExecuteAt;
int newMinUndecidedById = cfk.minUndecidedById;
- int newMaxAppliedPreBootstrapWriteById = cfk.maxAppliedPreBootstrapWriteById;
+ int newMaxAppliedUnreadyWriteById = cfk.maxAppliedUnreadyWriteById;
Object[] newLoadingPruned = cfk.loadingPruned;
TxnId[] prunedIds = NO_TXNIDS;
int clearMissingCount = missingCount;
@@ -1092,7 +1092,7 @@ class Updating
// we can safely use missing[prunedIndex] here because we only fill missing with transactions for which we manage execution
if (minUndecided != null)
newMinUndecidedById = Arrays.binarySearch(newById, minUndecided);
- newMaxAppliedPreBootstrapWriteById = advanceByIdIndex(newMaxAppliedPreBootstrapWriteById, byId, Integer.MAX_VALUE, false, missing, missingCount);
+ newMaxAppliedUnreadyWriteById = advanceByIdIndex(newMaxAppliedUnreadyWriteById, byId, Integer.MAX_VALUE, false, missing, missingCount);
}
}
cachedTxnIds().discard(missing, clearMissingCount);
@@ -1119,7 +1119,7 @@ class Updating
{
int prunedBeforeById = cfk.prunedBeforeById;
Invariants.require(prunedBeforeById < 0 || newById[prunedBeforeById].equals(cfk.prunedBefore()));
- newCfk = new CommandsForKey(cfk.key(), cfk.bounds, newById, newMinUndecidedById, newMaxAppliedPreBootstrapWriteById, newCommittedByExecuteAt, cfk.maxAppliedWriteByExecuteAt, cfk.maxUniqueHlc, newLoadingPruned, prunedBeforeById, newUnmanaged, true);
+ newCfk = new CommandsForKey(cfk.key(), cfk.bounds, newById, newMinUndecidedById, newMaxAppliedUnreadyWriteById, newCommittedByExecuteAt, cfk.maxAppliedWriteByExecuteAt, cfk.maxUniqueHlc, newLoadingPruned, prunedBeforeById, newUnmanaged, true);
}
CommandsForKeyUpdate result = newCfk;
diff --git a/accord-core/src/main/java/accord/local/durability/DurabilityService.java b/accord-core/src/main/java/accord/local/durability/DurabilityService.java
index 26f78b2a..4adfaab7 100644
--- a/accord-core/src/main/java/accord/local/durability/DurabilityService.java
+++ b/accord-core/src/main/java/accord/local/durability/DurabilityService.java
@@ -100,6 +100,7 @@ public class DurabilityService implements ConfigurationService.Listener
{
shards.stop();
global.stop();
+ started = false;
}
public AsyncResult<Void> close(String requestedBy, Txn.Kind kind, Ranges ranges, long timeoutDelay, TimeUnit timeoutUnits)
diff --git a/accord-core/src/main/java/accord/local/durability/GlobalDurability.java b/accord-core/src/main/java/accord/local/durability/GlobalDurability.java
index 883c1feb..b31e18e6 100644
--- a/accord-core/src/main/java/accord/local/durability/GlobalDurability.java
+++ b/accord-core/src/main/java/accord/local/durability/GlobalDurability.java
@@ -35,7 +35,7 @@ import accord.messages.GetDurableBefore;
import accord.messages.GetDurableBefore.DurableBeforeReply;
import accord.messages.SetGloballyDurable;
import accord.topology.Topology;
-import accord.utils.Invariants;
+
import static java.util.concurrent.TimeUnit.MICROSECONDS;
public class GlobalDurability implements Callback<Object>
@@ -110,7 +110,6 @@ public class GlobalDurability implements Callback<Object>
*/
synchronized void start()
{
- Invariants.require(!stop); // cannot currently restart safely
long nowMicros = node.elapsed(MICROSECONDS);
long scheduleAt = computeNextGlobalSyncTime(nowMicros);
scheduled = node.scheduler().selfRecurring(this::run, scheduleAt - nowMicros, MICROSECONDS);
diff --git a/accord-core/src/main/java/accord/local/durability/ShardDurability.java b/accord-core/src/main/java/accord/local/durability/ShardDurability.java
index 77ca52b3..09685981 100644
--- a/accord-core/src/main/java/accord/local/durability/ShardDurability.java
+++ b/accord-core/src/main/java/accord/local/durability/ShardDurability.java
@@ -510,7 +510,6 @@ public class ShardDurability
*/
public synchronized void start()
{
- Invariants.require(!stop); // cannot currently restart safely
scheduled = node.scheduler().recurring(this::tick, shardCycleTimeMicros / targetShardSplits, MICROSECONDS);
if (waitingOnStartup != null)
{
diff --git a/accord-core/src/main/java/accord/messages/Accept.java b/accord-core/src/main/java/accord/messages/Accept.java
index 2566a7e5..f6fa2cb5 100644
--- a/accord-core/src/main/java/accord/messages/Accept.java
+++ b/accord-core/src/main/java/accord/messages/Accept.java
@@ -59,7 +59,7 @@ import static accord.primitives.Known.KnownDeps.DepsKnown;
// TODO (low priority, efficiency): use different objects for send and receive, so can be more efficient
// (e.g. serialize without slicing, and without unnecessary fields)
-public class Accept extends TxnRequest.WithUnsynced<Accept.AcceptReply>
+public class Accept extends RouteRequest.WithUnsynced<Accept.AcceptReply>
{
public static class SerializerSupport
{
@@ -100,10 +100,10 @@ public class Accept extends TxnRequest.WithUnsynced<Accept.AcceptReply>
}
@Override
- public AcceptReply apply(SafeCommandStore safeStore)
+ public AcceptReply applyInternal(SafeCommandStore safeStore)
{
PartialDeps partialDeps = this.partialDeps;
- if (isCancelled()) // check cancellation after reading nullable fields
+ if (ifDoneExpectCancelled()) // check cancellation after reading nullable fields
return null; // we can't throw an exception here else we override any non-exceptional reply informing the reason
StoreParticipants participants = StoreParticipants.update(safeStore, scope, minEpoch, txnId, txnId.epoch(), executeAt.epoch());
@@ -197,7 +197,7 @@ public class Accept extends TxnRequest.WithUnsynced<Accept.AcceptReply>
@Override
public Cancellable submit()
{
- return node.mapReduceConsumeLocal(this, minEpoch, executeAt.epoch(), this);
+ return node.commandStores().mapReduceConsume(minEpoch, executeAt.epoch(), this);
}
@Override
@@ -374,30 +374,28 @@ public class Accept extends TxnRequest.WithUnsynced<Accept.AcceptReply>
}
}
- public static class NotAccept extends AbstractRequest<AcceptReply>
+ public static class NotAccept extends ParticipantsRequest<Participants<?>, AcceptReply>
{
public final Status status;
public final Ballot ballot;
- public final Participants<?> participants;
public NotAccept(Status status, Ballot ballot, TxnId txnId, Participants<?> participants)
{
- super(txnId);
+ super(txnId, participants, txnId.epoch());
this.status = status;
this.ballot = ballot;
- this.participants = participants;
}
@Override
public Cancellable submit()
{
- return node.mapReduceConsumeLocal(this, participants, txnId.epoch(), txnId.epoch(), this);
+ return node.commandStores().mapReduceConsume(txnId.epoch(), txnId.epoch(), this);
}
@Override
- public AcceptReply apply(SafeCommandStore safeStore)
+ public AcceptReply applyInternal(SafeCommandStore safeStore)
{
- StoreParticipants participants = StoreParticipants.notAccept(safeStore, this.participants, txnId);
+ StoreParticipants participants = StoreParticipants.notAccept(safeStore, scope, txnId);
SafeCommand safeCommand = safeStore.get(txnId, participants);
AcceptOutcome outcome = Commands.notAccept(safeStore, safeCommand, status, ballot);
switch (outcome)
@@ -432,7 +430,7 @@ public class Accept extends TxnRequest.WithUnsynced<Accept.AcceptReply>
@Override
public String toString()
{
- return "NotAccept{kind: " + status + ", ballot:" + ballot + ", txnId:" + txnId + ", key:" + participants + '}';
+ return "NotAccept{kind: " + status + ", ballot:" + ballot + ", txnId:" + txnId + ", key:" + scope + '}';
}
@Override
@@ -440,11 +438,5 @@ public class Accept extends TxnRequest.WithUnsynced<Accept.AcceptReply>
{
return status + "{" + txnId + '}';
}
-
- @Override
- public long waitForEpoch()
- {
- return txnId.epoch();
- }
}
}
diff --git a/accord-core/src/main/java/accord/messages/Apply.java b/accord-core/src/main/java/accord/messages/Apply.java
index 7ca0dbbb..070408e0 100644
--- a/accord-core/src/main/java/accord/messages/Apply.java
+++ b/accord-core/src/main/java/accord/messages/Apply.java
@@ -60,7 +60,7 @@ import static accord.primitives.SaveStatus.Applied;
import static accord.primitives.SaveStatus.Applying;
import static accord.primitives.SaveStatus.PreApplied;
-public class Apply extends TxnRequest<ApplyReply>
+public class Apply extends RouteRequest<ApplyReply>
{
public static final Factory FACTORY = Apply::new;
public static class SerializationSupport
@@ -132,13 +132,13 @@ public class Apply extends TxnRequest<ApplyReply>
public Cancellable submit()
{
if (flags.contains(READY_TO_EXECUTE) && fastWritesMayBypassSafeStore() && kind == Kind.Maximal && !txnId.isSyncPoint())
- return node.mapReduceConsumeLocal(scope, minEpoch, maxEpoch, this::applyDirect, this, this);
+ return node.commandStores().mapReduceConsume(minEpoch, maxEpoch, this.overrideWithSynchronousApply(this::applyDirect));
- return node.mapReduceConsumeLocal(this, minEpoch, maxEpoch, this);
+ return node.commandStores().mapReduceConsume(minEpoch, maxEpoch, this);
}
@Override
- public ApplyReply apply(SafeCommandStore safeStore)
+ public ApplyReply applyInternal(SafeCommandStore safeStore)
{
Route<?> route = bestRoute();
StoreParticipants participants = StoreParticipants.execute(safeStore, route, minEpoch, txnId, maxEpoch);
@@ -209,7 +209,7 @@ public class Apply extends TxnRequest<ApplyReply>
PartialDeps deps = this.deps;
Writes writes = this.writes;
Result result = this.result;
- if (isCancelled()) // check cancellation after reading nullable fields
+ if (ifDoneExpectCancelled()) // check cancellation after reading nullable fields
return null;
return apply(newSaveStatus, safeStore, participants, ballot, txn, txnId, executeAt, deps, bestRoute(), writes, result);
@@ -217,7 +217,6 @@ public class Apply extends TxnRequest<ApplyReply>
public static ApplyReply apply(SafeCommandStore safeStore, StoreParticipants participants, Ballot ballot, PartialTxn txn, TxnId txnId, Timestamp executeAt, PartialDeps deps, Route<?> route, Writes writes, Result result)
{
-
return apply(PreApplied, safeStore, participants, ballot, txn, txnId, executeAt, deps, route, writes, result);
}
diff --git a/accord-core/src/main/java/accord/messages/ApplyThenWaitUntilApplied.java b/accord-core/src/main/java/accord/messages/ApplyThenWaitUntilApplied.java
index ebe39b07..cb666926 100644
--- a/accord-core/src/main/java/accord/messages/ApplyThenWaitUntilApplied.java
+++ b/accord-core/src/main/java/accord/messages/ApplyThenWaitUntilApplied.java
@@ -41,7 +41,7 @@ import accord.topology.Topologies;
import accord.utils.UnhandledEnum;
import static accord.messages.MessageType.StandardMessage.APPLY_THEN_WAIT_UNTIL_APPLIED_REQ;
-import static accord.messages.TxnRequest.computeScope;
+import static accord.messages.RouteRequest.computeScope;
/*
* Used by local and global inclusive sync points to effect the sync point at each node
@@ -106,7 +106,7 @@ public class ApplyThenWaitUntilApplied extends WaitUntilApplied
}
@Override
- public CommitOrReadNack apply(SafeCommandStore safeStore)
+ public CommitOrReadNack applyInternal(SafeCommandStore safeStore)
{
PartialTxn txn = this.txn;
PartialDeps deps = this.deps;
@@ -133,7 +133,7 @@ public class ApplyThenWaitUntilApplied extends WaitUntilApplied
// things were applied
break;
}
- return super.apply(safeStore);
+ return super.applyInternal(safeStore);
}
@Override
diff --git a/accord-core/src/main/java/accord/messages/Await.java b/accord-core/src/main/java/accord/messages/Await.java
index d265dbe2..8b171ebc 100644
--- a/accord-core/src/main/java/accord/messages/Await.java
+++ b/accord-core/src/main/java/accord/messages/Await.java
@@ -22,8 +22,7 @@ import java.util.Collection;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
-import java.util.function.Consumer;
-import java.util.function.Predicate;
+import java.util.function.BiPredicate;
import javax.annotation.Nullable;
@@ -36,9 +35,10 @@ import accord.local.Command;
import accord.local.Commands;
import accord.local.Node;
import accord.local.Node.Id;
-import accord.local.PreLoadContext;
+import accord.local.MapReduceConsumeCommandStores;
import accord.local.SafeCommand;
import accord.local.SafeCommandStore;
+import accord.primitives.Ballot;
import accord.primitives.SaveStatus;
import accord.local.StoreParticipants;
import accord.primitives.Participants;
@@ -47,13 +47,14 @@ import accord.primitives.Status.Durability;
import accord.primitives.TxnId;
import accord.topology.Topologies;
import accord.utils.Invariants;
-import accord.utils.MapReduceConsume;
import accord.utils.UnhandledEnum;
+import accord.utils.async.Cancellable;
+import static accord.messages.Await.Until.CommittedOrNotFastPathCommit;
import static accord.messages.MessageType.StandardMessage.ASYNC_AWAIT_COMPLETE_REQ;
import static accord.messages.MessageType.StandardMessage.AWAIT_REQ;
import static accord.messages.MessageType.StandardMessage.AWAIT_RSP;
-import static accord.messages.TxnRequest.computeScope;
+import static accord.messages.RouteRequest.computeScope;
import static accord.primitives.Status.Durability.HasDecisionOrOutcome.None;
import static accord.primitives.Status.Durability.HasDecisionOrOutcome.DurablyStable;
import static accord.primitives.Status.Durability.HasDecisionOrOutcome.DurablyPreApplied;
@@ -67,7 +68,7 @@ import static java.util.concurrent.TimeUnit.MICROSECONDS;
*
* TODO (desired): return an OK message indicating we're waiting synchronously
*/
-public class Await implements Request, MapReduceConsume<SafeCommandStore, Void>, PreLoadContext, LocalListeners.ComplexListener, Timeouts.Timeout
+public class Await extends MapReduceConsumeCommandStores<Participants<?>, Void> implements Request, LocalListeners.ComplexListener, Timeouts.Timeout
{
public static class SerializerSupport
{
@@ -96,7 +97,10 @@ public class Await implements Request, MapReduceConsume<SafeCommandStore, Void>,
* This is used for transactions that may have used the coordinator optimisation.
* TODO (expected): why is additionallyUnblockedBy not set?
*/
- CommittedOrNotFastPathCommit(SaveStatus.Committed, None, null),
+ CommittedOrNotFastPathCommit(SaveStatus.Committed, None, (self, command) -> command.txnId().hasPrivilegedCoordinator()
+ && command.txnId().node.equals(self)
+ && !command.promised().equals(Ballot.ZERO)
+ ),
/**
* Wait for the transaction to be Committed.
@@ -146,14 +150,14 @@ public class Await implements Request, MapReduceConsume<SafeCommandStore, Void>,
private static final Until[] lookupByOrdinal = values();
public final SaveStatus requiredSaveStatus;
public final Durability.HasDecisionOrOutcome requiredDurability;
- public final @Nullable Predicate<SaveStatus> additionallyUnblockedBy;
+ public final @Nullable BiPredicate<Node.Id, Command> additionallyUnblockedBy;
Until(SaveStatus requiredSaveStatus, Durability.HasDecisionOrOutcome requiredDurability)
{
this(requiredSaveStatus, requiredDurability, null);
}
- Until(SaveStatus requiredSaveStatus, Durability.HasDecisionOrOutcome requiredDurability, @Nullable Predicate<SaveStatus> additionallyUnblockedBy)
+ Until(SaveStatus requiredSaveStatus, Durability.HasDecisionOrOutcome requiredDurability, @Nullable BiPredicate<Node.Id, Command> additionallyUnblockedBy)
{
this.requiredSaveStatus = requiredSaveStatus;
this.requiredDurability = requiredDurability;
@@ -189,7 +193,6 @@ public class Await implements Request, MapReduceConsume<SafeCommandStore, Void>,
}
public final TxnId txnId;
- public final Participants<?> scope;
public final Until until;
public final long minAwaitEpoch, maxAwaitEpoch;
public final int callbackId; // < 0 means synchronous await
@@ -215,10 +218,10 @@ public class Await implements Request, MapReduceConsume<SafeCommandStore, Void>,
public Await(Id to, Topologies topologies, TxnId txnId, Participants<?> participants, Until until, int callbackId, boolean notifyProgressLog)
{
+ super(computeScope(to, topologies, participants));
this.txnId = txnId;
this.callbackId = callbackId;
this.notifyProgressLog = notifyProgressLog;
- this.scope = computeScope(to, topologies, participants);
this.until = until;
this.maxAwaitEpoch = topologies.currentEpoch();
this.minAwaitEpoch = topologies.oldestEpoch();
@@ -232,8 +235,8 @@ public class Await implements Request, MapReduceConsume<SafeCommandStore, Void>,
public Await(TxnId txnId, Participants<?> scope, Until until, long minAwaitEpoch, long maxAwaitEpoch, int callbackId, boolean notifyProgressLog)
{
+ super(scope);
this.txnId = txnId;
- this.scope = scope;
this.until = until;
this.minAwaitEpoch = minAwaitEpoch;
this.maxAwaitEpoch = maxAwaitEpoch;
@@ -248,17 +251,19 @@ public class Await implements Request, MapReduceConsume<SafeCommandStore, Void>,
this.node = node;
this.replyTo = replyToNode;
this.replyContext = replyContext;
- node.mapReduceConsumeLocal(this, scope, minAwaitEpoch, maxAwaitEpoch, this);
+ // TODO (expected): integrate with cancellation
+ node.commandStores().mapReduceConsume(minAwaitEpoch, maxAwaitEpoch, this);
}
@Override
- public Void apply(SafeCommandStore safeStore)
+ public Void applyInternal(SafeCommandStore safeStore)
{
StoreParticipants participants = StoreParticipants.update(safeStore, scope, minAwaitEpoch, txnId, maxAwaitEpoch);
SafeCommand safeCommand = safeStore.get(txnId, participants);
Command command = safeCommand.current();
Invariants.require(minAwaitEpoch >= txnId.epoch());
- if (command.saveStatus().compareTo(until.requiredSaveStatus) >= 0)
+ if (command.saveStatus().compareTo(until.requiredSaveStatus) >= 0
+ || (until == CommittedOrNotFastPathCommit && node.id().equals(txnId.node) && !command.promised().equals(Ballot.ZERO)))
{
onNotWaiting(safeStore, safeCommand);
return null;
@@ -424,7 +429,7 @@ public class Await implements Request, MapReduceConsume<SafeCommandStore, Void>,
Command command = safeCommand.current();
SaveStatus saveStatus = command.saveStatus();
return saveStatus.compareTo(until.requiredSaveStatus) >= 0
- || (until.additionallyUnblockedBy != null && until.additionallyUnblockedBy.test(saveStatus));
+ || (until.additionallyUnblockedBy != null && until.additionallyUnblockedBy.test(node.id(), command));
}
protected void onUnavailable()
@@ -441,18 +446,14 @@ public class Await implements Request, MapReduceConsume<SafeCommandStore, Void>,
node.reply(replyTo, replyContext, unavailable ? AwaitOk.Unavailable : AwaitOk.Ready, null);
}
- public static class AsyncAwaitComplete implements Request, PreLoadContext, Consumer<SafeCommandStore>
+ public static class AsyncAwaitComplete extends RouteRequest<Reply>
{
- public final TxnId txnId;
- public final Route<?> route; // at least those Routable we registered with
public final SaveStatus newStatus;
public final int callbackId;
- transient Node.Id from;
public AsyncAwaitComplete(TxnId txnId, Route<?> route, SaveStatus newStatus, int callbackId)
{
- this.txnId = txnId;
- this.route = Invariants.requireArgument(route, route != null);
+ super(txnId, route, txnId.epoch());
this.newStatus = newStatus;
this.callbackId = callbackId;
}
@@ -463,11 +464,11 @@ public class Await implements Request, MapReduceConsume<SafeCommandStore, Void>,
return ASYNC_AWAIT_COMPLETE_REQ;
}
+ @Nullable
@Override
- public void process(Node node, Id from, ReplyContext replyContext)
+ protected Cancellable submit()
{
- this.from = from;
- node.forEachLocal(this, route, txnId.epoch(), Long.MAX_VALUE, this);
+ return node.commandStores().mapReduceConsume(txnId.epoch(), Long.MAX_VALUE, this);
}
@Nullable
@@ -484,14 +485,26 @@ public class Await implements Request, MapReduceConsume<SafeCommandStore, Void>,
}
@Override
- public void accept(SafeCommandStore safeStore)
+ protected void acceptInternal(Reply reply, Throwable failure)
+ {
+ }
+
+ @Override
+ public Reply reduce(Reply o1, Reply o2)
+ {
+ return null;
+ }
+
+ @Override
+ public Reply applyInternal(SafeCommandStore safeStore)
{
SafeCommand safeCommand = safeStore.unsafeGet(txnId);
if (safeCommand == null || safeCommand.current().saveStatus() == SaveStatus.Uninitialised)
- return;
+ return null;
- Commands.updateRoute(safeStore, safeCommand, route);
- safeStore.progressLog().remoteCallback(safeStore, safeCommand, newStatus, callbackId, from);
+ Commands.updateRoute(safeStore, safeCommand, scope);
+ safeStore.progressLog().remoteCallback(safeStore, safeCommand, newStatus, callbackId, node.id());
+ return null;
}
@Override
@@ -499,10 +512,10 @@ public class Await implements Request, MapReduceConsume<SafeCommandStore, Void>,
{
return "AsyncAwaitComplete{" +
"txnId=" + txnId +
- ", route=" + route +
+ ", route=" + scope +
", newStatus=" + newStatus +
", callbackId=" + callbackId +
- ", from=" + from +
+ ", from=" + node.id() +
'}';
}
}
diff --git a/accord-core/src/main/java/accord/messages/BeginInvalidation.java b/accord-core/src/main/java/accord/messages/BeginInvalidation.java
index 69cd462c..867fe104 100644
--- a/accord-core/src/main/java/accord/messages/BeginInvalidation.java
+++ b/accord-core/src/main/java/accord/messages/BeginInvalidation.java
@@ -37,35 +37,32 @@ import static accord.primitives.Route.castToFullRoute;
import static accord.primitives.Route.isFullRoute;
import static accord.utils.Functions.mapReduceNonNull;
-public class BeginInvalidation extends AbstractRequest<BeginInvalidation.InvalidateReply> implements Request, PreLoadContext
+public class BeginInvalidation extends ParticipantsRequest<Participants<?>, BeginInvalidation.InvalidateReply> implements Request, PreLoadContext
{
public final Ballot ballot;
- public final Participants<?> participants;
public BeginInvalidation(Id to, Topologies topologies, TxnId txnId, Participants<?> participants, Ballot ballot)
{
- super(txnId);
- this.participants = participants.slice(topologies.computeRangesForNode(to));
+ super(txnId, participants.slice(topologies.computeRangesForNode(to)), txnId.epoch());
this.ballot = ballot;
}
public BeginInvalidation(TxnId txnId, Participants<?> participants, Ballot ballot)
{
- super(txnId);
- this.participants = participants;
+ super(txnId, participants, txnId.epoch());
this.ballot = ballot;
}
@Override
public Cancellable submit()
{
- return node.mapReduceConsumeLocal(this, participants, txnId.epoch(), txnId.epoch(), this);
+ return node.commandStores().mapReduceConsume(txnId.epoch(), txnId.epoch(), this);
}
@Override
- public InvalidateReply apply(SafeCommandStore safeStore)
+ public InvalidateReply applyInternal(SafeCommandStore safeStore)
{
- StoreParticipants participants = StoreParticipants.notAccept(safeStore, this.participants, txnId);
+ StoreParticipants participants = StoreParticipants.notAccept(safeStore, scope, txnId);
SafeCommand safeCommand = safeStore.get(txnId, participants);
Command command = safeCommand.current();
Ballot supersededBy;
diff --git a/accord-core/src/main/java/accord/messages/BeginRecovery.java b/accord-core/src/main/java/accord/messages/BeginRecovery.java
index c196c4fa..a563fe8a 100644
--- a/accord-core/src/main/java/accord/messages/BeginRecovery.java
+++ b/accord-core/src/main/java/accord/messages/BeginRecovery.java
@@ -67,7 +67,7 @@ import static accord.primitives.Timestamp.Flag.HLC_BOUND;
import static accord.primitives.Txn.Kind.Write;
import static accord.primitives.TxnId.FastPath.PrivilegedCoordinatorWithDeps;
-public class BeginRecovery extends TxnRequest.WithUnsynced<BeginRecovery.RecoverReply>
+public class BeginRecovery extends RouteRequest.WithUnsynced<BeginRecovery.RecoverReply>
{
public static class SerializationSupport
{
@@ -107,11 +107,11 @@ public class BeginRecovery extends TxnRequest.WithUnsynced<BeginRecovery.Recover
@Override
protected Cancellable submit()
{
- return node.mapReduceConsumeLocal(this, minEpoch, executeAtOrTxnIdEpoch, this);
+ return node.commandStores().mapReduceConsume(minEpoch, executeAtOrTxnIdEpoch, this);
}
@Override
- public RecoverReply apply(SafeCommandStore safeStore)
+ public RecoverReply applyInternal(SafeCommandStore safeStore)
{
StoreParticipants participants = StoreParticipants.update(safeStore, route, minEpoch, txnId, executeAtOrTxnIdEpoch);
SafeCommand safeCommand = safeStore.get(txnId, participants);
@@ -373,6 +373,7 @@ public class BeginRecovery extends TxnRequest.WithUnsynced<BeginRecovery.Recover
case IS_NOT_COORD_DEP:
Invariants.requireArgument(testTxnId.is(PrivilegedCoordinatorWithDeps));
+ // TODO (expected): if we are the original coordinator and we know we cannot fast path commit then we should not include this in the reply
ensureLaterCoordRejects().add(keyOrRange, testTxnId);
}
}
diff --git a/accord-core/src/main/java/accord/messages/CheckStatus.java b/accord-core/src/main/java/accord/messages/CheckStatus.java
index 83ff5e97..6465f7d5 100644
--- a/accord-core/src/main/java/accord/messages/CheckStatus.java
+++ b/accord-core/src/main/java/accord/messages/CheckStatus.java
@@ -65,6 +65,7 @@ import static accord.primitives.Known.KnownDeps.DepsKnown;
import static accord.primitives.Known.KnownDeps.DepsUnknown;
import static accord.primitives.Known.KnownExecuteAt.ExecuteAtKnown;
import static accord.primitives.Known.KnownExecuteAt.ExecuteAtProposed;
+import static accord.primitives.Known.Nothing;
import static accord.primitives.Routables.Slice.Minimal;
import static accord.primitives.Status.Durability;
@@ -79,12 +80,11 @@ import static accord.primitives.Status.Durability.HasDecision.DurablyStable;
import static accord.primitives.Status.Durability.HasDecision.FastPathDecided;
import static accord.primitives.Status.Stable;
import static accord.primitives.Status.Truncated;
-import static accord.messages.TxnRequest.computeScope;
import static accord.primitives.WithQuorum.HasQuorum;
import static accord.primitives.Route.castToRoute;
import static accord.primitives.Route.isRoute;
-public class CheckStatus extends AbstractRequest<CheckStatus.CheckStatusReply>
+public class CheckStatus extends ParticipantsRequest<Participants<?>, CheckStatus.CheckStatusReply>
implements Request, PreLoadContext, MapReduceConsume<SafeCommandStore, CheckStatus.CheckStatusReply>
{
public static class SerializationSupport
@@ -115,29 +115,26 @@ public class CheckStatus extends AbstractRequest<CheckStatus.CheckStatusReply>
}
// query is usually a Route
- public final Participants<?> query;
- public final long sourceEpoch;
+ public final Participants<?> scope;
public final IncludeInfo includeInfo;
// if set, simply ensure the ballot on the command is equal or greater to this ballot
public final @Nullable Ballot bumpBallot;
- public CheckStatus(TxnId txnId, Participants<?> query, long sourceEpoch, IncludeInfo includeInfo, @Nullable Ballot bumpBallot)
+ public CheckStatus(TxnId txnId, Participants<?> scope, long sourceEpoch, IncludeInfo includeInfo, @Nullable Ballot bumpBallot)
{
- super(txnId);
+ super(txnId, scope, sourceEpoch);
this.bumpBallot = bumpBallot;
- Invariants.require(txnId.is(query.domain()));
- this.query = query;
- this.sourceEpoch = sourceEpoch;
+ Invariants.require(txnId.is(scope.domain()));
+ this.scope = scope;
this.includeInfo = includeInfo;
}
- public CheckStatus(Id to, Topologies topologies, TxnId txnId, Participants<?> query, long sourceEpoch, IncludeInfo includeInfo, Ballot bumpBallot)
+ public CheckStatus(Id to, Topologies topologies, TxnId txnId, Participants<?> scope, long sourceEpoch, IncludeInfo includeInfo, Ballot bumpBallot)
{
- super(txnId);
+ super(txnId, scope, sourceEpoch);
this.bumpBallot = bumpBallot;
- if (isRoute(query)) this.query = computeScope(to, topologies, castToRoute(query), 0, Route::slice, Route::with);
- else this.query = computeScope(to, topologies, (Participants) query, 0, Participants::slice, Participants::with);
- this.sourceEpoch = sourceEpoch;
+ if (isRoute(scope)) this.scope = computeScope(to, topologies, castToRoute(scope), 0, Route::slice, Route::with);
+ else this.scope = computeScope(to, topologies, (Participants) scope, 0, Participants::slice, Participants::with);
this.includeInfo = includeInfo;
}
@@ -145,13 +142,13 @@ public class CheckStatus extends AbstractRequest<CheckStatus.CheckStatusReply>
public Cancellable submit()
{
// TODO (expected): only contact sourceEpoch
- return node.mapReduceConsumeLocal(this, query, txnId.epoch(), sourceEpoch, this);
+ return node.commandStores().mapReduceConsume(txnId.epoch(), waitForEpoch, this);
}
@Override
- public CheckStatusReply apply(SafeCommandStore safeStore)
+ public CheckStatusReply applyInternal(SafeCommandStore safeStore)
{
- StoreParticipants participants = StoreParticipants.read(safeStore, query, txnId, sourceEpoch);
+ StoreParticipants participants = StoreParticipants.read(safeStore, scope, txnId, waitForEpoch);
SafeCommand safeCommand = safeStore.get(txnId, participants);
Command command = safeCommand.current();
@@ -190,9 +187,9 @@ public class CheckStatus extends AbstractRequest<CheckStatus.CheckStatusReply>
validFor = validFor.intersecting(command.participants().owns(), Minimal);
KnownMap result = KnownMap.create(validFor, saveStatus.known);
- // TODO (expected): consider this case more carefully - should we reply null for minOwned? Should we explicitly handle truncated states?
+ // TODO (required): consider this case more carefully - should we reply null for minOwned? Should we explicitly handle truncated states?
if (validFor != query.owns())
- result = KnownMap.merge(result, KnownMap.create(query.owns(), saveStatus.known.validForAll()));
+ result = KnownMap.merge(result, KnownMap.create(query.owns().without(validFor), saveStatus.known.validForAll()));
return result;
}
@@ -201,7 +198,7 @@ public class CheckStatus extends AbstractRequest<CheckStatus.CheckStatusReply>
if (known.deps().hasProposedOrDecidedDeps())
{
Invariants.require(command.participants().touches().containsAll(command.partialDeps().covering));
- result = KnownMap.create(command.partialDeps().covering, new MinAndMaxKnown(null, Known.Nothing.with(saveStatus.known.deps())));
+ result = KnownMap.create(command.partialDeps().covering, new MinAndMaxKnown(null, Nothing.with(saveStatus.known.deps())));
known = known.with(DepsUnknown);
}
if (known.definition() == DefinitionKnown && !txnId.isSystemTxn())
@@ -498,12 +495,12 @@ public class CheckStatus extends AbstractRequest<CheckStatus.CheckStatusReply>
public Known maxKnown()
{
- return map.foldl(MinAndMaxKnown::nonNullOrMax, Known.Nothing, i -> false);
+ return map.foldl(MinAndMaxKnown::nonNullOrMax, Nothing, i -> false);
}
public Known maxKnown(Unseekables<?> query)
{
- return map.foldl(query, MinAndMaxKnown::nonNullOrMax, Known.Nothing, i -> false);
+ return map.foldl(query, MinAndMaxKnown::nonNullOrMax, Nothing, i -> false);
}
/**
@@ -521,7 +518,7 @@ public class CheckStatus extends AbstractRequest<CheckStatus.CheckStatusReply>
public Known minKnown(Unseekables<?> query)
{
Known known = map.foldlWithDefault(query, MinAndMaxKnown::nonNullOrMin, MinAndMaxKnown.Nothing, null, i -> false);
- return known == null ? Known.Nothing : known;
+ return known == null ? Nothing : known;
}
public Known minMaxKnown(RoutingKey key)
@@ -720,10 +717,4 @@ public class CheckStatus extends AbstractRequest<CheckStatus.CheckStatusReply>
{
return CHECK_STATUS_REQ;
}
-
- @Override
- public long waitForEpoch()
- {
- return sourceEpoch;
- }
}
diff --git a/accord-core/src/main/java/accord/messages/Commit.java b/accord-core/src/main/java/accord/messages/Commit.java
index 4dbfba82..5e7dcf61 100644
--- a/accord-core/src/main/java/accord/messages/Commit.java
+++ b/accord-core/src/main/java/accord/messages/Commit.java
@@ -23,7 +23,6 @@ import accord.local.Commands;
import accord.local.LoadKeys;
import accord.local.Node;
import accord.local.Node.Id;
-import accord.local.PreLoadContext;
import accord.local.RedundantBefore;
import accord.local.SafeCommand;
import accord.local.SafeCommandStore;
@@ -63,7 +62,7 @@ import static accord.messages.Commit.WithTxn.HasTxn;
import static accord.messages.Commit.WithTxn.NoTxn;
import static accord.topology.Topologies.SelectNodeOwnership.SHARE;
-public class Commit extends TxnRequest.WithUnsynced<CommitOrReadNack>
+public class Commit extends RouteRequest.WithUnsynced<CommitOrReadNack>
{
public static class SerializerSupport
{
@@ -150,8 +149,8 @@ public class Commit extends TxnRequest.WithUnsynced<CommitOrReadNack>
public final Ballot ballot;
public final Timestamp executeAt;
// TODO (expected): share keys with partialTxn and partialDeps - in memory and on wire
- public final @Nullable PartialTxn partialTxn;
- public final @Nullable PartialDeps partialDeps;
+ private @Nullable PartialTxn partialTxn;
+ private @Nullable PartialDeps partialDeps;
public final @Nullable FullRoute<?> route;
@@ -180,6 +179,16 @@ public class Commit extends TxnRequest.WithUnsynced<CommitOrReadNack>
this.route = fullRoute;
}
+ public @Nullable PartialTxn partialTxn()
+ {
+ return partialTxn;
+ }
+
+ public @Nullable PartialDeps partialDeps()
+ {
+ return partialDeps;
+ }
+
public static void commitMinimalNoRead(SortedArrayList<Id> contact, Node node, SequentialAsyncExecutor executor, Topologies stabilise, Topologies all, Ballot ballot, TxnId txnId, Txn txn, FullRoute<?> route, Timestamp executeAt, Deps unstableDeps, Callback<ReadReply> callback)
{
Invariants.requireArgument(stabilise.size() == 1, "Invalid coordinate epochs: %s", stabilise);
@@ -198,12 +207,17 @@ public class Commit extends TxnRequest.WithUnsynced<CommitOrReadNack>
@Override
public Cancellable submit()
{
- return node.mapReduceConsumeLocal(this, minEpoch, executeAt.epoch(), this);
+ return node.commandStores().mapReduceConsume(minEpoch, executeAt.epoch(), this);
}
@Override
- public synchronized CommitOrReadNack apply(SafeCommandStore safeStore)
+ public CommitOrReadNack applyInternal(SafeCommandStore safeStore)
{
+ PartialDeps partialDeps = this.partialDeps;
+ PartialTxn partialTxn = this.partialTxn;
+ if (ifDoneExpectCancelled()) // check cancellation after reading nullable fields
+ return null; // we can't throw an exception here else we override any non-exceptional reply informing the reason
+
Route<?> route = this.route != null ? this.route : scope;
StoreParticipants participants = StoreParticipants.execute(safeStore, route, minEpoch, txnId, executeAt.epoch());
SafeCommand safeCommand = safeStore.get(txnId, participants);
@@ -230,7 +244,7 @@ public class Commit extends TxnRequest.WithUnsynced<CommitOrReadNack>
if (route == null) fullRoute = Route.tryCastToFullRoute(safeCommand.current().route());
Invariants.require(route != null);
return safeStore.redundantBefore().foldl(fullRoute, (RedundantBefore.Bounds b, Long v, TxnId id) -> {
- if (b.endEpoch < Long.MAX_VALUE && b.endEpoch <= v && !b.isLocallyRetiredOrPreBootstrap(id))
+ if (b.endEpoch < Long.MAX_VALUE && b.endEpoch <= v && !b.isLocallyRetiredOrUnready(id))
return b.endEpoch - 1;
return v;
}, Long.MAX_VALUE, txnId);
@@ -245,6 +259,8 @@ public class Commit extends TxnRequest.WithUnsynced<CommitOrReadNack>
@Override
protected void acceptInternal(CommitOrReadNack reply, Throwable failure)
{
+ partialDeps = null;
+ partialTxn = null;
if (reply != null || failure != null)
node.reply(replyTo, replyContext, reply, failure);
else if (kind.saveStatus == Committed)
@@ -267,7 +283,7 @@ public class Commit extends TxnRequest.WithUnsynced<CommitOrReadNack>
'}';
}
- public static class Invalidate implements Request, PreLoadContext
+ public static class Invalidate extends ParticipantsRequest<Participants<?>, Reply>
{
public static class SerializerSupport
{
@@ -296,29 +312,44 @@ public class Commit extends TxnRequest.WithUnsynced<CommitOrReadNack>
node.send(commitTo.nodes(), to -> new Invalidate(to, commitTo, txnId, inform));
}
- public final TxnId txnId;
- public final Participants<?> scope;
- public final long waitForEpoch;
public final long invalidateUntilEpoch;
Invalidate(Id to, Topologies topologies, TxnId txnId, Participants<?> scope)
{
- this.txnId = txnId;
- int latestRelevantIndex = latestRelevantEpochIndex(to, topologies, scope);
- this.scope = computeScope(to, topologies, (Participants)scope, latestRelevantIndex, Participants::slice, Participants::with);
- this.waitForEpoch = computeWaitForEpoch(to, topologies, latestRelevantIndex);
+ this(to, topologies, txnId, scope, latestRelevantEpochIndex(to, topologies, scope));
+ }
+
+ private Invalidate(Id to, Topologies topologies, TxnId txnId, Participants<?> scope, int latestRelevantIndex)
+ {
+ super(txnId, computeScope(to, topologies, (Participants)scope, latestRelevantIndex, Participants::slice, Participants::with), computeWaitForEpoch(to, topologies, latestRelevantIndex));
// TODO (expected): make sure we're picking the right upper limit - it can mean future owners that have never witnessed the command are invalidated
this.invalidateUntilEpoch = topologies.currentEpoch();
}
Invalidate(TxnId txnId, Participants<?> scope, long waitForEpoch, long invalidateUntilEpoch)
{
- this.txnId = txnId;
- this.scope = scope;
- this.waitForEpoch = waitForEpoch;
+ super(txnId, scope, waitForEpoch);
this.invalidateUntilEpoch = invalidateUntilEpoch;
}
+ @Nullable
+ @Override
+ protected Cancellable submit()
+ {
+ return node.commandStores().mapReduceConsume(txnId.epoch(), invalidateUntilEpoch, this);
+ }
+
+ @Override
+ protected void acceptInternal(Reply reply, Throwable failure)
+ {
+ }
+
+ @Override
+ public Reply reduce(Reply o1, Reply o2)
+ {
+ return null;
+ }
+
@Override
public TxnId primaryTxnId()
{
@@ -338,14 +369,26 @@ public class Commit extends TxnRequest.WithUnsynced<CommitOrReadNack>
}
@Override
- public void process(Node node, Id from, ReplyContext replyContext)
+ protected boolean supportsPartialRefusal()
+ {
+ return true;
+ }
+
+ @Override
+ protected Reply refuseInternal(SafeCommandStore safeStore)
+ {
+ // don't want to fail to invalidate on other shards because one shard has re-bootstrapped
+ return null;
+ }
+
+ @Override
+ protected Reply applyInternal(SafeCommandStore safeStore)
{
- node.forEachLocal(this, scope, txnId.epoch(), invalidateUntilEpoch, safeStore -> {
- // it's fine for this to operate on a non-participating home key, since invalidation is a terminal state,
- // so it doesn't matter if we resurrect a redundant entry
- StoreParticipants participants = StoreParticipants.notAccept(safeStore, scope, txnId);
- Commands.commitInvalidate(safeStore, safeStore.get(txnId, participants), scope);
- }).begin(node.agent());
+ // it's fine for this to operate on a non-participating home key, since invalidation is a terminal state,
+ // so it doesn't matter if we resurrect a redundant entry
+ StoreParticipants participants = StoreParticipants.notAccept(safeStore, scope, txnId);
+ Commands.commitInvalidate(safeStore, safeStore.get(txnId, participants), scope);
+ return null;
}
@Override
diff --git a/accord-core/src/main/java/accord/messages/GetEphemeralReadDeps.java b/accord-core/src/main/java/accord/messages/GetEphemeralReadDeps.java
index f2ab3659..72d43777 100644
--- a/accord-core/src/main/java/accord/messages/GetEphemeralReadDeps.java
+++ b/accord-core/src/main/java/accord/messages/GetEphemeralReadDeps.java
@@ -37,7 +37,7 @@ import accord.topology.Topologies;
import accord.utils.Invariants;
import accord.utils.async.Cancellable;
-public class GetEphemeralReadDeps extends TxnRequest.WithUnsynced<GetEphemeralReadDeps.GetEphemeralReadDepsOk>
+public class GetEphemeralReadDeps extends RouteRequest.WithUnsynced<GetEphemeralReadDeps.GetEphemeralReadDepsOk>
{
public static final class SerializationSupport
{
@@ -64,11 +64,11 @@ public class GetEphemeralReadDeps extends TxnRequest.WithUnsynced<GetEphemeralRe
@Override
public Cancellable submit()
{
- return node.mapReduceConsumeLocal(this, minEpoch, executionEpoch, this);
+ return node.commandStores().mapReduceConsume(minEpoch, executionEpoch, this);
}
@Override
- public GetEphemeralReadDepsOk apply(SafeCommandStore safeStore)
+ public GetEphemeralReadDepsOk applyInternal(SafeCommandStore safeStore)
{
long latestEpoch = Math.max(safeStore.node().epoch(), node.epoch());
diff --git a/accord-core/src/main/java/accord/messages/GetLatestDeps.java b/accord-core/src/main/java/accord/messages/GetLatestDeps.java
index 082d2ec7..e9993e50 100644
--- a/accord-core/src/main/java/accord/messages/GetLatestDeps.java
+++ b/accord-core/src/main/java/accord/messages/GetLatestDeps.java
@@ -44,7 +44,7 @@ import accord.utils.async.Cancellable;
import static accord.messages.MessageType.StandardMessage.GET_LATEST_DEPS_REQ;
import static accord.messages.MessageType.StandardMessage.GET_LATEST_DEPS_RSP;
-public class GetLatestDeps extends TxnRequest.WithUnsynced<GetLatestDeps.GetLatestDepsReply>
+public class GetLatestDeps extends RouteRequest.WithUnsynced<GetLatestDeps.GetLatestDepsReply>
{
public static final class SerializationSupport
{
@@ -74,11 +74,11 @@ public class GetLatestDeps extends TxnRequest.WithUnsynced<GetLatestDeps.GetLate
@Override
public Cancellable submit()
{
- return node.mapReduceConsumeLocal(this, minEpoch, executeAt.epoch(), this);
+ return node.commandStores().mapReduceConsume(minEpoch, executeAt.epoch(), this);
}
@Override
- public GetLatestDepsReply apply(SafeCommandStore safeStore)
+ public GetLatestDepsReply applyInternal(SafeCommandStore safeStore)
{
StoreParticipants participants = StoreParticipants.read(safeStore, scope, txnId, minEpoch, executeAt.epoch());
SafeCommand safeCommand = safeStore.get(txnId, participants);
diff --git a/accord-core/src/main/java/accord/messages/GetMaxConflict.java b/accord-core/src/main/java/accord/messages/GetMaxConflict.java
index ea1ed311..38ec9cc0 100644
--- a/accord-core/src/main/java/accord/messages/GetMaxConflict.java
+++ b/accord-core/src/main/java/accord/messages/GetMaxConflict.java
@@ -35,7 +35,7 @@ import accord.utils.async.Cancellable;
import static accord.messages.MessageType.StandardMessage.GET_MAX_CONFLICT_REQ;
import static accord.messages.MessageType.StandardMessage.GET_MAX_CONFLICT_RSP;
-public class GetMaxConflict extends TxnRequest.WithUnsynced<GetMaxConflict.GetMaxConflictOk>
+public class GetMaxConflict extends RouteRequest.WithUnsynced<GetMaxConflict.GetMaxConflictOk>
{
public static final class SerializationSupport
{
@@ -62,11 +62,11 @@ public class GetMaxConflict extends TxnRequest.WithUnsynced<GetMaxConflict.GetMa
@Override
public Cancellable submit()
{
- return node.mapReduceConsumeLocal(this, minEpoch, executionEpoch, this);
+ return node.commandStores().mapReduceConsume(minEpoch, executionEpoch, this);
}
@Override
- public GetMaxConflictOk apply(SafeCommandStore safeStore)
+ public GetMaxConflictOk applyInternal(SafeCommandStore safeStore)
{
Timestamp maxConflict = safeStore.commandStore().maxConflict(scope);
return new GetMaxConflictOk(maxConflict, Math.max(safeStore.node().epoch(), node.epoch()));
diff --git a/accord-core/src/main/java/accord/messages/InformDurable.java b/accord-core/src/main/java/accord/messages/InformDurable.java
index 51701df4..8645fc90 100644
--- a/accord-core/src/main/java/accord/messages/InformDurable.java
+++ b/accord-core/src/main/java/accord/messages/InformDurable.java
@@ -45,7 +45,7 @@ import static accord.api.ProtocolModifiers.Toggles.informOfDurability;
import static accord.messages.MessageType.StandardMessage.INFORM_DURABLE_REQ;
import static accord.messages.SimpleReply.Ok;
-public class InformDurable extends TxnRequest<Reply> implements PreLoadContext
+public class InformDurable extends RouteRequest<Reply> implements PreLoadContext
{
public static class SerializationSupport
{
@@ -123,11 +123,11 @@ public class InformDurable extends TxnRequest<Reply> implements PreLoadContext
public Cancellable submit()
{
// TODO (expected): do not load from disk to perform this update, just write a delta to any journal
- return node.mapReduceConsumeLocal(this, scope, minEpoch, maxEpoch, this);
+ return node.commandStores().mapReduceConsume(minEpoch, maxEpoch, this);
}
@Override
- public Reply apply(SafeCommandStore safeStore)
+ public Reply applyInternal(SafeCommandStore safeStore)
{
StoreParticipants participants = StoreParticipants.update(safeStore, scope, minEpoch, txnId, maxEpoch);
SafeCommand safeCommand = safeStore.get(txnId, participants);
diff --git a/accord-core/src/main/java/accord/messages/AbstractRequest.java b/accord-core/src/main/java/accord/messages/NoWaitRequest.java
similarity index 86%
rename from accord-core/src/main/java/accord/messages/AbstractRequest.java
rename to accord-core/src/main/java/accord/messages/NoWaitRequest.java
index 2d1ae568..fa58359e 100644
--- a/accord-core/src/main/java/accord/messages/AbstractRequest.java
+++ b/accord-core/src/main/java/accord/messages/NoWaitRequest.java
@@ -25,16 +25,15 @@ import javax.annotation.Nullable;
import accord.api.Timeouts;
import accord.api.Timeouts.RegisteredTimeout;
import accord.local.Node;
-import accord.local.PreLoadContext;
-import accord.local.SafeCommandStore;
+import accord.local.MapReduceConsumeCommandStores;
+import accord.primitives.Participants;
import accord.primitives.TxnId;
import accord.utils.Invariants;
-import accord.utils.MapReduceConsume;
import accord.utils.async.Cancellable;
import static java.util.concurrent.TimeUnit.MICROSECONDS;
-public abstract class AbstractRequest<R extends Reply> implements PreLoadContext, Request, MapReduceConsume<SafeCommandStore, R>, Timeouts.Timeout
+public abstract class NoWaitRequest<P extends Participants<?>, R extends Reply> extends MapReduceConsumeCommandStores<P, R> implements Request, Timeouts.Timeout
{
private static class Cancellation implements Cancellable
{
@@ -68,10 +67,11 @@ public abstract class AbstractRequest<R extends Reply> implements PreLoadContext
private boolean hasSentFinalReply;
private transient volatile Cancellation cancellation;
- private static final AtomicReferenceFieldUpdater<AbstractRequest, Cancellation> cancellationUpdater = AtomicReferenceFieldUpdater.newUpdater(AbstractRequest.class, Cancellation.class, "cancellation");
+ private static final AtomicReferenceFieldUpdater<NoWaitRequest, Cancellation> cancellationUpdater = AtomicReferenceFieldUpdater.newUpdater(NoWaitRequest.class, Cancellation.class, "cancellation");
- protected AbstractRequest(TxnId txnId)
+ protected NoWaitRequest(TxnId txnId, P scope)
{
+ super(scope);
this.txnId = txnId;
}
@@ -105,20 +105,23 @@ public abstract class AbstractRequest<R extends Reply> implements PreLoadContext
return cancellation == CANCEL;
}
+ protected boolean ifDoneExpectCancelled()
+ {
+ if (!isDone())
+ return false;
+ Invariants.require(cancellation == CANCEL);
+ return true;
+ }
+
protected abstract Cancellable submit();
@Override
public final void accept(R reply, Throwable failure)
{
- cleanup(processedInternal());
+ cleanup(failure == null ? clearInternal() : cancelInternal());
acceptInternal(reply, failure);
}
- protected @Nullable Cancellable processedInternal()
- {
- return clearInternal().timeout;
- }
-
protected void acceptInternal(R reply, Throwable failure)
{
if (failure != null || reply.isFinal())
@@ -126,6 +129,7 @@ public abstract class AbstractRequest<R extends Reply> implements PreLoadContext
Invariants.require(!hasSentFinalReply);
hasSentFinalReply = true;
}
+ if (failure != null) cancel();
node.reply(replyTo, replyContext, reply, failure);
}
@@ -184,6 +188,11 @@ public abstract class AbstractRequest<R extends Reply> implements PreLoadContext
}
}
+ public ReplyContext replyContext()
+ {
+ return replyContext;
+ }
+
@Override
public R reduce(R o1, R o2)
{
diff --git a/accord-core/src/main/java/accord/messages/TxnRequest.java b/accord-core/src/main/java/accord/messages/ParticipantsRequest.java
similarity index 68%
rename from accord-core/src/main/java/accord/messages/TxnRequest.java
rename to accord-core/src/main/java/accord/messages/ParticipantsRequest.java
index ec171079..970c9f42 100644
--- a/accord-core/src/main/java/accord/messages/TxnRequest.java
+++ b/accord-core/src/main/java/accord/messages/ParticipantsRequest.java
@@ -19,10 +19,8 @@
package accord.messages;
import java.util.function.BiFunction;
-
import javax.annotation.Nullable;
-import accord.local.Node;
import accord.local.Node.Id;
import accord.local.PreLoadContext;
import accord.local.SafeCommandStore;
@@ -42,58 +40,14 @@ import accord.utils.async.Cancellable;
import static accord.topology.Shard.Flag.MUST_WITNESS;
import static accord.utils.Invariants.illegalArgument;
-public abstract class TxnRequest<R extends Reply> extends AbstractRequest<R> implements Request, PreLoadContext, MapReduceConsume<SafeCommandStore, R>
+public abstract class ParticipantsRequest<P extends Participants<?>, R extends Reply> extends NoWaitRequest<P, R> implements Request, PreLoadContext, MapReduceConsume<SafeCommandStore, R>
{
- public static abstract class WithUnsynced<R extends Reply> extends TxnRequest<R>
- {
- public final long minEpoch; // TODO (low priority, clarity): can this just always be TxnId.epoch?
-
- public WithUnsynced(Id to, Topologies topologies, TxnId txnId, Route<?> route)
- {
- this(to, topologies, txnId, route, latestRelevantEpochIndex(to, topologies, route));
- }
-
- public WithUnsynced(Id to, Topologies topologies, Route<?> route)
- {
- this(to, topologies, TxnId.NONE, route, latestRelevantEpochIndex(to, topologies, route));
- }
-
- protected WithUnsynced(Id to, Topologies topologies, TxnId txnId, Route<?> route, int startIndex)
- {
- super(to, topologies, route, txnId, startIndex);
- this.minEpoch = topologies.oldestEpoch();
- }
-
- protected WithUnsynced(TxnId txnId, Route<?> scope, long waitForEpoch, long minEpoch)
- {
- super(txnId, scope, waitForEpoch);
- this.minEpoch = minEpoch;
- }
- }
-
- public final Route<?> scope;
public final long waitForEpoch;
- public ReplyContext replyContext()
+ public ParticipantsRequest(TxnId txnId, P scope, long waitForEpoch)
{
- return replyContext;
- }
-
- public TxnRequest(Node.Id to, Topologies topologies, Route<?> route, TxnId txnId)
- {
- this(to, topologies, route, txnId, latestRelevantEpochIndex(to, topologies, route));
- }
-
- public TxnRequest(Node.Id to, Topologies topologies, Route<?> route, TxnId txnId, int startIndex)
- {
- this(txnId, computeScope(to, topologies, route, startIndex), computeWaitForEpoch(to, topologies, startIndex));
- }
-
- public TxnRequest(TxnId txnId, Route<?> scope, long waitForEpoch)
- {
- super(txnId);
+ super(txnId, scope);
Invariants.require(!scope.isEmpty());
- this.scope = scope;
this.waitForEpoch = waitForEpoch;
}
@@ -101,7 +55,7 @@ public abstract class TxnRequest<R extends Reply> extends AbstractRequest<R> imp
* The portion of the complete Route that this TxnRequest applies to. Should represent the complete
* range owned by the target node for the involved epochs.
*/
- public Route<?> scope()
+ public P scope()
{
return scope;
}
@@ -125,8 +79,10 @@ public abstract class TxnRequest<R extends Reply> extends AbstractRequest<R> imp
protected abstract @Nullable Cancellable submit();
+ protected abstract R applyInternal(SafeCommandStore safeStore);
+
// finds the first topology index that intersects with the node
- protected static int latestRelevantEpochIndex(Node.Id node, Topologies topologies, Unseekables<?> route)
+ protected static int latestRelevantEpochIndex(Id node, Topologies topologies, Unseekables<?> route)
{
if (topologies.size() == 1)
return 0;
@@ -171,12 +127,12 @@ public abstract class TxnRequest<R extends Reply> extends AbstractRequest<R> imp
* on the assumption that this might also mean some local shard rearrangement
* (ignoring the case where the latest epochs do not intersect the keys at all)
*/
- public static long computeWaitForEpoch(Node.Id node, Topologies topologies, Unseekables<?> scope)
+ public static long computeWaitForEpoch(Id node, Topologies topologies, Unseekables<?> scope)
{
return computeWaitForEpoch(node, topologies, latestRelevantEpochIndex(node, topologies, scope));
}
- public static long computeWaitForEpoch(Node.Id node, Topologies topologies, int startIndex)
+ public static long computeWaitForEpoch(Id node, Topologies topologies, int startIndex)
{
int i = Math.max(1, startIndex);
int mi = topologies.size();
@@ -212,33 +168,33 @@ public abstract class TxnRequest<R extends Reply> extends AbstractRequest<R> imp
return topologies.get(i - 1).epoch();
}
- public static Route<?> computeScope(Node.Id node, Topologies topologies, FullRoute<?> fullRoute)
+ public static Route<?> computeScope(Id node, Topologies topologies, FullRoute<?> fullRoute)
{
return computeScope(node, topologies, fullRoute, latestRelevantEpochIndex(node, topologies, fullRoute));
}
- public static Participants<?> computeScope(Node.Id node, Topologies topologies, Participants<?> participants)
+ public static Participants<?> computeScope(Id node, Topologies topologies, Participants<?> participants)
{
return computeScope(node, topologies, participants, latestRelevantEpochIndex(node, topologies, participants));
}
- public static Route<?> computeScope(Node.Id node, Topologies topologies, Route<?> route)
+ public static Route<?> computeScope(Id node, Topologies topologies, Route<?> route)
{
return computeScope(node, topologies, route, latestRelevantEpochIndex(node, topologies, route));
}
- public static Route<?> computeScope(Node.Id node, Topologies topologies, Route<?> route, int startIndex)
+ public static Route<?> computeScope(Id node, Topologies topologies, Route<?> route, int startIndex)
{
return computeScope(node, topologies, route, startIndex, Route::slice, Route::with);
}
- public static Participants<?> computeScope(Node.Id node, Topologies topologies, Participants<?> route, int startIndex)
+ public static Participants<?> computeScope(Id node, Topologies topologies, Participants<?> route, int startIndex)
{
return computeScope(node, topologies, route, startIndex, Participants::slice, Participants::with);
}
// TODO (low priority, clarity): move to Topologies
- public static <I, O> O computeScope(Node.Id node, Topologies topologies, I keys, int startIndex, BiFunction<I, Ranges, O> slice, BiFunction<O, O, O> merge)
+ public static <I, O> O computeScope(Id node, Topologies topologies, I keys, int startIndex, BiFunction<I, Ranges, O> slice, BiFunction<O, O, O> merge)
{
Ranges last = null;
O scope = null;
diff --git a/accord-core/src/main/java/accord/messages/PreAccept.java b/accord-core/src/main/java/accord/messages/PreAccept.java
index 0711fda1..53bcb457 100644
--- a/accord-core/src/main/java/accord/messages/PreAccept.java
+++ b/accord-core/src/main/java/accord/messages/PreAccept.java
@@ -35,7 +35,7 @@ import accord.local.SafeCommand;
import accord.local.SafeCommandStore;
import accord.primitives.PartialDeps;
import accord.local.StoreParticipants;
-import accord.messages.TxnRequest.WithUnsynced;
+import accord.messages.RouteRequest.WithUnsynced;
import accord.primitives.Deps;
import accord.primitives.FullRoute;
import accord.primitives.PartialTxn;
@@ -110,11 +110,11 @@ public class PreAccept extends WithUnsynced<PreAccept.PreAcceptReply>
@Override
protected Cancellable submit()
{
- return node.mapReduceConsumeLocal(this, minEpoch, acceptEpoch, this);
+ return node.commandStores().mapReduceConsume(minEpoch, acceptEpoch, this);
}
@Override
- public PreAcceptReply apply(SafeCommandStore safeStore)
+ public PreAcceptReply applyInternal(SafeCommandStore safeStore)
{
StoreParticipants participants = StoreParticipants.update(safeStore, route, minEpoch, txnId, acceptEpoch);
SafeCommand safeCommand = safeStore.get(txnId, participants);
diff --git a/accord-core/src/main/java/accord/messages/Propagate.java b/accord-core/src/main/java/accord/messages/Propagate.java
index 8d510f62..4a9bd59a 100644
--- a/accord-core/src/main/java/accord/messages/Propagate.java
+++ b/accord-core/src/main/java/accord/messages/Propagate.java
@@ -29,7 +29,7 @@ import accord.local.CommandStores.LatentStoreSelector;
import accord.local.CommandStores.StoreSelector;
import accord.local.Commands;
import accord.local.Node;
-import accord.local.PreLoadContext;
+import accord.local.MapReduceConsumeCommandStores;
import accord.local.RedundantStatus;
import accord.local.SafeCommand;
import accord.local.SafeCommandStore;
@@ -51,7 +51,6 @@ import accord.primitives.TxnId;
import accord.primitives.Unseekables;
import accord.primitives.Writes;
import accord.utils.Invariants;
-import accord.utils.MapReduceConsume;
import accord.utils.UnhandledEnum;
import javax.annotation.Nullable;
@@ -78,11 +77,10 @@ import static accord.primitives.WithQuorum.HasQuorum;
import static accord.utils.Invariants.illegalState;
// TODO (required): detect propagate loops where we don't manage to update anything but should
-public class Propagate implements PreLoadContext, MapReduceConsume<SafeCommandStore, Void>
+public class Propagate extends MapReduceConsumeCommandStores<Route<?>, Void>
{
final Node node;
final TxnId txnId;
- final Route<?> route;
final Known target;
@Nullable final Known foundAtLeast; // the minimum we found for the keys we queried
final InvalidIf invalidIf;
@@ -126,9 +124,9 @@ public class Propagate implements PreLoadContext, MapReduceConsume<SafeCommandSt
@Nullable Result result,
BiConsumer<? super FetchResult, Throwable> callback, @Nullable Tracing tracing)
{
+ super(route);
this.node = node;
this.txnId = txnId;
- this.route = route;
this.target = target;
this.foundAtLeast = foundAtLeast;
this.invalidIf = invalidIf;
@@ -179,7 +177,7 @@ public class Propagate implements PreLoadContext, MapReduceConsume<SafeCommandSt
untilEpoch = Math.max(untilEpoch, committedExecuteAt.epoch());
StoreSelector selector = reportTo.refine(txnId, committedExecuteAt, route);
- node.withEpochAtLeast(untilEpoch, null, propagate, () -> node.mapReduceConsumeLocal(propagate, selector, propagate));
+ node.withEpochAtLeast(untilEpoch, null, propagate, () -> node.commandStores().mapReduceConsume(selector, propagate));
}
@Override
@@ -195,11 +193,11 @@ public class Propagate implements PreLoadContext, MapReduceConsume<SafeCommandSt
}
@Override
- public Void apply(SafeCommandStore safeStore)
+ public Void applyInternal(SafeCommandStore safeStore)
{
long executeAtEpoch = committedExecuteAt == null ? txnId.epoch() : committedExecuteAt.epoch();
- long lowEpoch = StoreParticipants.computePropagateLowEpoch(safeStore, txnId, route);
- StoreParticipants participants = StoreParticipants.update(safeStore, route, lowEpoch, txnId, executeAtEpoch, executeAtEpoch, committedExecuteAt != null);
+ long lowEpoch = StoreParticipants.computePropagateLowEpoch(safeStore, txnId, scope);
+ StoreParticipants participants = StoreParticipants.update(safeStore, scope, lowEpoch, txnId, executeAtEpoch, executeAtEpoch, committedExecuteAt != null);
// TODO (expected): can we come up with a better more universal pattern for avoiding updating a command we don't intersect with?
// ideally integrated with safeStore.get()
if (participants.owns().isEmpty() && safeStore.ifInitialised(txnId) == null)
@@ -216,7 +214,7 @@ public class Propagate implements PreLoadContext, MapReduceConsume<SafeCommandSt
if (participants.executes() == null && executeAtIfKnown != null)
{
executeAtEpoch = executeAtIfKnown.epoch();
- participants = StoreParticipants.update(safeStore, route, lowEpoch, txnId, executeAtEpoch, executeAtEpoch, true);
+ participants = StoreParticipants.update(safeStore, scope, lowEpoch, txnId, executeAtEpoch, executeAtEpoch, true);
}
switch (command.saveStatus().phase)
@@ -289,15 +287,24 @@ public class Propagate implements PreLoadContext, MapReduceConsume<SafeCommandSt
SaveStatus propagate = found.atLeast(currentlyKnown).propagatesSaveStatus();
if (propagate.known.isSatisfiedBy(currentlyKnown))
{
- if (invalidIf == IfUncommitted)
+ if (found.is(Outcome.Erased) && participants.stillTouches().isEmpty())
+ {
+ if (tracing != null)
+ tracing.trace(safeStore.commandStore(), "No longer interact with this command, and received erased response. Erasing.");
+ purge(safeStore, safeCommand, command, participants, ERASE, true);
+ }
+ else if (invalidIf == IfUncommitted)
{
if (tracing != null)
tracing.trace(safeStore.commandStore(), "Marking invalidIfUncommitted");
safeStore.progressLog().invalidIfUncommitted(txnId);
}
+ else
+ {
+ if (tracing != null)
+ tracing.trace(safeStore.commandStore(), "Already know at least as much as peer responses.");
+ }
- if (tracing != null)
- tracing.trace(safeStore.commandStore(), "Already know at least as much as peer responses.");
updateFetchResult(found, participants.owns());
return updateDurability(safeStore, safeCommand, participants);
}
@@ -349,7 +356,7 @@ public class Propagate implements PreLoadContext, MapReduceConsume<SafeCommandSt
case PreAccepted:
// only preaccept if we coordinate the transaction
- if (safeStore.ranges().coordinates(txnId).intersects(route) && Route.isFullRoute(route))
+ if (safeStore.ranges().coordinates(txnId).intersects(scope) && Route.isFullRoute(scope))
{
if (tracing != null)
tracing.trace(safeStore.commandStore(), "Pre-accepting");
@@ -362,6 +369,12 @@ public class Propagate implements PreLoadContext, MapReduceConsume<SafeCommandSt
return updateDurability(safeStore, safeCommand, participants);
}
+ @Override
+ protected Void refuseInternal(SafeCommandStore safeStore)
+ {
+ return null;
+ }
+
private void updateFetchResult(Known found, Participants<?> owns)
{
Known achieved = found.propagates();
@@ -385,7 +398,7 @@ public class Propagate implements PreLoadContext, MapReduceConsume<SafeCommandSt
{
FetchResult current = fetchResult;
if (current == null)
- return new FetchResult(target, route.slice(0, 0), foundAtLeast);
+ return new FetchResult(target, scope.slice(0, 0), foundAtLeast);
return current;
}
diff --git a/accord-core/src/main/java/accord/messages/ReadData.java b/accord-core/src/main/java/accord/messages/ReadData.java
index 31674c8b..a5be06f0 100644
--- a/accord-core/src/main/java/accord/messages/ReadData.java
+++ b/accord-core/src/main/java/accord/messages/ReadData.java
@@ -35,8 +35,8 @@ import accord.local.Command;
import accord.local.Command.Committed;
import accord.local.CommandStore;
import accord.local.CommandStores;
+import accord.local.MapReduceConsumeCommandStores;
import accord.local.Node;
-import accord.local.PreLoadContext;
import accord.local.SafeCommand;
import accord.local.SafeCommandStore;
import accord.primitives.RoutingKeys;
@@ -51,7 +51,6 @@ import accord.primitives.TxnId;
import accord.primitives.Unseekables;
import accord.topology.Topologies;
import accord.utils.Invariants;
-import accord.utils.MapReduceConsume;
import accord.utils.UnhandledEnum;
import accord.utils.async.AsyncChain;
import accord.utils.async.AsyncChains;
@@ -68,7 +67,7 @@ import static accord.coordinate.ExecuteFlag.HAS_UNIQUE_HLC;
import static accord.coordinate.ExecuteFlag.READY_TO_EXECUTE;
import static accord.messages.MessageType.StandardMessage.READ_RSP;
import static accord.messages.ReadData.CommitOrReadNack.Kind.InsufficientEpochs;
-import static accord.messages.TxnRequest.latestRelevantEpochIndex;
+import static accord.messages.RouteRequest.latestRelevantEpochIndex;
import static accord.primitives.Routables.Slice.Minimal;
import static accord.primitives.Txn.Kind.EphemeralRead;
import static accord.primitives.Txn.Kind.Write;
@@ -79,7 +78,7 @@ import static java.util.concurrent.TimeUnit.MICROSECONDS;
// TODO (required): (v1.1) if one shard times out waiting to reply, but another shard produces a reply, return a partial response (or response with suitably populated unavailable)
// this means timing out a little earlier so the reply has time to arrive (but this should anyway be the case).
// This ensures a multi-key transaction interacting with replicas that have different stale shards on a replica can make progress
-public abstract class ReadData implements PreLoadContext, Request, MapReduceConsume<SafeCommandStore, ReadData.CommitOrReadNack>, LocalListeners.ComplexListener, Timeouts.Timeout
+public abstract class ReadData extends MapReduceConsumeCommandStores<Participants<?>, ReadData.CommitOrReadNack> implements Request, LocalListeners.ComplexListener, Timeouts.Timeout
{
private static final Logger logger = LoggerFactory.getLogger(ReadData.class);
@@ -134,7 +133,6 @@ public abstract class ReadData implements PreLoadContext, Request, MapReduceCons
}
public final TxnId txnId;
- public final Participants<?> scope;
public final long executeAtEpoch;
public final ExecuteFlags flags;
final boolean requiresListenersDuringExecution;
@@ -164,10 +162,9 @@ public abstract class ReadData implements PreLoadContext, Request, MapReduceCons
public ReadData(Node.Id to, Topologies topologies, TxnId txnId, Participants<?> scope, @Nullable Txn txn, @Nullable Timestamp executeAt, long executeAtEpoch, ExecuteFlags flags)
{
+ super(RouteRequest.computeScope(to, topologies, scope, latestRelevantEpochIndex(to, topologies, scope), Participants::slice, Participants::with));
this.txnId = txnId;
this.flags = flags;
- int startIndex = latestRelevantEpochIndex(to, topologies, scope);
- this.scope = TxnRequest.computeScope(to, topologies, scope, startIndex, Participants::slice, Participants::with);
this.partialTxn = txn == null ? null : txn.intersecting(scope, true);
this.executeAt = executeAt;
this.executeAtEpoch = executeAtEpoch;
@@ -181,8 +178,8 @@ public abstract class ReadData implements PreLoadContext, Request, MapReduceCons
protected ReadData(TxnId txnId, Participants<?> scope, @Nullable PartialTxn partialTxn, @Nullable Timestamp executeAt, long executeAtEpoch, ExecuteFlags flags)
{
+ super(scope);
this.txnId = txnId;
- this.scope = scope;
this.partialTxn = partialTxn;
this.executeAt = executeAt;
this.executeAtEpoch = executeAtEpoch;
@@ -268,9 +265,9 @@ public abstract class ReadData implements PreLoadContext, Request, MapReduceCons
stamp = node.currentStamp();
if (flags.contains(READY_TO_EXECUTE) && fastReadsMayBypassSafeStore(txnId) && partialTxn != null && executeAt != null && mayFastExecute() && (txnId.is(EphemeralRead) || flags.contains(HAS_UNIQUE_HLC)))
- return node.commandStores().mapReduceConsume(scope, minEpoch(), executeAtEpoch, this::applyFastRead, this, this);
+ return node.commandStores().mapReduceConsume(minEpoch(), executeAtEpoch, this.overrideWithSynchronousApply(this::applyFastRead));
- return node.mapReduceConsumeLocal(this, scope, minEpoch(), executeAtEpoch, this);
+ return node.commandStores().mapReduceConsume(minEpoch(), executeAtEpoch, this);
}
@Override
@@ -285,15 +282,23 @@ public abstract class ReadData implements PreLoadContext, Request, MapReduceCons
else return a.minEpoch <= b.minEpoch ? a : b;
}
- @Override
- public CommitOrReadNack apply(SafeCommandStore safeStore)
+ protected CommitOrReadNack applyInternal(SafeCommandStore safeStore)
{
StoreParticipants participants = StoreParticipants.execute(safeStore, scope, txnId, minEpoch(), executeAtEpoch);
SafeCommand safeCommand = safeStore.get(txnId, participants);
- return apply(safeStore, safeCommand, participants);
+ return applyInternal(safeStore, safeCommand, participants);
}
- protected CommitOrReadNack apply(SafeCommandStore safeStore, SafeCommand safeCommand, StoreParticipants participants)
+ protected CommitOrReadNack refuseInternal(SafeCommandStore safeStore)
+ {
+ synchronized (this)
+ {
+ updateUnavailable(safeStore.ranges().allAt(executeAtEpoch));
+ }
+ return null;
+ }
+
+ protected CommitOrReadNack applyInternal(SafeCommandStore safeStore, SafeCommand safeCommand, StoreParticipants participants)
{
synchronized (this)
{
@@ -692,10 +697,15 @@ public abstract class ReadData implements PreLoadContext, Request, MapReduceCons
cancel.cancel();
}
- public void timeout()
+ public final void timeout()
+ {
+ timeoutInternal();
+ }
+
+ protected boolean timeoutInternal()
{
timeout = null;
- cancel();
+ return cancel();
}
public int stripe()
diff --git a/accord-core/src/main/java/accord/messages/ReadEphemeralTxnData.java b/accord-core/src/main/java/accord/messages/ReadEphemeralTxnData.java
index 2af8b03e..49f00424 100644
--- a/accord-core/src/main/java/accord/messages/ReadEphemeralTxnData.java
+++ b/accord-core/src/main/java/accord/messages/ReadEphemeralTxnData.java
@@ -45,8 +45,8 @@ import org.agrona.collections.IntHashSet;
import static accord.local.Commands.eraseEphemeralRead;
import static accord.messages.MessageType.StandardMessage.READ_EPHEMERAL_REQ;
-import static accord.messages.TxnRequest.computeScope;
-import static accord.messages.TxnRequest.latestRelevantEpochIndex;
+import static accord.messages.RouteRequest.computeScope;
+import static accord.messages.RouteRequest.latestRelevantEpochIndex;
public class ReadEphemeralTxnData extends ReadData
{
@@ -90,18 +90,18 @@ public class ReadEphemeralTxnData extends ReadData
}
@Override
- public CommitOrReadNack apply(SafeCommandStore safeStore)
+ public CommitOrReadNack applyInternal(SafeCommandStore safeStore)
{
StoreParticipants participants = StoreParticipants.execute(safeStore, route, txnId, minEpoch(), executeAtEpoch);
SafeCommand safeCommand = safeStore.get(txnId, participants);
- return apply(safeStore, safeCommand, participants);
+ return applyInternal(safeStore, safeCommand, participants);
}
@Override
- protected synchronized CommitOrReadNack apply(SafeCommandStore safeStore, SafeCommand safeCommand, StoreParticipants participants)
+ protected synchronized CommitOrReadNack applyInternal(SafeCommandStore safeStore, SafeCommand safeCommand, StoreParticipants participants)
{
Commands.ephemeralRead(safeStore, safeCommand, participants, route, txnId, partialTxn, partialDeps);
- return super.apply(safeStore, safeCommand, participants);
+ return super.applyInternal(safeStore, safeCommand, participants);
}
public final PartialDeps partialDeps()
@@ -175,8 +175,11 @@ public class ReadEphemeralTxnData extends ReadData
}
@Override
- public void timeout()
+ protected boolean cancel()
{
+ if (!super.cancel())
+ return false;
+
synchronized (this)
{
IntHashSet.IntIterator iter = waitingOn.iterator();
@@ -188,7 +191,7 @@ public class ReadEphemeralTxnData extends ReadData
}, node.agent());
}
}
- super.timeout();
+ return true;
}
@Override
diff --git a/accord-core/src/main/java/accord/messages/RouteRequest.java b/accord-core/src/main/java/accord/messages/RouteRequest.java
new file mode 100644
index 00000000..9d62de6f
--- /dev/null
+++ b/accord-core/src/main/java/accord/messages/RouteRequest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package accord.messages;
+
+import javax.annotation.Nullable;
+
+import accord.local.Node;
+import accord.local.Node.Id;
+import accord.local.PreLoadContext;
+import accord.local.SafeCommandStore;
+import accord.primitives.Route;
+import accord.primitives.TxnId;
+import accord.topology.Topologies;
+import accord.utils.MapReduceConsume;
+import accord.utils.async.Cancellable;
+
+public abstract class RouteRequest<R extends Reply> extends ParticipantsRequest<Route<?>, R> implements Request, PreLoadContext, MapReduceConsume<SafeCommandStore, R>
+{
+ public static abstract class WithUnsynced<R extends Reply> extends RouteRequest<R>
+ {
+ public final long minEpoch; // TODO (low priority, clarity): can this just always be TxnId.epoch?
+
+ public WithUnsynced(Id to, Topologies topologies, TxnId txnId, Route<?> route)
+ {
+ this(to, topologies, txnId, route, latestRelevantEpochIndex(to, topologies, route));
+ }
+
+ public WithUnsynced(Id to, Topologies topologies, Route<?> route)
+ {
+ this(to, topologies, TxnId.NONE, route, latestRelevantEpochIndex(to, topologies, route));
+ }
+
+ protected WithUnsynced(Id to, Topologies topologies, TxnId txnId, Route<?> route, int startIndex)
+ {
+ super(to, topologies, route, txnId, startIndex);
+ this.minEpoch = topologies.oldestEpoch();
+ }
+
+ protected WithUnsynced(TxnId txnId, Route<?> scope, long waitForEpoch, long minEpoch)
+ {
+ super(txnId, scope, waitForEpoch);
+ this.minEpoch = minEpoch;
+ }
+ }
+
+ public RouteRequest(Node.Id to, Topologies topologies, Route<?> route, TxnId txnId)
+ {
+ this(to, topologies, route, txnId, latestRelevantEpochIndex(to, topologies, route));
+ }
+
+ public RouteRequest(Node.Id to, Topologies topologies, Route<?> route, TxnId txnId, int startIndex)
+ {
+ this(txnId, computeScope(to, topologies, route, startIndex), computeWaitForEpoch(to, topologies, startIndex));
+ }
+
+ public RouteRequest(TxnId txnId, Route<?> scope, long waitForEpoch)
+ {
+ super(txnId, scope, waitForEpoch);
+ }
+
+ protected abstract @Nullable Cancellable submit();
+}
diff --git a/accord-core/src/main/java/accord/messages/SetShardDurable.java b/accord-core/src/main/java/accord/messages/SetShardDurable.java
index 77af8e2d..4b9296c0 100644
--- a/accord-core/src/main/java/accord/messages/SetShardDurable.java
+++ b/accord-core/src/main/java/accord/messages/SetShardDurable.java
@@ -18,14 +18,13 @@
package accord.messages;
-import accord.local.PreLoadContext;
import accord.local.SafeCommandStore;
import accord.primitives.AbstractRanges;
+import accord.primitives.Route;
import accord.primitives.Status.Durability.HasOutcome;
import accord.primitives.SyncPoint;
import accord.primitives.TxnId;
import accord.utils.Invariants;
-import accord.utils.MapReduceConsume;
import accord.utils.async.Cancellable;
import static accord.messages.MessageType.StandardMessage.SET_SHARD_DURABLE_REQ;
@@ -34,15 +33,14 @@ import static accord.primitives.Status.Durability.HasOutcome.Quorum;
import static accord.primitives.Status.Durability.HasOutcome.Universal;
import static accord.primitives.Timestamp.Flag.SHARD_BOUND;
-public class SetShardDurable extends AbstractRequest<SimpleReply>
- implements Request, PreLoadContext, MapReduceConsume<SafeCommandStore, SimpleReply>
+public class SetShardDurable extends NoWaitRequest<Route<?>, SimpleReply>
{
public final SyncPoint exclusiveSyncPoint;
public final HasOutcome durability;
public SetShardDurable(SyncPoint exclusiveSyncPoint, HasOutcome durability)
{
- super(exclusiveSyncPoint.syncId);
+ super(exclusiveSyncPoint.syncId, exclusiveSyncPoint.route);
this.exclusiveSyncPoint = exclusiveSyncPoint;
this.durability = durability;
Invariants.require(durability.compareTo(Quorum) >= 0);
@@ -61,18 +59,24 @@ public class SetShardDurable extends AbstractRequest<SimpleReply>
node.markDurable(exclusiveSyncPoint.route.toRanges(), syncIdWithFlags, durability.compareTo(Universal) >= 0 ? syncIdWithFlags : TxnId.NONE)
.invoke((success, fail) -> {
if (fail != null) node.reply(replyTo, replyContext, null, fail);
- else node.mapReduceConsumeLocal(this, exclusiveSyncPoint.route, waitForEpoch(), waitForEpoch(), this);
+ else node.commandStores().mapReduceConsume(waitForEpoch(), waitForEpoch(), this);
});
return null;
}
@Override
- public SimpleReply apply(SafeCommandStore safeStore)
+ public SimpleReply applyInternal(SafeCommandStore safeStore)
{
safeStore.commandStore().markShardDurable(safeStore, syncIdWithFlags(), ((AbstractRanges) exclusiveSyncPoint.route).toRanges(), durability);
return Ok;
}
+ @Override
+ protected SimpleReply refuseInternal(SafeCommandStore safeStore)
+ {
+ return applyInternal(safeStore);
+ }
+
@Override
public SimpleReply reduce(SimpleReply r1, SimpleReply r2)
{
diff --git a/accord-core/src/main/java/accord/messages/StableThenRead.java b/accord-core/src/main/java/accord/messages/StableThenRead.java
index f2e58075..509f4630 100644
--- a/accord-core/src/main/java/accord/messages/StableThenRead.java
+++ b/accord-core/src/main/java/accord/messages/StableThenRead.java
@@ -23,6 +23,7 @@ import javax.annotation.Nullable;
import accord.local.Commands;
import accord.local.Commands.CommitOutcome;
import accord.local.LoadKeys;
+import accord.local.LogUnavailableException;
import accord.local.Node;
import accord.local.SafeCommand;
import accord.local.SafeCommandStore;
@@ -83,7 +84,7 @@ public class StableThenRead extends ReadData
}
@Override
- public CommitOrReadNack apply(SafeCommandStore safeStore)
+ public CommitOrReadNack applyInternal(SafeCommandStore safeStore)
{
Route<?> route = this.route == null ? (Route)scope : this.route;
StoreParticipants participants = StoreParticipants.execute(safeStore, route, txnId, minEpoch(), executeAtEpoch);
@@ -98,12 +99,19 @@ public class StableThenRead extends ReadData
if (participants.executes().isEmpty())
return null;
- CommitOrReadNack reply = super.apply(safeStore, safeCommand, participants);
+ CommitOrReadNack reply = super.applyInternal(safeStore, safeCommand, participants);
if (outcome == CommitOutcome.InsufficientEpochs)
return new CommitOrReadNack(InsufficientEpochs, Commit.insufficientEpoch(safeStore, safeCommand, txnId, route));
return reply;
}
+ @Override
+ protected CommitOrReadNack refuseInternal(SafeCommandStore safeStore)
+ {
+ // cannot commit, so should not reply implying we have done so, even with an unavailable exception (as may incorrectly infer quorum stability)
+ throw new LogUnavailableException();
+ }
+
@Override
public void accept(CommitOrReadNack reply, Throwable failure)
{
diff --git a/accord-core/src/main/java/accord/primitives/LatestDeps.java b/accord-core/src/main/java/accord/primitives/LatestDeps.java
index 605674b3..1fbb4e8f 100644
--- a/accord-core/src/main/java/accord/primitives/LatestDeps.java
+++ b/accord-core/src/main/java/accord/primitives/LatestDeps.java
@@ -69,7 +69,7 @@ public class LatestDeps extends ReducingRangeMap<LatestDeps.LatestEntry>
public static void withCommitted(CoordinationAdapter<?> adapter, Node node, SequentialAsyncExecutor executor, Merge merge, FullRoute<?> route, Ballot ballot, TxnId txnId, Timestamp executeAt, Txn txn, BiConsumer<?, Throwable> failureCallback, Consumer<Deps> withDeps)
{
- if (!node.topology().hasEpoch(executeAt.epoch()))
+ if (!node.topology().hasAtLeastEpoch(executeAt.epoch()))
{
node.withEpochAtLeast(executeAt.epoch(), executor, failureCallback, () -> withCommitted(adapter, node, executor, merge, route, ballot, txnId, executeAt, txn, failureCallback, withDeps));
return;
diff --git a/accord-core/src/main/java/accord/primitives/Ranges.java b/accord-core/src/main/java/accord/primitives/Ranges.java
index 93578d58..3f611823 100644
--- a/accord-core/src/main/java/accord/primitives/Ranges.java
+++ b/accord-core/src/main/java/accord/primitives/Ranges.java
@@ -25,8 +25,10 @@ import javax.annotation.Nonnull;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.function.Function;
import java.util.function.Predicate;
import java.util.stream.Stream;
@@ -250,17 +252,17 @@ public class Ranges extends AbstractRanges implements Iterable<Range>, Seekables
return mergeTouching(this, Ranges::ofSortedAndDeoverlapped);
}
- public Map<Boolean, Ranges> partitioningBy(Predicate<? super Range> test)
+ public <T> Map<T, Ranges> partitioningBy(Function<? super Range, T> split)
{
if (isEmpty())
return Collections.emptyMap();
- List<Range> trues = new ArrayList<>();
- List<Range> falses = new ArrayList<>();
+ Map<T, List<Range>> collector = new HashMap<>();
for (Range range : this)
- (test.test(range) ? trues : falses).add(range);
- if (trues.isEmpty()) return ImmutableMap.of(Boolean.FALSE, this);
- if (falses.isEmpty()) return ImmutableMap.of(Boolean.TRUE, this);
- return ImmutableMap.of(Boolean.TRUE, Ranges.ofSortedAndDeoverlapped(trues.toArray(new Range[0])),
- Boolean.FALSE, Ranges.ofSortedAndDeoverlapped(falses.toArray(new Range[0])));
+ collector.computeIfAbsent(split.apply(range), ignore -> new ArrayList<>()).add(range);
+
+ ImmutableMap.Builder<T, Ranges> builder = ImmutableMap.builder();
+ for (Map.Entry<T, List<Range>> e : collector.entrySet())
+ builder.put(e.getKey(), Ranges.ofSortedAndDeoverlapped(e.getValue().toArray(Range[]::new)));
+ return builder.build();
}
}
diff --git a/accord-core/src/main/java/accord/utils/DeterministicSet.java b/accord-core/src/main/java/accord/utils/DeterministicSet.java
index 837d1e00..5404e755 100644
--- a/accord-core/src/main/java/accord/utils/DeterministicSet.java
+++ b/accord-core/src/main/java/accord/utils/DeterministicSet.java
@@ -210,4 +210,10 @@ public class DeterministicSet<T> extends AbstractSet<T>
cur = cur.next;
}
}
+
+ public void clear()
+ {
+ lookup.clear();
+ head.prev = head.next = head;
+ }
}
diff --git a/accord-core/src/main/java/accord/utils/IntrusivePriorityHeap.java b/accord-core/src/main/java/accord/utils/IntrusivePriorityHeap.java
index 97843ed9..16c9c75c 100644
--- a/accord-core/src/main/java/accord/utils/IntrusivePriorityHeap.java
+++ b/accord-core/src/main/java/accord/utils/IntrusivePriorityHeap.java
@@ -22,6 +22,7 @@ import java.util.Arrays;
import java.util.Comparator;
import java.util.function.BiConsumer;
import java.util.function.BiPredicate;
+import java.util.stream.Stream;
/**
* A simple array-based priority heap with intrusive elements permitting worst case logarithmic removals.
@@ -308,4 +309,9 @@ public abstract class IntrusivePriorityHeap<N extends IntrusivePriorityHeap.Node
size -= removedCount;
}
}
+
+ protected Stream<N> stream()
+ {
+ return Arrays.stream(heap, 0, size).map(n -> (N)n);
+ }
}
diff --git a/accord-core/src/main/java/accord/utils/MapReduceConsume.java b/accord-core/src/main/java/accord/utils/MapReduceConsume.java
index 4b93737b..283acf4e 100644
--- a/accord-core/src/main/java/accord/utils/MapReduceConsume.java
+++ b/accord-core/src/main/java/accord/utils/MapReduceConsume.java
@@ -19,19 +19,10 @@
package accord.utils;
import java.util.function.BiConsumer;
-import java.util.function.Consumer;
+// TODO (required): introduce e.g. MapReduceConsumeSafeStore, to encapsulate
public interface MapReduceConsume<I, O> extends MapReduce<I, O>, BiConsumer<O, Throwable>
{
@Override
void accept(O result, Throwable failure);
-
- static <I> MapReduceConsume<I, Void> forEach(Consumer<I> forEach, BiConsumer<Object, Throwable> consume)
- {
- return new MapReduceConsume<I, Void>() {
- @Override public void accept(Void result, Throwable failure) { consume.accept(result, failure); }
- @Override public Void apply(I in) { forEach.accept(in); return null; }
- @Override public Void reduce(Void o1, Void o2) {return null; }
- };
- }
}
diff --git a/accord-core/src/main/java/accord/utils/async/AsyncResults.java b/accord-core/src/main/java/accord/utils/async/AsyncResults.java
index 81b66951..76c49e4a 100644
--- a/accord-core/src/main/java/accord/utils/async/AsyncResults.java
+++ b/accord-core/src/main/java/accord/utils/async/AsyncResults.java
@@ -18,6 +18,7 @@
package accord.utils.async;
+import java.util.Arrays;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
@@ -379,7 +380,6 @@ public class AsyncResults
* An AsyncResult that also implements Runnable
* @param <V>
*/
- @VisibleForImplementation
public static class RunnableResult<V> extends AbstractResult<V> implements Runnable
{
protected final Callable<V> callable;
@@ -413,14 +413,17 @@ public class AsyncResults
return new RunnableResult<>(callable);
}
- public static RunnableResult<Void> runnableResult(Runnable runnable)
+ public static <V> AsyncResult<List<V>> allOf(List<? extends AsyncResult<? extends V>> results)
{
- return new RunnableResult<>(() -> {
- runnable.run();
- return null;
- });
- }
+ return new AsyncCombiner.ResultCombiner<V, List<V>>(results) {
+ @Override
+ List<V> process(V[] inputs)
+ {
+ return Arrays.asList(inputs);
+ }
+ }.beginAsResult();
+ }
public static <V> AsyncResult<V> reduce(List<? extends AsyncResult<? extends V>> results, Reduce<V, V> reducer)
{
diff --git a/accord-core/src/test/java/accord/burn/BurnTestBase.java b/accord-core/src/test/java/accord/burn/BurnTestBase.java
index a88c2b52..812acf68 100644
--- a/accord-core/src/test/java/accord/burn/BurnTestBase.java
+++ b/accord-core/src/test/java/accord/burn/BurnTestBase.java
@@ -48,7 +48,6 @@ import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.BiConsumer;
import java.util.function.BiFunction;
import java.util.function.Consumer;
import java.util.function.Function;
@@ -64,6 +63,7 @@ import org.slf4j.LoggerFactory;
import accord.api.Agent;
import accord.api.Journal;
import accord.api.Key;
+import accord.api.OwnershipEventListener;
import accord.api.ProtocolModifiers.Toggles;
import accord.api.ProtocolModifiers.Toggles.FastExec;
import accord.api.Scheduler;
@@ -96,7 +96,6 @@ import accord.messages.Reply;
import accord.primitives.Keys;
import accord.primitives.Range;
import accord.primitives.Ranges;
-import accord.primitives.Timestamp;
import accord.primitives.Txn;
import accord.primitives.Txn.Kind;
import accord.primitives.TxnId;
@@ -104,6 +103,7 @@ import accord.primitives.TxnId.FastPath;
import accord.primitives.TxnId.MediumPath;
import accord.topology.Shard;
import accord.topology.Topology;
+import accord.topology.TopologyRandomizer;
import accord.utils.DefaultRandom;
import accord.utils.Gen;
import accord.utils.Gens;
@@ -454,10 +454,8 @@ public class BurnTestBase
.asLongSupplier(forked);
};
Supplier<TimeService> timeServiceSupplier = () -> TimeService.ofNonMonotonic(nowSupplier.get(), MILLISECONDS);
- TriFunction<BiConsumer<Timestamp, Ranges>, Scheduler, NodeSink.TimeoutSupplier, Agent> agentSupplier = (onStale, scheduler, timeoutSupplier) -> new ListAgent(scheduler, random.fork(), 1000L, failures::add, retryBootstrap, onStale, coordinationDelays, progressDelays, timeoutDelays, pendingQueue::nowInMillis, timeServiceSupplier.get(), timeoutSupplier);
- SimulatedDelayedExecutorService globalExecutor = new SimulatedDelayedExecutorService(queue, new ListAgent(null, random.fork(), 1000L, failures::add, retryBootstrap, (i1, i2) -> {
- throw new IllegalAccessError("Global executor should enver get a stale event");
- }, coordinationDelays, progressDelays, timeoutDelays, queue::nowInMillis, timeServiceSupplier.get(), null), null);
+ TriFunction<OwnershipEventListener, Scheduler, NodeSink.TimeoutSupplier, Agent> agentSupplier = (ownershipEventListener, scheduler, timeoutSupplier) -> new ListAgent(scheduler, random.fork(), 1000L, failures::add, retryBootstrap, ownershipEventListener, coordinationDelays, progressDelays, timeoutDelays, pendingQueue::nowInMillis, timeServiceSupplier.get(), timeoutSupplier);
+ SimulatedDelayedExecutorService globalExecutor = new SimulatedDelayedExecutorService(queue, new ListAgent(null, random.fork(), 1000L, failures::add, retryBootstrap, OwnershipEventListener.FAIL, coordinationDelays, progressDelays, timeoutDelays, queue::nowInMillis, timeServiceSupplier.get(), null), null);
Verifier verifier = createVerifier(keyCount * prefixCount);
MessageListener listener = MessageListener.get();
@@ -704,11 +702,11 @@ public class BurnTestBase
List<Id> clients = generateIds(true, 1 + random.nextInt(4));
int rf;
float chance = random.nextFloat();
- if (chance < 0.2f) { rf = random.nextInt(2, 9); }
- else if (chance < 0.4f) { rf = 3; }
- else if (chance < 0.7f) { rf = 5; }
- else if (chance < 0.8f) { rf = 7; }
- else { rf = 9; }
+ if (chance < 0.2f) { rf = random.nextInt(TopologyRandomizer.MIN_RF, random.nextInt(5, 9)); }
+ else if (chance < 0.4f) { rf = 3; }
+ else if (chance < 0.7f) { rf = 5; }
+ else if (chance < 0.95f) { rf = 7; }
+ else { rf = 9; }
List<Id> nodes = generateIds(false, random.nextInt(rf, rf * 3));
diff --git a/accord-core/src/test/java/accord/burn/BurnTestConfigurationService.java b/accord-core/src/test/java/accord/burn/BurnTestConfigurationService.java
index 8c997b52..bdf3d8c6 100644
--- a/accord-core/src/test/java/accord/burn/BurnTestConfigurationService.java
+++ b/accord-core/src/test/java/accord/burn/BurnTestConfigurationService.java
@@ -29,8 +29,7 @@ import javax.annotation.concurrent.GuardedBy;
import accord.api.Agent;
import accord.api.AsyncExecutor;
-import accord.api.TestableConfigurationService;
-import accord.impl.AbstractConfigurationService;
+import accord.impl.AbstractTestConfigurationService;
import accord.local.Node;
import accord.messages.Callback;
import accord.messages.MessageType;
@@ -42,7 +41,7 @@ import accord.topology.Topology;
import accord.utils.RandomSource;
import accord.utils.async.AsyncResults;
-public class BurnTestConfigurationService extends AbstractConfigurationService.Minimal implements TestableConfigurationService
+public class BurnTestConfigurationService extends AbstractTestConfigurationService
{
private final AsyncExecutor executor;
private final Function<Node.Id, Node> lookup;
@@ -183,17 +182,11 @@ public class BurnTestConfigurationService extends AbstractConfigurationService.M
}
@Override
- protected void localSyncComplete(Topology topology, boolean startSync)
+ protected void onReadyToCoordinate(Topology topology, boolean startSync)
{
topologyUpdates.syncComplete(lookup.apply(localId), topology.nodes(), topology.epoch());
}
- @Override
- protected void localBootstrapsComplete(Topology topology)
- {
- topologyUpdates.bootstrapComplete(lookup.apply(localId), topology.epoch());
- }
-
@Override
protected void topologyUpdatePostListenerNotify(Topology topology)
{
diff --git a/accord-core/src/test/java/accord/burn/TopologyUpdates.java b/accord-core/src/test/java/accord/burn/TopologyUpdates.java
index 4a60aae8..4a73b205 100644
--- a/accord-core/src/test/java/accord/burn/TopologyUpdates.java
+++ b/accord-core/src/test/java/accord/burn/TopologyUpdates.java
@@ -21,7 +21,6 @@ package accord.burn;
import accord.api.AsyncExecutor;
import accord.api.TestableConfigurationService;
import accord.local.Node;
-import accord.primitives.Range;
import accord.primitives.Ranges;
import accord.topology.Topology;
import accord.utils.Invariants;
@@ -30,12 +29,10 @@ import org.agrona.collections.Long2ObjectHashMap;
import java.util.*;
import java.util.function.Function;
-import java.util.function.Predicate;
public class TopologyUpdates
{
private final Long2ObjectHashMap<Map<Node.Id, Ranges>> pendingSyncTopologies = new Long2ObjectHashMap<>();
- private final Long2ObjectHashMap<Map<Node.Id, Ranges>> pendingBootstrap = new Long2ObjectHashMap<>();
Function<Node.Id, AsyncExecutor> executors;
public TopologyUpdates(Function<Node.Id, AsyncExecutor> executors)
@@ -54,7 +51,6 @@ public class TopologyUpdates
nodeToNewRanges.put(node, newRanges);
}
pendingSyncTopologies.put(update.epoch(), nodeToNewRanges);
- pendingBootstrap.put(update.epoch(), new HashMap<>(nodeToNewRanges));
return MessageTask.begin(originator, nodes, executors.apply(originator.id()), "TopologyNotify:" + update.epoch(), (node, from, onDone) -> {
long nodeEpoch = node.epoch();
if (nodeEpoch + 1 < update.epoch())
@@ -81,18 +77,6 @@ public class TopologyUpdates
});
}
- public synchronized void bootstrapComplete(Node originator, long epoch)
- {
- if (pendingBootstrap.isEmpty())
- return;
-
- Map<Node.Id, Ranges> pending = pendingBootstrap.get(epoch);
- Invariants.require(pending != null && pending.remove(originator.id()) != null);
-
- if (pending.isEmpty())
- pendingBootstrap.remove(epoch);
- }
-
public synchronized void epochClosed(Node originator, Collection<Node.Id> cluster, Ranges ranges, long epoch)
{
executors.apply(originator.id()).execute(() -> {
@@ -113,16 +97,6 @@ public class TopologyUpdates
});
}
- public boolean isPending(Range range, Node.Id id)
- {
- Predicate<Map<Node.Id, Ranges>> test = map -> {
- Ranges rs = map.get(id);
- return rs != null && rs.intersects(range);
- };
- return pendingSyncTopologies.values().stream().anyMatch(test)
- || pendingBootstrap.values().stream().anyMatch(test);
- }
-
public int pendingTopologies()
{
return pendingSyncTopologies.size();
diff --git a/accord-core/src/test/java/accord/coordinate/TopologyChangeTest.java b/accord-core/src/test/java/accord/coordinate/TopologyChangeTest.java
index 569ea196..26c5fbb1 100644
--- a/accord-core/src/test/java/accord/coordinate/TopologyChangeTest.java
+++ b/accord-core/src/test/java/accord/coordinate/TopologyChangeTest.java
@@ -22,7 +22,6 @@ import accord.impl.mock.MockCluster;
import accord.impl.mock.MockConfigurationService;
import accord.local.Command;
import accord.local.Node;
-import accord.local.PreLoadContext;
import accord.local.StoreParticipants;
import accord.messages.Message;
import accord.messages.PreAccept;
@@ -48,7 +47,6 @@ import static accord.primitives.Routable.Domain.Key;
import static accord.primitives.Txn.Kind.ExclusiveSyncPoint;
import static accord.primitives.Txn.Kind.Write;
-import static accord.local.PreLoadContext.contextFor;
import static accord.utils.async.AsyncChainUtils.getUninterruptibly;
public class TopologyChangeTest
@@ -74,7 +72,8 @@ public class TopologyChangeTest
TxnId txnId1 = node1.nextTxnIdWithDefaultFlags(Write, Key);
Txn txn1 = writeTxn(keys);
getUninterruptibly(node1.coordinate(txnId1, txn1));
- getUninterruptibly(node1.commandStores().forEach(PreLoadContext.contextFor(txnId1, "Test"), keys.toParticipants(), 1, 1, safeStore -> {
+
+ getUninterruptibly(node1.commandStores().forEach("Test", txnId1, keys.toParticipants(), 1, 1, safeStore -> {
StoreParticipants participants = StoreParticipants.read(safeStore, keys.toParticipants(), txnId1);
Command command = safeStore.get(txnId1, participants).current();
Assertions.assertTrue(command.partialDeps().isEmpty());
@@ -101,7 +100,7 @@ public class TopologyChangeTest
cluster.nodes(4, 5).forEach(node -> {
try
{
- getUninterruptibly(node.commandStores().forEach(contextFor(txnId1, txnId2, "Test"), keys.toParticipants(), 2, 2, safeStore -> {
+ getUninterruptibly(node.commandStores().forEach("Test", txnId2, keys.toParticipants(), 2, 2, safeStore -> {
StoreParticipants participants = StoreParticipants.update(safeStore, keys.toParticipants(), txnId2.epoch(), txnId2, txnId2.epoch());
Command command = safeStore.get(txnId2, participants).current();
Assertions.assertTrue(command.partialDeps().contains(txnId1));
@@ -132,7 +131,7 @@ public class TopologyChangeTest
if (node.epoch() < epoch)
throw new AssertionError(String.format("node[%s] epoch %s is less than check epoch %s", node.id(), node.epoch(), epoch));
- node.forEachLocal((PreLoadContext.Empty) () -> "Test", participants, 1, node.epoch(), safeStore -> {
+ node.commandStores().forEach("Test", participants, 1, node.epoch(), safeStore -> {
boolean rejected = safeStore.commandStore().isRejectedIfNotPreAccepted(TxnId.minForEpoch(epoch), participants);
if (rejected != rejectionExpected)
{
diff --git a/accord-core/src/test/java/accord/impl/AbstractConfigurationServiceTest.java b/accord-core/src/test/java/accord/impl/AbstractConfigurationServiceTest.java
index 37fdc0cf..9bfbf7ef 100644
--- a/accord-core/src/test/java/accord/impl/AbstractConfigurationServiceTest.java
+++ b/accord-core/src/test/java/accord/impl/AbstractConfigurationServiceTest.java
@@ -36,7 +36,7 @@ import accord.utils.async.AsyncResults;
import com.google.common.collect.ImmutableSet;
import accord.api.ConfigurationService;
-import accord.impl.AbstractConfigurationService.Minimal.EpochHistory;
+import accord.impl.AbstractTestConfigurationService.EpochHistory;
import accord.local.Node.Id;
import accord.primitives.Range;
import accord.topology.Shard;
@@ -126,7 +126,7 @@ public class AbstractConfigurationServiceTest
}
}
- private static class TestableConfigurationService extends AbstractConfigurationService.Minimal
+ private static class TestableConfigurationService extends AbstractTestConfigurationService
{
final Set<Long> syncStarted = new HashSet<>();
final Set<Long> epochsFetched = new HashSet<>();
@@ -143,7 +143,7 @@ public class AbstractConfigurationServiceTest
}
@Override
- protected void localSyncComplete(Topology topology, boolean startSync)
+ protected void onReadyToCoordinate(Topology topology, boolean startSync)
{
if (!syncStarted.add(topology.epoch()))
Assertions.fail("Sync started multiple times for " + topology.epoch());
diff --git a/accord-core/src/test/java/accord/impl/AbstractTestConfigurationService.java b/accord-core/src/test/java/accord/impl/AbstractTestConfigurationService.java
new file mode 100644
index 00000000..5fb98f56
--- /dev/null
+++ b/accord-core/src/test/java/accord/impl/AbstractTestConfigurationService.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package accord.impl;
+
+import accord.api.Agent;
+import accord.api.TestableConfigurationService;
+import accord.local.Node;
+import accord.topology.Topology;
+
+public abstract class AbstractTestConfigurationService
+extends AbstractConfigurationService<AbstractTestConfigurationService.EpochState, AbstractTestConfigurationService.EpochHistory>
+implements TestableConfigurationService
+{
+ public static class EpochState extends AbstractEpochState
+ {
+ public EpochState(long epoch)
+ {
+ super(epoch);
+ }
+
+ public Topology topology()
+ {
+ return topology;
+ }
+ }
+
+ public static class EpochHistory extends AbstractEpochHistory<EpochState>
+ {
+ @Override
+ protected EpochState createEpochState(long epoch)
+ {
+ return new EpochState(epoch);
+ }
+ }
+
+ public AbstractTestConfigurationService(Node.Id node, Agent agent)
+ {
+ super(node, agent);
+ }
+
+ @Override
+ protected EpochHistory createEpochHistory()
+ {
+ return new EpochHistory();
+ }
+}
diff --git a/accord-core/src/test/java/accord/impl/MessageListener.java b/accord-core/src/test/java/accord/impl/MessageListener.java
index 6be9f8de..f20c4e23 100644
--- a/accord-core/src/test/java/accord/impl/MessageListener.java
+++ b/accord-core/src/test/java/accord/impl/MessageListener.java
@@ -25,7 +25,7 @@ import accord.messages.Message;
import accord.messages.ReadData.ReadOk;
import accord.messages.Request;
import accord.messages.SimpleReply;
-import accord.messages.TxnRequest;
+import accord.messages.RouteRequest;
import accord.primitives.Timestamp;
import accord.primitives.TxnId;
import accord.topology.Topology;
@@ -199,8 +199,8 @@ public interface MessageListener
private static boolean containsAny(Request message)
{
- if (message instanceof TxnRequest<?>)
- return txnIdFilter.contains(((TxnRequest<?>) message).txnId);
+ if (message instanceof RouteRequest<?>)
+ return txnIdFilter.contains(((RouteRequest<?>) message).txnId);
// this includes txn that depend on the txn, should this limit for the first txnId?
if (message instanceof PreLoadContext)
return ((PreLoadContext) message).txnIds().stream().anyMatch(txnIdFilter::contains);
diff --git a/accord-core/src/test/java/accord/impl/TestAgent.java b/accord-core/src/test/java/accord/impl/TestAgent.java
index d4db2701..b3348c61 100644
--- a/accord-core/src/test/java/accord/impl/TestAgent.java
+++ b/accord-core/src/test/java/accord/impl/TestAgent.java
@@ -25,6 +25,7 @@ import org.slf4j.LoggerFactory;
import accord.api.Agent;
import accord.api.CoordinatorEventListener;
+import accord.api.OwnershipEventListener;
import accord.api.ProgressLog;
import accord.api.Result;
import accord.impl.mock.MockStore;
@@ -47,7 +48,7 @@ import static java.util.concurrent.TimeUnit.MICROSECONDS;
import static java.util.concurrent.TimeUnit.MINUTES;
import static java.util.concurrent.TimeUnit.SECONDS;
-public class TestAgent implements Agent
+public class TestAgent implements Agent, OwnershipEventListener
{
private static final Logger logger = LoggerFactory.getLogger(TestAgent.class);
@@ -119,6 +120,12 @@ public class TestAgent implements Agent
}
+ @Override
+ public OwnershipEventListener ownershipEvents()
+ {
+ return this;
+ }
+
@Override
public void onUncaughtException(Throwable t)
{
diff --git a/accord-core/src/test/java/accord/impl/basic/Cluster.java b/accord-core/src/test/java/accord/impl/basic/Cluster.java
index a83abc1b..f4d0c2e3 100644
--- a/accord-core/src/test/java/accord/impl/basic/Cluster.java
+++ b/accord-core/src/test/java/accord/impl/basic/Cluster.java
@@ -36,7 +36,6 @@ import java.util.TreeMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.BiConsumer;
import java.util.function.BiFunction;
import java.util.function.BooleanSupplier;
import java.util.function.Consumer;
@@ -56,6 +55,7 @@ import accord.api.Agent;
import accord.api.AsyncExecutor;
import accord.api.Journal;
import accord.api.MessageSink;
+import accord.api.OwnershipEventListener;
import accord.api.RoutingKey;
import accord.api.Scheduler;
import accord.api.Scheduler.Scheduled;
@@ -80,6 +80,7 @@ import accord.local.Cleanup;
import accord.local.Command;
import accord.local.CommandStore;
import accord.local.CommandStores;
+import accord.local.LogUnavailableException;
import accord.local.Node;
import accord.local.Node.Id;
import accord.local.RedundantBefore;
@@ -100,7 +101,6 @@ import accord.primitives.Ranges;
import accord.primitives.RoutableKey;
import accord.primitives.SaveStatus;
import accord.primitives.Status;
-import accord.primitives.Timestamp;
import accord.primitives.Txn;
import accord.primitives.TxnId;
import accord.topology.Topology;
@@ -126,7 +126,6 @@ import static accord.local.Cleanup.INVALIDATE;
import static accord.local.Cleanup.Input.FULL;
import static accord.local.Command.NotDefined.uninitialised;
import static accord.local.StoreParticipants.Filter.LOAD;
-import static accord.primitives.Status.Durability.HasOutcome.Quorum;
import static accord.utils.Invariants.Paranoia.LINEAR;
import static accord.utils.Invariants.ParanoiaCostFactor.HIGH;
import static java.util.Collections.emptyMap;
@@ -234,7 +233,6 @@ public class Cluster
final RandomSource random;
final LinkConfig linkConfig;
final Function<Id, Node> lookup;
- final Function<Id, Journal> journalLookup;
final PendingQueue pending;
final Runnable checkFailures;
final List<Runnable> onDone = new ArrayList<>();
@@ -245,14 +243,13 @@ public class Cluster
BiFunction<Id, Id, Link> links;
long droppedAt;
- public Cluster(RandomSource random, MessageListener messageListener, Supplier<PendingQueue> queueSupplier, Runnable checkFailures, Function<Id, Node> lookup, Function<Id, Journal> journalLookup, IntSupplier rf, Consumer<Packet> responseSink)
+ public Cluster(RandomSource random, MessageListener messageListener, Supplier<PendingQueue> queueSupplier, Runnable checkFailures, Function<Id, Node> lookup, IntSupplier rf, Consumer<Packet> responseSink)
{
this.random = random;
this.messageListener = messageListener;
this.pending = queueSupplier.get();
this.checkFailures = checkFailures;
this.lookup = lookup;
- this.journalLookup = journalLookup;
this.responseSink = responseSink;
this.linkConfig = defaultLinkConfig(random, rf);
this.links = linkConfig.defaultLinks;
@@ -427,10 +424,8 @@ public class Cluster
.asLongSupplier(forked);
};
Supplier<TimeService> timeServiceSupplier = () -> TimeService.ofNonMonotonic(nowSupplier.get(), MILLISECONDS);
- TriFunction<BiConsumer<Timestamp, Ranges>, Scheduler, NodeSink.TimeoutSupplier, Agent> agentSupplier = (onStale, scheduler, timeoutSupplier) -> new ListAgent(scheduler, randomSupplier.get(), 1000L, failures::add, retryBootstrap, onStale, coordinationDelays, progressDelays, timeoutDelays, queue::nowInMillis, timeServiceSupplier.get(), timeoutSupplier);
- SimulatedDelayedExecutorService globalExecutor = new SimulatedDelayedExecutorService(queue, new ListAgent(null, randomSupplier.get(), 1000L, failures::add, retryBootstrap, (i1, i2) -> {
- throw new IllegalAccessError("Global executor should never get a stale event");
- }, () -> { throw new UnsupportedOperationException(); }, () -> { throw new UnsupportedOperationException(); }, timeoutDelays, queue::nowInMillis, timeServiceSupplier.get(), null), null);
+ TriFunction<OwnershipEventListener, Scheduler, NodeSink.TimeoutSupplier, Agent> agentSupplier = (ownershipEvents, scheduler, timeoutSupplier) -> new ListAgent(scheduler, randomSupplier.get(), 1000L, failures::add, retryBootstrap, ownershipEvents, coordinationDelays, progressDelays, timeoutDelays, queue::nowInMillis, timeServiceSupplier.get(), timeoutSupplier);
+ SimulatedDelayedExecutorService globalExecutor = new SimulatedDelayedExecutorService(queue, new ListAgent(null, randomSupplier.get(), 1000L, failures::add, retryBootstrap, OwnershipEventListener.FAIL, timeoutDelays, timeoutDelays, timeoutDelays, queue::nowInMillis, timeServiceSupplier.get(), null), null);
TopologyFactory topologyFactory = new TopologyFactory(initialTopology.maxRf(), initialTopology.ranges().stream().toArray(Range[]::new))
{
@Override
@@ -590,7 +585,9 @@ public class Cluster
++cmdCounter;
command = command.updateParticipants(command.participants().filter(LOAD, commandStore.unsafeGetRedundantBefore(), command.txnId(), command.executeAtIfKnown()));
// Journal will not have result persisted. This part is here for test purposes and ensuring that we have strict object equality.
- Command reconstructed = journal.loadCommand(commandStore.id(), command.txnId(), commandStore.unsafeGetRedundantBefore(), commandStore.durableBefore());
+ Command reconstructed = null;
+ try { reconstructed = journal.loadCommand(commandStore.id(), command.txnId(), commandStore.unsafeGetRedundantBefore(), commandStore.durableBefore()); }
+ catch (LogUnavailableException ignore) {}
if (reconstructed == null || reconstructed.saveStatus().hasBeen(Status.Truncated))
return;
@@ -622,7 +619,7 @@ public class Cluster
public static Map<MessageType, Stats> run(Id[] nodes, int[] prefixes, MessageListener messageListener, Supplier<PendingQueue> queueSupplier,
Function<Id, AsyncExecutor> nodeExecutorSupplier,
- TriFunction<BiConsumer<Timestamp, Ranges>, Scheduler, NodeSink.TimeoutSupplier, Agent> agentSupplier,
+ TriFunction<OwnershipEventListener, Scheduler, NodeSink.TimeoutSupplier, Agent> agentSupplier,
Runnable checkFailures, Consumer<Packet> responseSink,
Supplier<RandomSource> randomSupplier,
Supplier<TimeService> timeServiceSupplier,
@@ -636,7 +633,7 @@ public class Cluster
try
{
RandomSource random = randomSupplier.get();
- Cluster sinks = new Cluster(randomSupplier.get(), messageListener, queueSupplier, checkFailures, nodeMap::get, journalMap::get, () -> topologyFactory.rf, responseSink);
+ Cluster sinks = new Cluster(randomSupplier.get(), messageListener, queueSupplier, checkFailures, nodeMap::get, () -> topologyFactory.rf, responseSink);
TopologyUpdates topologyUpdates = new TopologyUpdates(executorMap::get);
TopologyRandomizer.Listener schemaApply = t -> {
for (Node node : nodeMap.values())
@@ -676,7 +673,7 @@ public class Cluster
@Override public long now() { return sinks.pending.nowInMillis(); }
@Override public TimeUnit units() { return MILLISECONDS; }
};
- TopologyRandomizer configRandomizer = new TopologyRandomizer(randomSupplier, prefixes, topology, topologyUpdates, nodeMap::get, schemaApply);
+ TopologyRandomizer topologyRandomizer = new TopologyRandomizer(randomSupplier, prefixes, topology, topologyUpdates, nodeMap::get, schemaApply);
List<DurabilityService> durabilityServices = new ArrayList<>();
List<Service> services = new ArrayList<>();
for (Id id : nodes)
@@ -684,9 +681,9 @@ public class Cluster
ClusterScheduler scheduler = sinks.new ClusterScheduler(id.id);
MessageSink messageSink = sinks.create(id, timeouts);
TimeService timeService = timeServiceSupplier.get();
- BiConsumer<Timestamp, Ranges> onStale = (sinceAtLeast, ranges) -> configRandomizer.onStale(id, sinceAtLeast, ranges);
+ OwnershipEventListener ownershipEventListener = topologyRandomizer.listener(id);
AsyncExecutor nodeExecutor = nodeExecutorSupplier.apply(id);
- Agent agent = agentSupplier.apply(onStale, scheduler, timeouts);
+ Agent agent = agentSupplier.apply(ownershipEventListener, scheduler, timeouts);
executorMap.put(id, nodeExecutor);
Journal journal = journalFactory.apply(id, random);
journalMap.put(id, journal);
@@ -731,7 +728,7 @@ public class Cluster
Runnable updateProgressLogConcurrency;
{
updateProgressLogConcurrency = () -> {
- nodeMap.values().forEach(node -> node.commandStores().forEachCommandStore(cs -> cs.execute(() -> {
+ nodeMap.values().forEach(node -> node.commandStores().forAllUnsafe(cs -> cs.execute(() -> {
((TestProgressLog)cs.unsafeProgressLog()).config().concurrency = random.nextInt(8, 32);
})));
};
@@ -757,59 +754,98 @@ public class Cluster
updateProgressLogConcurrency.run();
}, 5L, SECONDS);
- Scheduled reconfigure = clusterScheduler.recurring(configRandomizer::maybeUpdateTopology, 1, SECONDS);
+ Scheduled reconfigure = clusterScheduler.recurring(topologyRandomizer::maybeUpdateTopology, 1, SECONDS);
Purge purge = new Purge(clusterScheduler, random, nodesList, nodeMap, journalMap);
Scheduled restart = clusterScheduler.recurring(() -> {
- Id id = pickNodeNotBootstrapping(random, nodesList, nodeMap);
+ Id id = pickNodeNotBootstrapping(random, nodesList, topologyRandomizer);
if (id == null)
return;
- CommandStores stores = nodeMap.get(id).commandStores();
+ Node node = nodeMap.get(id);
+ CommandStores stores = node.commandStores();
while (sinks.drain(getPendingPredicate(id, stores.all()))) ;
- trace.debug("Triggering store cleanup and journal replay for node " + id);
+ // TODO (expected): we include too many ranges here, including any that are retired
+ Ranges ranges = Ranges.of(Stream.of(node.commandStores().all()).flatMap(cs -> cs.unsafeGetRangesForEpoch().all().stream()).toArray(Range[]::new));
+ boolean rebootstrap = topologyFactory.rf > 2 && random.nextBoolean() && !topologyRandomizer.unsafeToRebootstrap(ranges);
+ trace.debug(String.format("Triggering %s for node %s",
+ rebootstrap ? "rebootstrap" : "bounce and journal replay",
+ id));
+
CommandsForKey.disableLinearizabilityViolationsReporting();
+ if (rebootstrap)
+ topologyRandomizer.markRebootstrapping(node);
// Clean data and restore from snapshot
- ListStore listStore = (ListStore) nodeMap.get(id).commandStores().dataStore();
+ ListStore listStore = (ListStore) node.commandStores().dataStore();
NavigableMap<RoutableKey, Timestamped<int[]>> prevData = listStore.copyOfCurrentData();
- Int2ObjectHashMap<NavigableMap<TxnId, Command>> beforeStores = copyCommands(stores.all());
-
listStore.clear();
+
// We are simulating node restart, so its remote listeners will also be gone
- ((DefaultRemoteListeners) nodeMap.get(id).remoteListeners()).clear();
- for (CommandStore store : stores.all())
- {
- InMemoryCommandStore commandStore = (InMemoryCommandStore) store;
- commandStore.clear();
- }
+ ((DefaultRemoteListeners) node.remoteListeners()).clear();
+ Int2ObjectHashMap<NavigableMap<TxnId, Command>> beforeStores = copyCommands(stores.all());
- // Replay journal
Journal journal = journalMap.get(id);
- List<? extends Journal.TopologyUpdate> list = journal.replayTopologies();
+
Journal.TopologyUpdate lastUpdate = null;
- for (Journal.TopologyUpdate update : list)
{
- Invariants.require(lastUpdate == null || update.global.epoch() > lastUpdate.global.epoch());
- lastUpdate = update;
+ Iterator<? extends Journal.TopologyUpdate> iter = journal.replayTopologies().iterator();
+ while (iter.hasNext())
+ {
+ Journal.TopologyUpdate update = iter.next();
+ Invariants.require(lastUpdate == null || update.global.epoch() > lastUpdate.global.epoch());
+ lastUpdate = update;
+ }
+
+ // Reset and restore command store states
+ for (CommandStore store : stores.all())
+ {
+ DelayedCommandStore store1 = ((DelayedCommandStore) store);
+ CommandStores.RangesForEpoch beforeRestore = store1.unsafeGetRangesForEpoch();
+ store1.unsafeClearForTesting();
+ if (lastUpdate != null)
+ store1.unsafeSetRangesForEpoch(lastUpdate.commandStores.get(store.id()));
+ CommandStores.RangesForEpoch afterRestore = store1.unsafeGetRangesForEpoch();
+ if (!beforeRestore.equals(afterRestore))
+ Invariants.require(beforeRestore.equals(afterRestore));
+ }
+
+ if (lastUpdate != null)
+ node.commandStores().resetTopology(lastUpdate);
}
- if (lastUpdate != null)
- ((DelayedCommandStores) nodeMap.get(id).commandStores()).validateShardStateForTesting(lastUpdate);
+ if (rebootstrap)
+ {
+ node.durability().stop();
+
+ // make sure to flush anything in flight before truncating journal
+ ((InMemoryJournal)journal).dropAll();
+ stores.rebootstrap(node).invoke(node.agent());
- listStore.restore();
- for (CommandStore store : stores.all())
- ((ListAgent) store.agent()).restore((InMemoryCommandStore) store);
- journal.replay(stores);
+ while (sinks.drain(getPendingPredicate(id, stores.all())));
- // Re-enable safety checks
- while (sinks.drain(getPendingPredicate(id, stores.all()))) ;
- CommandsForKey.enableLinearizabilityViolationsReporting();
- verifyConsistentRestore(beforeStores, stores.all());
- // we can get ahead of prior state by executing further if we skip some earlier phase's dependencies
- listStore.checkAtLeast(stores, prevData);
+ CommandsForKey.enableLinearizabilityViolationsReporting();
+ node.durability().start();
+ }
+ else
+ {
+ if (lastUpdate != null)
+ ((DelayedCommandStores) node.commandStores()).validateShardStateForTesting(lastUpdate);
+
+ listStore.restore();
+ for (CommandStore store : stores.all())
+ ((ListAgent) store.agent()).restore((InMemoryCommandStore) store);
+ journal.replay(stores);
+
+ // Re-enable safety checks
+ while (sinks.drain(getPendingPredicate(id, stores.all()))) ;
+ CommandsForKey.enableLinearizabilityViolationsReporting();
+ verifyConsistentRestore(beforeStores, stores.all());
+ // we can get ahead of prior state by executing further if we skip some earlier phase's dependencies
+ listStore.checkAtLeast(stores, prevData);
+ }
trace.debug("Done with replay.");
}, () -> random.nextInt(10, 30), SECONDS);
@@ -915,15 +951,14 @@ public class Cluster
return result;
}
- private static Id pickNodeNotBootstrapping(RandomSource random, List<Id> ids, Map<Id, Node> nodeMap)
+ private static Id pickNodeNotBootstrapping(RandomSource random, List<Id> ids, TopologyRandomizer topologyRandomizer)
{
List<Id> remaining = new ArrayList<>(ids);
while (!remaining.isEmpty())
{
int i = random.nextInt(remaining.size());
Id id = remaining.get(i);
- CommandStore[] stores = nodeMap.get(id).commandStores().all();
- if (!Stream.of(stores).anyMatch(CommandStore::isBootstrapping))
+ if (!topologyRandomizer.isBootstrapping(id))
return id;
remaining.set(i, remaining.get(remaining.size() - 1));
@@ -989,7 +1024,7 @@ public class Cluster
if (store.unsafeGetRedundantBefore().min(beforeCommand.participants().owns(), RedundantBefore.Bounds::shardAndLocallyRedundantBefore).compareTo(txnId) > 0)
continue;
- if (beforeCommand.participants().owns().isEmpty() && store.durableBefore().min(txnId).compareTo(Quorum) >= 0)
+ if (beforeCommand.participants().owns().isEmpty() && store.durableBefore().min(txnId).compareTo(Status.Durability.HasOutcome.Quorum) >= 0)
continue;
if (!beforeCommand.saveStatus().hasBeen(Status.PreCommitted) && store.unsafeGetRedundantBefore().min(beforeCommand.participants().owns(), RedundantBefore.Bounds::locallyRedundantBefore).compareTo(txnId) > 0)
diff --git a/accord-core/src/test/java/accord/impl/basic/DelayedCommandStores.java b/accord-core/src/test/java/accord/impl/basic/DelayedCommandStores.java
index 08b56118..64f4f141 100644
--- a/accord-core/src/test/java/accord/impl/basic/DelayedCommandStores.java
+++ b/accord-core/src/test/java/accord/impl/basic/DelayedCommandStores.java
@@ -48,7 +48,6 @@ import accord.impl.basic.TaskExecutorService.Task;
import accord.local.Command;
import accord.local.CommandStore;
import accord.local.CommandStores;
-import accord.local.Node;
import accord.local.NodeCommandStoreService;
import accord.local.PreLoadContext;
import accord.local.RedundantBefore;
@@ -84,17 +83,6 @@ public class DelayedCommandStores extends InMemoryCommandStores.SingleThread
new DelayedCommandStores(time, agent, store, random, shardDistributor, progressLogFactory, listenersFactory, new SimulatedDelayedExecutorService(pending, agent, time.id()), isLoadedCheck, journal);
}
- @Override
- protected boolean shouldBootstrap(Node node, Topology previous, Topology updated, Range range)
- {
- if (!super.shouldBootstrap(node, previous, updated, range))
- return false;
- if (!(range.start() instanceof PrefixedIntHashKey)) return true;
- int prefix = ((PrefixedIntHashKey) range.start()).prefix;
- // we see new prefix when a new prefix is added, so avoid bootstrap in these cases
- return contains(previous, prefix);
- }
-
public void validateShardStateForTesting(Journal.TopologyUpdate lastUpdate)
{
ShardHolder[] shards = new ShardHolder[lastUpdate.commandStores.size()];
diff --git a/accord-core/src/test/java/accord/impl/basic/InMemoryJournal.java b/accord-core/src/test/java/accord/impl/basic/InMemoryJournal.java
index 40dbc5d4..c291156f 100644
--- a/accord-core/src/test/java/accord/impl/basic/InMemoryJournal.java
+++ b/accord-core/src/test/java/accord/impl/basic/InMemoryJournal.java
@@ -45,6 +45,7 @@ import accord.local.Command;
import accord.local.CommandStore;
import accord.local.CommandStores;
import accord.local.DurableBefore;
+import accord.local.LogUnavailableException;
import accord.local.Node;
import accord.local.RedundantBefore;
import accord.local.StoreParticipants;
@@ -131,6 +132,12 @@ public class InMemoryJournal implements Journal
this.node = node;
}
+ public void dropAll()
+ {
+ diffsPerCommandStore.clear();
+ fieldStates.clear();
+ }
+
@Override
public Command loadCommand(int commandStoreId, TxnId txnId, RedundantBefore redundantBefore, DurableBefore durableBefore)
{
@@ -522,7 +529,11 @@ public class InMemoryJournal implements Journal
Input input = isPartialCompaction ? PARTIAL : FULL;
++counter;
- Cleanup cleanup = builder.shouldCleanup(input, store.unsafeGetRedundantBefore(), store.durableBefore());
+
+ Cleanup cleanup;
+ try {cleanup = builder.shouldCleanup(input, store.unsafeGetRedundantBefore(), store.durableBefore()); }
+ catch (LogUnavailableException ignore) {cleanup = ERASE; }
+
cleanup = builder.maybeCleanup(true, cleanup);
if (cleanup != NO)
{
@@ -581,21 +592,25 @@ public class InMemoryJournal implements Journal
}
Builder before = reconstruct(diffs, ALL);
- before.maybeCleanup(true, FULL, store.unsafeGetRedundantBefore(), store.durableBefore());
+ boolean unavailableBefore = false, unavailableAfter = false;
+ try { before.maybeCleanup(true, FULL, store.unsafeGetRedundantBefore(), store.durableBefore()); }
+ catch (LogUnavailableException ignore) { unavailableBefore = true; }
diffs.size -= removeCount;
diffs.flushed.removeAll(subset.flushed);
diffs.files.removeAll(subset.files);
diffs.files.add(new DiffFile(sorted));
diffs.sorted = null;
Builder after = reconstruct(diffs, ALL);
- after.maybeCleanup(true, FULL, store.unsafeGetRedundantBefore(), store.durableBefore());
+ try { after.maybeCleanup(true, FULL, store.unsafeGetRedundantBefore(), store.durableBefore()); }
+ catch (LogUnavailableException ignore) { unavailableAfter = true; }
+ Invariants.require(unavailableBefore == unavailableAfter);
Invariants.require(Objects.equals(before.construct(store.unsafeGetRedundantBefore()), after.construct(store.unsafeGetRedundantBefore())));
}
}
}
@Override
- public void replay(CommandStores commandStores)
+ public boolean replay(CommandStores commandStores)
{
for (Map.Entry<Integer, NavigableMap<TxnId, Diffs>> diffEntry : diffsPerCommandStore.entrySet())
{
@@ -619,6 +634,7 @@ public class InMemoryJournal implements Journal
}
}
+ return true;
}
static class TruncatedList extends ArrayList<Diff>
diff --git a/accord-core/src/test/java/accord/impl/basic/LoggingJournal.java b/accord-core/src/test/java/accord/impl/basic/LoggingJournal.java
index b57e29f1..00f675fc 100644
--- a/accord-core/src/test/java/accord/impl/basic/LoggingJournal.java
+++ b/accord-core/src/test/java/accord/impl/basic/LoggingJournal.java
@@ -130,9 +130,9 @@ public class LoggingJournal implements Journal
}
@Override
- public void replay(CommandStores commandStores)
+ public boolean replay(CommandStores commandStores)
{
- delegate.replay(commandStores);
+ return delegate.replay(commandStores);
}
@Override
diff --git a/accord-core/src/test/java/accord/impl/basic/RandomDelayQueue.java b/accord-core/src/test/java/accord/impl/basic/RandomDelayQueue.java
index fd37de67..6749faaf 100644
--- a/accord-core/src/test/java/accord/impl/basic/RandomDelayQueue.java
+++ b/accord-core/src/test/java/accord/impl/basic/RandomDelayQueue.java
@@ -25,14 +25,15 @@ import java.util.ArrayList;
import java.util.IdentityHashMap;
import java.util.Iterator;
import java.util.List;
-import java.util.PriorityQueue;
import java.util.concurrent.TimeUnit;
import java.util.function.LongSupplier;
import java.util.function.Predicate;
import java.util.function.Supplier;
+import java.util.stream.Stream;
import accord.burn.random.FrequentLargeRange;
import accord.impl.basic.DelayedCommandStores.DelayedCommandStore.DelayedTask;
+import accord.utils.IntrusivePriorityHeap;
import accord.utils.Invariants;
import accord.utils.RandomSource;
@@ -58,17 +59,23 @@ public class RandomDelayQueue implements PendingQueue
}
}
- static class Item implements Comparable<Item>
+ static class Item extends IntrusivePriorityHeap.Node implements Comparable<Item>
{
- final long time;
- final int seq;
+ long time;
+ int seq;
+ boolean submitted;
final Pending item;
- Item(long time, int seq, Pending item)
+ Item(Pending item)
+ {
+ this.item = item;
+ }
+
+ void submitting(long time, int seq)
{
this.time = time;
this.seq = seq;
- this.item = item;
+ submitted = true;
}
@Override
@@ -102,11 +109,27 @@ public class RandomDelayQueue implements PendingQueue
}
}
- final PriorityQueue<Item> queue = new PriorityQueue<>();
+ static class Queue extends IntrusivePriorityHeap<Item>
+ {
+ @Override public int compare(Item o1, Item o2) { return o1.compareTo(o2); }
+ @Override protected void append(Item node) { super.append(node); }
+ @Override protected void remove(Item node) { super.remove(node); }
+ @Override protected boolean contains(Item node) { return super.contains(node); }
+ @Override protected void clear() { super.clear(); }
+ @Override protected Stream<Item> stream() { return super.stream(); }
+ Item poll()
+ {
+ ensureHeapified();
+ return pollNode();
+ }
+ }
+
+ final Queue queue = new Queue();
+
private final LongSupplier jitterMillis;
long now;
int seq;
- final IdentityHashMap<Pending, Boolean> preregistered = new IdentityHashMap<>();
+ final IdentityHashMap<Pending, Item> preregistered = new IdentityHashMap<>();
int recurring;
public RandomDelayQueue(RandomSource random)
@@ -125,40 +148,60 @@ public class RandomDelayQueue implements PendingQueue
add(item, 0, TimeUnit.NANOSECONDS);
}
+ private Item prepareToSubmit(Pending submitting, long time, int seq)
+ {
+ Item item = preregistered.get(submitting);
+ if (item == null)
+ {
+ item = new Item(submitting);
+ preregistered.put(submitting, item);
+ if (isRecurring(submitting))
+ ++recurring;
+ }
+ Invariants.require(!item.submitted);
+ item.submitting(time, seq);
+ return item;
+ }
+
@Override
- public void addNoDelay(Pending item)
+ public void addNoDelay(Pending add)
{
- if (null == preregistered.remove(item) && isRecurring(item))
- ++recurring;
- queue.add(new Item(now, seq++, item));
+ Item item = prepareToSubmit(add, now, seq++);
+ queue.append(item);
}
@Override
- public void add(Pending item, long delay, TimeUnit units)
+ public void add(Pending add, long delay, TimeUnit units)
{
if (delay < 0)
throw illegalArgument("Delay must be positive or 0, but given " + delay);
- if (null == preregistered.remove(item) && isRecurring(item))
- ++recurring;
- queue.add(new Item(now + units.toMillis(delay) + jitterMillis.getAsLong(), seq++, item));
+ Item item = prepareToSubmit(add, now + units.toMillis(delay) + jitterMillis.getAsLong(), seq++);
+ queue.append(item);
}
@Override
public void preregister(Pending item)
{
- if (null != preregistered.put(item, true))
- throw new IllegalStateException();
+ if (null != preregistered.put(item, new Item(item)))
+ throw illegalState();
if (isRecurring(item))
++recurring;
}
@Override
- public boolean remove(Pending item)
+ public boolean remove(Pending remove)
{
- if (!queue.removeIf(i -> i.item == item) && preregistered.remove(item) == null)
+ Item item = preregistered.get(remove);
+ if (item == null)
return false;
- if (isRecurring(item))
+ Invariants.require(!item.submitted || queue.contains(item));
+
+ preregistered.remove(remove);
+ if (queue.contains(item))
+ queue.remove(item);
+
+ if (isRecurring(remove))
--recurring;
return true;
}
@@ -170,6 +213,7 @@ public class RandomDelayQueue implements PendingQueue
if (item == null)
return null;
+ Invariants.require(item == preregistered.remove(item.item));
if (isRecurring(item.item))
--recurring;
@@ -180,8 +224,9 @@ public class RandomDelayQueue implements PendingQueue
@Override
public List<Pending> drain(Predicate<Pending> toDrain)
{
- List<Item> items = new ArrayList<>(queue);
- queue.clear();
+ List<Item> items = new ArrayList<>(queue.size());
+ for (Item i = queue.poll(); i != null ; i = queue.poll())
+ items.add(i);
List<Pending> ret = new ArrayList<>();
for (Item item : items)
{
@@ -190,10 +235,11 @@ public class RandomDelayQueue implements PendingQueue
if (isRecurring(item.item))
--recurring;
ret.add(item.item);
+ preregistered.remove(item.item);
}
else
{
- queue.add(item);
+ queue.append(item);
}
}
return ret;
@@ -214,7 +260,7 @@ public class RandomDelayQueue implements PendingQueue
@Override
public boolean hasNonRecurring()
{
- return recurring != queue.size() + preregistered.size();
+ return recurring != preregistered.size();
}
@Override
@@ -233,10 +279,10 @@ public class RandomDelayQueue implements PendingQueue
protected abstract void added(Pending pending, long delay);
@Override
- public void add(Pending item, long delay, TimeUnit units)
+ public void add(Pending add, long delay, TimeUnit units)
{
- added(item, units.toNanos(delay));
- super.add(item, delay, units);
+ added(add, units.toNanos(delay));
+ super.add(add, delay, units);
}
}
diff --git a/accord-core/src/test/java/accord/impl/list/ListAgent.java b/accord-core/src/test/java/accord/impl/list/ListAgent.java
index 50730a97..6c280661 100644
--- a/accord-core/src/test/java/accord/impl/list/ListAgent.java
+++ b/accord-core/src/test/java/accord/impl/list/ListAgent.java
@@ -24,7 +24,6 @@ import java.util.Set;
import java.util.concurrent.CancellationException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
-import java.util.function.BiConsumer;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.IntSupplier;
@@ -33,6 +32,7 @@ import java.util.function.LongSupplier;
import javax.annotation.Nullable;
import accord.api.CoordinatorEventListener;
+import accord.api.OwnershipEventListener;
import accord.api.ProgressLog.BlockedUntil;
import accord.api.Result;
import accord.api.Scheduler;
@@ -47,8 +47,10 @@ import accord.impl.basic.NodeSink;
import accord.impl.basic.Packet;
import accord.impl.basic.SimulatedFault;
import accord.impl.mock.Network;
+import accord.local.CommandStore;
import accord.local.Node;
import accord.local.PreLoadContext;
+import accord.local.LogUnavailableException;
import accord.local.SafeCommandStore;
import accord.local.TimeService;
import accord.messages.ReplyContext;
@@ -74,14 +76,14 @@ import static com.google.common.base.Functions.identity;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.SECONDS;
-public class ListAgent implements InMemoryAgent, CoordinatorEventListener
+public class ListAgent implements InMemoryAgent, CoordinatorEventListener, OwnershipEventListener
{
final Scheduler scheduler;
final RandomSource rnd;
final long timeout;
final Consumer<Throwable> onFailure;
final Consumer<Runnable> retryBootstrap;
- final BiConsumer<Timestamp, Ranges> onStale;
+ final OwnershipEventListener ownershipEventListener;
final IntSupplier coordinationDelays;
final IntSupplier progressDelays;
final IntSupplier timeoutDelays;
@@ -90,14 +92,14 @@ public class ListAgent implements InMemoryAgent, CoordinatorEventListener
final NodeSink.TimeoutSupplier timeoutSupplier;
final Int2ObjectHashMap<Snapshotter<Snapshot>> snapshotters = new Int2ObjectHashMap<>();
- public ListAgent(Scheduler scheduler, RandomSource rnd, long timeout, Consumer<Throwable> onFailure, Consumer<Runnable> retryBootstrap, BiConsumer<Timestamp, Ranges> onStale, IntSupplier coordinationDelays, IntSupplier progressDelays, IntSupplier timeoutDelays, LongSupplier queueTimeMillis, TimeService time, NodeSink.TimeoutSupplier timeoutSupplier)
+ public ListAgent(Scheduler scheduler, RandomSource rnd, long timeout, Consumer<Throwable> onFailure, Consumer<Runnable> retryBootstrap, OwnershipEventListener ownershipEventListener, IntSupplier coordinationDelays, IntSupplier progressDelays, IntSupplier timeoutDelays, LongSupplier queueTimeMillis, TimeService time, NodeSink.TimeoutSupplier timeoutSupplier)
{
this.scheduler = scheduler;
this.rnd = rnd;
this.timeout = timeout;
this.onFailure = onFailure;
this.retryBootstrap = retryBootstrap;
- this.onStale = onStale;
+ this.ownershipEventListener = ownershipEventListener;
this.timeoutSupplier = timeoutSupplier;
this.coordinationDelays = coordinationDelays;
this.progressDelays = progressDelays;
@@ -132,9 +134,9 @@ public class ListAgent implements InMemoryAgent, CoordinatorEventListener
}
@Override
- public void onFailedBootstrap(int attempt, String phase, Ranges ranges, Runnable retry, Throwable failure)
+ public OwnershipEventListener ownershipEvents()
{
- retryBootstrap.accept(retry);
+ return this;
}
@Override
@@ -146,10 +148,23 @@ public class ListAgent implements InMemoryAgent, CoordinatorEventListener
@Override
public void onStale(Timestamp staleSince, Ranges ranges)
{
- onStale.accept(staleSince, ranges);
+ ownershipEventListener.onStale(staleSince, ranges);
+ }
+
+ @Override
+ public void onSuccessfulBootstrap(CommandStore commandStore, int attempt, long epoch, Ranges ranges)
+ {
+ ownershipEventListener.onSuccessfulBootstrap(commandStore, attempt, epoch, ranges);
+ }
+
+ @Override
+ public void onFailedBootstrap(int attempt, String phase, Ranges ranges, Runnable retry, Throwable failure)
+ {
+ retryBootstrap.accept(retry);
+ ownershipEventListener.onFailedBootstrap(attempt, phase, ranges, retry, failure);
}
- private static final Set<Class<?>> expectedExceptions = new HashSet<>(Arrays.asList(SimulatedFault.class, ExecuteSyncPoint.SyncPointErased.class, CancellationException.class, TopologyManager.TopologyRetiredException.class, Snapshotter.SnapshotAborted.class, TimeoutException.class));
+ private static final Set<Class<?>> expectedExceptions = new HashSet<>(Arrays.asList(SimulatedFault.class, ExecuteSyncPoint.SyncPointErased.class, CancellationException.class, TopologyManager.TopologyRetiredException.class, Snapshotter.SnapshotAborted.class, TimeoutException.class, LogUnavailableException.class));
@Override
public void onUncaughtException(Throwable t)
{
diff --git a/accord-core/src/test/java/accord/impl/list/ListData.java b/accord-core/src/test/java/accord/impl/list/ListData.java
index f94c3507..03010c7d 100644
--- a/accord-core/src/test/java/accord/impl/list/ListData.java
+++ b/accord-core/src/test/java/accord/impl/list/ListData.java
@@ -65,7 +65,8 @@ public class ListData extends TreeMap<Key, Timestamped<int[]>> implements Data
{
if (v.timestamp.compareTo(executeAt) >= 0)
{
- Invariants.require(futureReadPossible);
+ Invariants.require(futureReadPossible, "For %s, read a timestamp (%s) from future (execute at is %s)",
+ txnId, v.timestamp, executeAt);
return false;
}
Invariants.require(!futureReadPossible || v.timestamp.hlc() < executeAt.hlc());
diff --git a/accord-core/src/test/java/accord/impl/list/ListStore.java b/accord-core/src/test/java/accord/impl/list/ListStore.java
index 4a4c5605..1f92c7ce 100644
--- a/accord-core/src/test/java/accord/impl/list/ListStore.java
+++ b/accord-core/src/test/java/accord/impl/list/ListStore.java
@@ -370,7 +370,7 @@ public class ListStore extends Snapshotter<ListStore.Snapshot> implements DataSt
private final Int2ObjectHashMap<Ranges> pendingFetches = new Int2ObjectHashMap<>();
@Override
- public FetchResult fetch(Node node, SafeCommandStore safeStore, Ranges ranges, SyncPoint syncPoint, FetchRanges delegate)
+ public FetchResult fetch(Node node, SafeCommandStore safeStore, Ranges ranges, SyncPoint syncPoint, FetchRanges delegate, FetchKind kind)
{
int storeId = safeStore.commandStore().id();
synchronized (this)
diff --git a/accord-core/src/test/java/accord/impl/mock/MockConfigurationService.java b/accord-core/src/test/java/accord/impl/mock/MockConfigurationService.java
index 1fd6f759..161540d8 100644
--- a/accord-core/src/test/java/accord/impl/mock/MockConfigurationService.java
+++ b/accord-core/src/test/java/accord/impl/mock/MockConfigurationService.java
@@ -78,14 +78,14 @@ public class MockConfigurationService implements TestableConfigurationService
return;
fetchTopologyHandler.apply(epoch, this);
- return;
}
@Override
- public synchronized void acknowledgeEpoch(EpochReady epoch, boolean startSync)
+ public synchronized boolean acknowledgeEpoch(EpochReady epoch, boolean startSync)
{
Assertions.assertFalse(acks.containsKey(epoch.epoch));
acks.put(epoch.epoch, epoch);
+ return true;
}
public synchronized EpochReady ackFor(long epoch)
diff --git a/accord-core/src/test/java/accord/impl/mock/MockStore.java b/accord-core/src/test/java/accord/impl/mock/MockStore.java
index f1c86c9a..a4fb2093 100644
--- a/accord-core/src/test/java/accord/impl/mock/MockStore.java
+++ b/accord-core/src/test/java/accord/impl/mock/MockStore.java
@@ -151,7 +151,7 @@ public class MockStore implements DataStore
}
@Override
- public FetchResult fetch(Node node, SafeCommandStore safeStore, Ranges ranges, SyncPoint syncPoint, FetchRanges callback)
+ public FetchResult fetch(Node node, SafeCommandStore safeStore, Ranges ranges, SyncPoint syncPoint, FetchRanges callback, FetchKind kind)
{
callback.starting(ranges).started(Timestamp.NONE);
callback.fetched(ranges);
diff --git a/accord-core/src/test/java/accord/local/RedundantBeforeTest.java b/accord-core/src/test/java/accord/local/RedundantBeforeTest.java
index c0b0fec4..cd940a01 100644
--- a/accord-core/src/test/java/accord/local/RedundantBeforeTest.java
+++ b/accord-core/src/test/java/accord/local/RedundantBeforeTest.java
@@ -44,8 +44,8 @@ import org.assertj.core.api.Assertions;
import static accord.impl.IntKey.routing;
import static accord.local.RedundantStatus.ONLY_LE_MASK;
-import static accord.local.RedundantStatus.PRE_BOOTSTRAP_MERGE_MASK;
-import static accord.local.RedundantStatus.Property.PRE_BOOTSTRAP_OR_STALE;
+import static accord.local.RedundantStatus.UNREADY_MERGE_MASK;
+import static accord.local.RedundantStatus.Property.UNREADY;
import static accord.local.RedundantStatus.selectOrCreate;
import static accord.local.RedundantStatus.toAll;
@@ -65,7 +65,7 @@ public class RedundantBeforeTest
static class Unmerged extends RedundantStatus
{
final int insertCounter;
- Unmerged(int encoded, int insertCounter)
+ Unmerged(long encoded, int insertCounter)
{
super(encoded);
this.insertCounter = insertCounter;
@@ -195,20 +195,15 @@ public class RedundantBeforeTest
else return e.getValue().mergedLt;
}
- private static int merge(int a, int b)
- {
- return a | b;
- }
-
private static SomeStatus merge(SomeStatus a, SomeStatus b)
{
- return RedundantStatus.selectOrCreate(merge(a.encoded, b.encoded), a, b);
+ return RedundantStatus.selectOrCreate(a.encoded | b.encoded, a, b);
}
private static SomeStatus filter(SomeStatus a, SomeStatus history)
{
- if (history.is(PRE_BOOTSTRAP_OR_STALE))
- return selectOrCreate((short) (a.encoded & PRE_BOOTSTRAP_MERGE_MASK), a, history);
+ if (history.is(UNREADY))
+ return selectOrCreate((short) (a.encoded & UNREADY_MERGE_MASK), a, history);
return a;
}
diff --git a/accord-core/src/test/java/accord/local/cfk/CommandsForKeyTest.java b/accord-core/src/test/java/accord/local/cfk/CommandsForKeyTest.java
index f43efe15..fa693126 100644
--- a/accord-core/src/test/java/accord/local/cfk/CommandsForKeyTest.java
+++ b/accord-core/src/test/java/accord/local/cfk/CommandsForKeyTest.java
@@ -41,6 +41,7 @@ import accord.api.Agent;
import accord.api.Data;
import accord.api.DataStore;
import accord.api.Journal;
+import accord.api.OwnershipEventListener;
import accord.api.ProgressLog;
import accord.api.ProgressLog.BlockedUntil;
import accord.api.Query;
@@ -632,7 +633,7 @@ public class CommandsForKeyTest
public void testMany()
{
long seed = System.nanoTime();
- for (int i = 0 ; i < 1000 ; ++i)
+ for (int i = 0 ; i < 100 ; ++i)
{
test(seed++, 1000);
}
@@ -942,7 +943,7 @@ public class CommandsForKeyTest
}
}
- private static class TestCommandStore extends CommandStore implements Agent
+ private static class TestCommandStore extends CommandStore implements Agent, OwnershipEventListener
{
static class Task extends AsyncResults.AbstractResult<Void>
{
@@ -1050,6 +1051,12 @@ public class CommandsForKeyTest
throw new UnsupportedOperationException();
}
+ @Override
+ public OwnershipEventListener ownershipEvents()
+ {
+ return this;
+ }
+
@Override
public void onUncaughtException(Throwable t)
{
diff --git a/accord-core/src/test/java/accord/messages/ReadDataTest.java b/accord-core/src/test/java/accord/messages/ReadDataTest.java
index 1d450683..8f91e143 100644
--- a/accord-core/src/test/java/accord/messages/ReadDataTest.java
+++ b/accord-core/src/test/java/accord/messages/ReadDataTest.java
@@ -165,8 +165,6 @@ class ReadDataTest
ReplyContext replyContext = state.process();
- Mockito.verifyNoInteractions(state.sink);
-
state.apply();
state.readResult.setSuccess(Mockito.mock(Data.class));
@@ -300,7 +298,7 @@ class ReadDataTest
{
AsyncResults.SettableResult<Void> writeResult = new AsyncResults.SettableResult<>();
Write write = Mockito.mock(Write.class);
- Mockito.when(write.apply(any(), any(), any(), any(), any())).thenReturn(writeResult.chain());
+ Mockito.when(write.apply(any(), any(), any(), any(), any())).thenAnswer(mock -> writeResult.chain());
Writes writes = new Writes(txnId, executeAt, keys, write);
forEach(store -> check(store.chain(PreLoadContext.contextFor(txnId, "Test"), safe -> {
diff --git a/accord-core/src/test/java/accord/messages/TxnRequestScopeTest.java b/accord-core/src/test/java/accord/messages/RouteRequestScopeTest.java
similarity index 73%
rename from accord-core/src/test/java/accord/messages/TxnRequestScopeTest.java
rename to accord-core/src/test/java/accord/messages/RouteRequestScopeTest.java
index 1ab71499..66683909 100644
--- a/accord-core/src/test/java/accord/messages/TxnRequestScopeTest.java
+++ b/accord-core/src/test/java/accord/messages/RouteRequestScopeTest.java
@@ -34,7 +34,7 @@ import static accord.impl.IntKey.keys;
import static accord.impl.IntKey.range;
import static accord.impl.IntKey.scope;
-public class TxnRequestScopeTest
+public class RouteRequestScopeTest
{
@Test
void createDisjointScopeTest()
@@ -48,14 +48,14 @@ public class TxnRequestScopeTest
Topologies.Multi topologies = new Topologies.Multi((TopologySorter.StaticSorter)(a, b, s)->0, topology2, topology1);
// 3 remains a member across both topologies, so can process requests without waiting for latest topology data
- Assertions.assertEquals(scope(150), ((KeyRoute)TxnRequest.computeScope(id(3), topologies, route)).toParticipants());
- Assertions.assertEquals(1, TxnRequest.computeWaitForEpoch(id(3), topologies, route));
+ Assertions.assertEquals(scope(150), ((KeyRoute) RouteRequest.computeScope(id(3), topologies, route)).toParticipants());
+ Assertions.assertEquals(1, RouteRequest.computeWaitForEpoch(id(3), topologies, route));
// 1 leaves the shard, and 4 joins, so both need the latest information
- Assertions.assertEquals(scope(150), ((KeyRoute)TxnRequest.computeScope(id(1), topologies, route)).toParticipants());
- Assertions.assertEquals(2, TxnRequest.computeWaitForEpoch(id(1), topologies, route));
- Assertions.assertEquals(scope(150), ((KeyRoute)TxnRequest.computeScope(id(4), topologies, route)).toParticipants());
- Assertions.assertEquals(2, TxnRequest.computeWaitForEpoch(id(4), topologies, route));
+ Assertions.assertEquals(scope(150), ((KeyRoute) RouteRequest.computeScope(id(1), topologies, route)).toParticipants());
+ Assertions.assertEquals(2, RouteRequest.computeWaitForEpoch(id(1), topologies, route));
+ Assertions.assertEquals(scope(150), ((KeyRoute) RouteRequest.computeScope(id(4), topologies, route)).toParticipants());
+ Assertions.assertEquals(2, RouteRequest.computeWaitForEpoch(id(4), topologies, route));
}
@Test
@@ -76,9 +76,9 @@ public class TxnRequestScopeTest
Topologies.Multi topologies = new Topologies.Multi((TopologySorter.StaticSorter)(a,b,s)->0, topology2, topology1);
- Assertions.assertEquals(scope(150, 250), ((PartialKeyRoute)TxnRequest.computeScope(id(1), topologies, route)).toParticipants());
- Assertions.assertEquals(2, TxnRequest.computeWaitForEpoch(id(1), topologies, route));
- Assertions.assertEquals(scope(150, 250), ((PartialKeyRoute)TxnRequest.computeScope(id(4), topologies, route)).toParticipants());
- Assertions.assertEquals(2, TxnRequest.computeWaitForEpoch(id(4), topologies, route));
+ Assertions.assertEquals(scope(150, 250), ((PartialKeyRoute) RouteRequest.computeScope(id(1), topologies, route)).toParticipants());
+ Assertions.assertEquals(2, RouteRequest.computeWaitForEpoch(id(1), topologies, route));
+ Assertions.assertEquals(scope(150, 250), ((PartialKeyRoute) RouteRequest.computeScope(id(4), topologies, route)).toParticipants());
+ Assertions.assertEquals(2, RouteRequest.computeWaitForEpoch(id(4), topologies, route));
}
}
diff --git a/accord-core/src/test/java/accord/topology/TopologyManagerTest.java b/accord-core/src/test/java/accord/topology/TopologyManagerTest.java
index 98af652d..a5d6ce8f 100644
--- a/accord-core/src/test/java/accord/topology/TopologyManagerTest.java
+++ b/accord-core/src/test/java/accord/topology/TopologyManagerTest.java
@@ -61,7 +61,7 @@ import static accord.Utils.topology;
import static accord.impl.IntKey.keys;
import static accord.impl.IntKey.range;
import static accord.impl.SizeOfIntersectionSorter.SUPPLIER;
-import static accord.messages.TxnRequest.computeWaitForEpoch;
+import static accord.messages.RouteRequest.computeWaitForEpoch;
import static accord.topology.Topologies.SelectNodeOwnership.SHARE;
import static accord.utils.ExtendedAssertions.assertThat;
import static accord.utils.Property.qt;
diff --git a/accord-core/src/test/java/accord/topology/TopologyRandomizer.java b/accord-core/src/test/java/accord/topology/TopologyRandomizer.java
index fd505ec8..775acced 100644
--- a/accord-core/src/test/java/accord/topology/TopologyRandomizer.java
+++ b/accord-core/src/test/java/accord/topology/TopologyRandomizer.java
@@ -28,7 +28,9 @@ import java.util.Queue;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.TimeUnit;
import java.util.function.BiFunction;
+import java.util.function.BiPredicate;
import java.util.function.Function;
import java.util.function.Supplier;
import javax.annotation.Nullable;
@@ -38,16 +40,20 @@ import com.google.common.collect.Sets;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import accord.api.OwnershipEventListener;
import accord.burn.TopologyUpdates;
import accord.impl.PrefixedIntHashKey;
import accord.impl.PrefixedIntHashKey.Hash;
import accord.impl.PrefixedIntHashKey.PrefixedIntRoutingKey;
+import accord.local.CommandStore;
import accord.local.Node;
import accord.local.Node.Id;
+import accord.local.durability.DurabilityService;
import accord.primitives.Range;
import accord.primitives.Ranges;
import accord.primitives.Routables;
import accord.primitives.Timestamp;
+import accord.primitives.Txn;
import accord.utils.Invariants;
import accord.utils.RandomSource;
import accord.utils.SortedArrays.SortedArrayList;
@@ -55,10 +61,15 @@ import org.agrona.collections.IntHashSet;
import static accord.burn.BurnTestBase.HASH_RANGE_END;
import static accord.burn.BurnTestBase.HASH_RANGE_START;
+import static accord.local.durability.DurabilityService.SyncLocal.NoLocal;
+import static accord.primitives.AbstractRanges.UnionMode.MERGE_ADJACENT;
// TODO (testing): add change replication factor
public class TopologyRandomizer
{
+ // TODO (expected): relax this once we can recover from availability loss
+ public static final int MIN_RF = 3;
+
public interface Listener
{
void onUpdate(Topology topology);
@@ -88,7 +99,8 @@ public class TopologyRandomizer
private final RandomSource random;
private final List<Topology> epochs = new ArrayList<>();
- private final Function<Id, Node> nodeLookup;
+ private final @Nullable Function<Id, Node> nodeLookup;
+ private final Map<Id, Ranges> bootstrapping = new HashMap<>();
private final ConcurrentLinkedQueue<Integer> newPrefixes = new ConcurrentLinkedQueue<>();
// TODO (required): remove this restriction, we should be able to replicate previously owned ranges just fine
private final Map<Id, Ranges> previouslyReplicated = new HashMap<>();
@@ -138,6 +150,17 @@ public class TopologyRandomizer
}
}
+ public boolean isBootstrapping(Node.Id node)
+ {
+ return bootstrapping.containsKey(node);
+ }
+
+ public void markRebootstrapping(Node node)
+ {
+ Ranges ranges = node.topology().currentLocal().ranges();
+ bootstrapping.merge(node.id(), ranges, Ranges::with);
+ }
+
private static Shard[] updateBoundary(Shard[] shards, RandomSource random)
{
int idx = random.nextInt(shards.length - 1);
@@ -340,6 +363,7 @@ public class TopologyRandomizer
Arrays.sort(result);
nodes = result;
}
+
int rf;
if (nodes.length <= 3)
{
@@ -348,11 +372,11 @@ public class TopologyRandomizer
else
{
float chance = random.nextFloat();
- if (chance < 0.2f) { rf = random.nextInt(2, Math.min(9, nodes.length)); }
- else if (chance < 0.4f) { rf = 3; }
- else if (chance < 0.7f) { rf = Math.min(5, nodes.length); }
- else if (chance < 0.8f) { rf = Math.min(7, nodes.length); }
- else { rf = Math.min(9, nodes.length); }
+ if (chance < 0.2f) { rf = random.nextInt(MIN_RF, Math.min(random.nextInt(5, 9), nodes.length)); }
+ else if (chance < 0.4f) { rf = 3; }
+ else if (chance < 0.7f) { rf = Math.min(5, nodes.length); }
+ else if (chance < 0.95f) { rf = Math.min(7, nodes.length); }
+ else { rf = Math.min(9, nodes.length); }
}
List<Shard> result = new ArrayList<>(shards.length + nodes.length);
result.addAll(Arrays.asList(shards));
@@ -439,6 +463,7 @@ public class TopologyRandomizer
updateTopology();
}
+ private int counter = 0;
public synchronized Topology updateTopology()
{
Topology current = epochs.get(epochs.size() - 1);
@@ -451,13 +476,13 @@ public class TopologyRandomizer
state.newPrefixes = newPrefixes;
while (remainingMutations > 0 && rejectedMutations < 10)
{
+ ++counter;
UpdateType type = UpdateType.kind(random);
state.shards = newShards;
Shard[] testShards = type.apply(state, random);
Arrays.sort(testShards, (a, b) -> a.range.compareTo(b.range));
- if (!everyShardHasOverlaps(oldShards, testShards)
- || reassignsRanges(current, testShards, previouslyReplicated)
- )
+ if (!everyShardHasQuorumOverlaps(oldShards, testShards)
+ || reassignsRanges(current, testShards, previouslyReplicated))
{
++rejectedMutations;
}
@@ -476,13 +501,10 @@ public class TopologyRandomizer
Map<Id, Ranges> nextAdditions = getAdditions(current, nextTopology);
for (Map.Entry<Id, Ranges> entry : nextAdditions.entrySet())
{
- Ranges previous = previouslyReplicated.getOrDefault(entry.getKey(), Ranges.EMPTY);
- Ranges added = entry.getValue();
- Ranges merged = previous.with(added).mergeTouching();
- previouslyReplicated.put(entry.getKey(), merged);
+ previouslyReplicated.merge(entry.getKey(), entry.getValue(), (a, b) -> a.union(MERGE_ADJACENT, b));
+ bootstrapping.merge(entry.getKey(), entry.getValue(), Ranges::with);
}
-// logger.debug("topology update to: {} from: {}", nextTopology, current);
epochs.add(nextTopology);
listener.onUpdate(nextTopology);
@@ -495,15 +517,43 @@ public class TopologyRandomizer
return nextTopology;
}
- private boolean everyShardHasOverlaps(Shard[] in, Shard[] out)
+ // TODO (expected): relax this to checking only that we have fewer than maxFailures intersections
+ public boolean unsafeToRebootstrap(Ranges ranges)
+ {
+ for (Ranges test : bootstrapping.values())
+ {
+ if (ranges.intersects(test))
+ return true;
+ }
+ return false;
+ }
+
+ private boolean everyShardHasQuorumOverlaps(Shard[] in, Shard[] out)
+ {
+ return testChanges(in, out, (iv, ov) -> {
+ // TODO (expected): support availability loss and recovery, and minority quorums
+ int common = (int) ov.nodes.stream().filter(iv::contains).count();
+ int commonBootstrap = (int) ov.nodes.stream().filter(iv::contains).filter(id -> {
+ Ranges ranges = bootstrapping.getOrDefault(id, Ranges.EMPTY);
+ return ranges.intersects(iv.range) || ranges.intersects(ov.range);
+ }).count();
+ int inBootstrap = (int) iv.nodes.stream().filter(id -> bootstrapping.getOrDefault(id, Ranges.EMPTY).intersects(iv.range)).count();
+ return inBootstrap <= iv.maxFailures && commonBootstrap + (ov.rf - common) <= ov.maxFailures;
+ });
+ }
+
+ private boolean testChanges(Shard[] in, Shard[] out, BiPredicate<Shard, Shard> consumer)
{
int i = 0, o = 0;
while (i < in.length && o < out.length)
{
Shard iv = in[i];
Shard ov = out[o];
- if (ov.nodes.stream().filter(iv::contains).allMatch(id -> topologyUpdates.isPending(ov.range, id)))
- return false;
+ if (ov != iv)
+ {
+ if (!consumer.test(iv, ov))
+ return false;
+ }
int c = iv.range.end().compareTo(ov.range.end());
if (c <= 0) ++i;
if (c >= 0) ++o;
@@ -511,16 +561,45 @@ public class TopologyRandomizer
return true;
}
- public void onStale(Id id, Timestamp sinceAtLeast, Ranges ranges)
+ public OwnershipEventListener listener(Id id)
{
- int epoch = (int) sinceAtLeast.epoch();
- Invariants.require(epochs.get(epoch).nodeLookup.get(id.id).ranges.containsAll(ranges));
- while (++epoch < epochs.size())
+ return new OwnershipEventListener()
{
- ranges = ranges.slice(epochs.get(epoch).nodeLookup.get(id.id).ranges, Routables.Slice.Minimal);
- if (ranges.isEmpty())
- return;
- }
- Invariants.illegalState("Stale ranges: " + ranges + ", but "+ id + " still replicates these ranges");
+ @Override
+ public void onFailedBootstrap(int attempt, String phase, Ranges ranges, Runnable retry, Throwable failure)
+ {
+ }
+
+ @Override
+ public void onSuccessfulBootstrap(CommandStore commandStore, int attempt, long epoch, Ranges ranges)
+ {
+ Node node = nodeLookup.apply(id);
+ node.durability().sync("Rebootstrap", Txn.Kind.ExclusiveSyncPoint, ranges, NoLocal, DurabilityService.SyncRemote.Quorum, 100L, TimeUnit.DAYS)
+ .flatMap(ignore -> commandStore.awaitVisibility(epoch, ranges))
+ .invoke((success, fail) -> {
+ Invariants.require(fail == null);
+ bootstrapping.compute(id, (ignore, rs) -> {
+ Invariants.require(rs != null && rs.intersects(ranges));
+ rs = rs.without(ranges);
+ return rs.isEmpty() ? null : rs;
+ });
+ });
+ }
+
+ @Override
+ public void onStale(Timestamp sinceAtLeast, Ranges ranges)
+ {
+ int epoch = (int) sinceAtLeast.epoch();
+ Invariants.require(epochs.get(epoch).nodeLookup.get(id.id).ranges.containsAll(ranges));
+ while (++epoch < epochs.size())
+ {
+ ranges = ranges.slice(epochs.get(epoch).nodeLookup.get(id.id).ranges, Routables.Slice.Minimal);
+ if (ranges.isEmpty())
+ return;
+ }
+ Invariants.illegalState("Stale ranges: " + ranges + ", but "+ id + " still replicates these ranges");
+ }
+ };
}
+
}
diff --git a/accord-core/src/test/java/accord/utils/ExtendedAssertions.java b/accord-core/src/test/java/accord/utils/ExtendedAssertions.java
index 51dada3f..00b4b2b1 100644
--- a/accord-core/src/test/java/accord/utils/ExtendedAssertions.java
+++ b/accord-core/src/test/java/accord/utils/ExtendedAssertions.java
@@ -27,7 +27,7 @@ import accord.local.Node;
import accord.messages.PreAccept;
import accord.messages.Reply;
import accord.messages.ReplyContext;
-import accord.messages.TxnRequest;
+import accord.messages.RouteRequest;
import accord.primitives.Range;
import accord.primitives.Ranges;
import accord.primitives.Unseekables;
@@ -248,7 +248,7 @@ public class ExtendedAssertions
}
}
- public static <T extends Reply> ObjectAssert<T> process(TxnRequest<?> request, Node on, Node.Id replyTo, Class<T> replyType)
+ public static <T extends Reply> ObjectAssert<T> process(RouteRequest<?> request, Node on, Node.Id replyTo, Class<T> replyType)
{
ReplyContext replyContext = Mockito.mock(ReplyContext.class);
request.process(on, replyTo, replyContext);
diff --git a/accord-maelstrom/src/main/java/accord/maelstrom/Cluster.java b/accord-maelstrom/src/main/java/accord/maelstrom/Cluster.java
index ffa41145..308ae492 100644
--- a/accord-maelstrom/src/main/java/accord/maelstrom/Cluster.java
+++ b/accord-maelstrom/src/main/java/accord/maelstrom/Cluster.java
@@ -396,7 +396,7 @@ public class Cluster implements Scheduler
@Override public List<TopologyUpdate> replayTopologies() { throw new IllegalStateException("Not impelemented"); }
@Override public void saveTopology(TopologyUpdate topologyUpdate, Runnable onFlush) { throw new IllegalStateException("Not impelemented"); }
@Override public void purge(CommandStores commandStores, EpochSupplier minEpoch) { throw new IllegalStateException("Not impelemented"); }
- @Override public void replay(CommandStores commandStores) { throw new IllegalStateException("Not impelemented"); }
+ @Override public boolean replay(CommandStores commandStores) { throw new IllegalStateException("Not impelemented"); }
@Override public RedundantBefore loadRedundantBefore(int store) { throw new IllegalStateException("Not impelemented"); }
@Override public NavigableMap<TxnId, Ranges> loadBootstrapBeganAt(int store) { throw new IllegalStateException("Not impelemented"); }
@Override public NavigableMap<Timestamp, Ranges> loadSafeToRead(int store) { throw new IllegalStateException("Not impelemented"); }
diff --git a/accord-maelstrom/src/main/java/accord/maelstrom/MaelstromAgent.java b/accord-maelstrom/src/main/java/accord/maelstrom/MaelstromAgent.java
index 01620da7..2d827b56 100644
--- a/accord-maelstrom/src/main/java/accord/maelstrom/MaelstromAgent.java
+++ b/accord-maelstrom/src/main/java/accord/maelstrom/MaelstromAgent.java
@@ -22,6 +22,7 @@ import java.util.concurrent.TimeUnit;
import accord.api.Agent;
import accord.api.CoordinatorEventListener;
+import accord.api.OwnershipEventListener;
import accord.api.ProgressLog;
import accord.api.Result;
import accord.local.Node;
@@ -44,7 +45,7 @@ import static java.util.concurrent.TimeUnit.MICROSECONDS;
import static java.util.concurrent.TimeUnit.MINUTES;
import static java.util.concurrent.TimeUnit.SECONDS;
-public class MaelstromAgent implements Agent, CoordinatorEventListener
+public class MaelstromAgent implements Agent, CoordinatorEventListener, OwnershipEventListener
{
static final MaelstromAgent INSTANCE = new MaelstromAgent();
@@ -74,6 +75,12 @@ public class MaelstromAgent implements Agent, CoordinatorEventListener
{
}
+ @Override
+ public OwnershipEventListener ownershipEvents()
+ {
+ return this;
+ }
+
@Override
public CoordinatorEventListener coordinatorEvents()
{
diff --git a/accord-maelstrom/src/main/java/accord/maelstrom/MaelstromStore.java b/accord-maelstrom/src/main/java/accord/maelstrom/MaelstromStore.java
index e5ed170f..241100c5 100644
--- a/accord-maelstrom/src/main/java/accord/maelstrom/MaelstromStore.java
+++ b/accord-maelstrom/src/main/java/accord/maelstrom/MaelstromStore.java
@@ -56,7 +56,7 @@ public class MaelstromStore implements DataStore
}
@Override
- public FetchResult fetch(Node node, SafeCommandStore safeStore, Ranges ranges, SyncPoint syncPoint, FetchRanges callback)
+ public FetchResult fetch(Node node, SafeCommandStore safeStore, Ranges ranges, SyncPoint syncPoint, FetchRanges callback, FetchKind kind)
{
return new ImmediateFetchResult(ranges);
}
diff --git a/accord-maelstrom/src/main/java/accord/maelstrom/SimpleConfigService.java b/accord-maelstrom/src/main/java/accord/maelstrom/SimpleConfigService.java
index e4e36644..fa014ff2 100644
--- a/accord-maelstrom/src/main/java/accord/maelstrom/SimpleConfigService.java
+++ b/accord-maelstrom/src/main/java/accord/maelstrom/SimpleConfigService.java
@@ -56,8 +56,9 @@ public class SimpleConfigService implements ConfigurationService
}
@Override
- public void acknowledgeEpoch(EpochReady ready, boolean startSync)
+ public boolean acknowledgeEpoch(EpochReady ready, boolean startSync)
{
+ return true;
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org