You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by if...@apache.org on 2024/09/30 15:01:34 UTC
(cassandra-accord) 01/09: squash 19869 to 80414c2e
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch ex-sp-pl
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git
commit 34c35d8c7678f0aa4c8f57c473fc8871851534a5
Author: Alex Petrov <ol...@gmail.com>
AuthorDate: Tue Jul 30 20:57:04 2024 +0200
squash 19869 to 80414c2e
---
accord-core/build.gradle | 8 +-
.../src/main/java/accord/api/DataStore.java | 2 +
.../src/main/java/accord/api/ProgressLog.java | 3 +
.../main/java/accord/coordinate/ExecuteTxn.java | 9 +
.../coordinate/tracking/AbstractTracker.java | 2 +-
.../java/accord/impl/InMemoryCommandStore.java | 256 +++++++---
.../main/java/accord/impl/TimestampsForKeys.java | 2 +-
.../accord/impl/progresslog/CallbackInvoker.java | 6 +
.../impl/progresslog/DefaultProgressLog.java | 23 +-
.../java/accord/impl/progresslog/WaitingState.java | 3 +-
.../src/main/java/accord/local/Bootstrap.java | 34 +-
.../src/main/java/accord/local/Cleanup.java | 14 +-
.../src/main/java/accord/local/Command.java | 5 +-
.../src/main/java/accord/local/CommandStore.java | 248 ++++++----
.../src/main/java/accord/local/CommandStores.java | 50 +-
.../src/main/java/accord/local/Commands.java | 67 ++-
.../src/main/java/accord/local/DurableBefore.java | 5 +
.../src/main/java/accord/local/PreLoadContext.java | 61 ++-
.../main/java/accord/local/RedundantBefore.java | 41 +-
.../main/java/accord/local/SafeCommandStore.java | 68 ++-
.../main/java/accord/local/cfk/CommandsForKey.java | 18 +-
.../src/main/java/accord/local/cfk/NotifySink.java | 5 +-
.../src/main/java/accord/local/cfk/Updating.java | 5 +-
.../src/main/java/accord/messages/PreAccept.java | 1 +
.../java/accord/messages/SetGloballyDurable.java | 5 +-
.../src/main/java/accord/primitives/Writes.java | 4 +
.../src/main/java/accord/topology/Topologies.java | 42 +-
.../src/main/java/accord/topology/Topology.java | 10 +-
.../src/main/java/accord/utils/LogGroupTimers.java | 21 +
.../java/accord/utils/TriConsumer.java} | 14 +-
.../main/java/accord/utils/async/AsyncChain.java | 11 +
.../test/java/accord/impl/RemoteListenersTest.java | 13 +-
.../src/test/java/accord/impl/basic/Cluster.java | 135 +++++-
.../accord/impl/basic/DelayedCommandStores.java | 32 +-
.../src/test/java/accord/impl/basic/Journal.java | 518 ++++++++++++---------
.../src/test/java/accord/impl/basic/NodeSink.java | 9 +-
.../test/java/accord/impl/basic/PendingQueue.java | 7 +-
.../accord/impl/basic/PropagatingPendingQueue.java | 20 +
.../java/accord/impl/basic/RandomDelayQueue.java | 54 ++-
.../impl/basic/RecurringPendingRunnable.java | 14 +-
.../accord/impl/list/ListFetchCoordinator.java | 2 +-
.../src/test/java/accord/impl/list/ListStore.java | 104 ++++-
.../src/test/java/accord/impl/mock/MockStore.java | 8 +
.../java/accord/local/BootstrapLocalTxnTest.java | 127 -----
.../java/accord/local/cfk/CommandsForKeyTest.java | 28 +-
.../verify/StrictSerializabilityVerifier.java | 4 +-
.../main/java/accord/maelstrom/MaelstromStore.java | 9 +
47 files changed, 1416 insertions(+), 711 deletions(-)
diff --git a/accord-core/build.gradle b/accord-core/build.gradle
index d87e5000..b30737c1 100644
--- a/accord-core/build.gradle
+++ b/accord-core/build.gradle
@@ -58,7 +58,7 @@ dependencies {
task burn(type: JavaExec) {
classpath sourceSets.main.runtimeClasspath, sourceSets.test.runtimeClasspath
mainClass = 'accord.burn.BurnTest'
- jvmArgs '-Dlogback.configurationFile=burn-logback.xml'
+ jvmArgs '-Dlogback.configurationFile=burn-logback.xml -Xmx4g'
args = ['-c', '1']
}
@@ -122,7 +122,7 @@ task burnloop {
javaexec {
classpath sourceSets.main.runtimeClasspath, sourceSets.test.runtimeClasspath
mainClass = 'accord.burn.BurnTest'
- jvmArgs '-Dlogback.configurationFile=burn-logback.xml'
+ jvmArgs '-Dlogback.configurationFile=burn-logback.xml -Xmx4g'
args = ['-c', project.hasProperty('burnTimes') ? project.getProperty('burnTimes') : '1000000']
}
}
@@ -142,4 +142,8 @@ task allTestJar(type: Jar) {
from { configurations.testRuntimeClasspath.collect { it.isDirectory() ? it : zipTree(it) } }
from { sourceSets.test.output }
with jar
+}
+
+test {
+ maxHeapSize = '4g'
}
\ No newline at end of file
diff --git a/accord-core/src/main/java/accord/api/DataStore.java b/accord-core/src/main/java/accord/api/DataStore.java
index d71c742b..623b021f 100644
--- a/accord-core/src/main/java/accord/api/DataStore.java
+++ b/accord-core/src/main/java/accord/api/DataStore.java
@@ -25,6 +25,7 @@ import accord.local.SafeCommandStore;
import accord.primitives.Ranges;
import accord.primitives.SyncPoint;
import accord.primitives.Timestamp;
+import accord.primitives.TxnId;
import accord.utils.async.AsyncResult;
/**
@@ -111,4 +112,5 @@ public interface DataStore
}
FetchResult fetch(Node node, SafeCommandStore safeStore, Ranges ranges, SyncPoint syncPoint, FetchRanges callback);
+ AsyncResult<Void> snapshot(Ranges ranges, TxnId before);
}
diff --git a/accord-core/src/main/java/accord/api/ProgressLog.java b/accord-core/src/main/java/accord/api/ProgressLog.java
index 3a82ccd3..6bdcefae 100644
--- a/accord-core/src/main/java/accord/api/ProgressLog.java
+++ b/accord-core/src/main/java/accord/api/ProgressLog.java
@@ -185,11 +185,14 @@ public interface ProgressLog
*/
void clear(TxnId txnId);
+ void clear();
+
class NoOpProgressLog implements ProgressLog
{
@Override public void update(SafeCommandStore safeStore, TxnId txnId, Command before, Command after) {}
@Override public void remoteCallback(SafeCommandStore safeStore, SafeCommand safeCommand, SaveStatus remoteStatus, int callbackId, Node.Id from) {}
@Override public void waiting(BlockedUntil blockedUntil, SafeCommandStore safeStore, SafeCommand blockedBy, Route<?> blockedOnRoute, Participants<?> blockedOnParticipants) {}
@Override public void clear(TxnId txnId) {}
+ @Override public void clear() {}
}
}
diff --git a/accord-core/src/main/java/accord/coordinate/ExecuteTxn.java b/accord-core/src/main/java/accord/coordinate/ExecuteTxn.java
index 34d3e095..3f689156 100644
--- a/accord-core/src/main/java/accord/coordinate/ExecuteTxn.java
+++ b/accord-core/src/main/java/accord/coordinate/ExecuteTxn.java
@@ -159,4 +159,13 @@ public class ExecuteTxn extends ReadCoordinator<ReadReply>
{
return node.coordinationAdapter(txnId, Continue);
}
+
+ @Override
+ public String toString()
+ {
+ return "ExecuteTxn{" +
+ "txn=" + txn +
+ ", route=" + route +
+ '}';
+ }
}
diff --git a/accord-core/src/main/java/accord/coordinate/tracking/AbstractTracker.java b/accord-core/src/main/java/accord/coordinate/tracking/AbstractTracker.java
index e58f3c67..183b4879 100644
--- a/accord-core/src/main/java/accord/coordinate/tracking/AbstractTracker.java
+++ b/accord-core/src/main/java/accord/coordinate/tracking/AbstractTracker.java
@@ -180,7 +180,7 @@ public abstract class AbstractTracker<ST extends ShardTracker>
return topologies.nodes();
}
- public Set<Id> nonStaleNodes()
+ public Collection<Id> nonStaleNodes()
{
return topologies.nonStaleNodes();
}
diff --git a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
index bcc6d6f7..d932f6b8 100644
--- a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
+++ b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
@@ -42,22 +42,33 @@ import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import com.google.common.annotations.VisibleForTesting;
-
-import accord.api.LocalListeners;
-import accord.impl.progresslog.DefaultProgressLog;
-import accord.local.*;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import accord.api.Agent;
import accord.api.DataStore;
import accord.api.Key;
+import accord.api.LocalListeners;
import accord.api.ProgressLog;
+import accord.impl.progresslog.DefaultProgressLog;
+import accord.local.Cleanup;
+import accord.local.Command;
+import accord.local.CommandStore;
import accord.local.CommandStores.RangesForEpoch;
+import accord.local.Commands;
+import accord.local.KeyHistory;
+import accord.local.Node;
+import accord.local.NodeTimeService;
+import accord.local.PreLoadContext;
+import accord.local.RedundantStatus;
+import accord.local.SafeCommand;
+import accord.local.SafeCommandStore;
+import accord.local.SaveStatus;
+import accord.local.Status;
import accord.local.cfk.CommandsForKey;
import accord.primitives.AbstractKeys;
import accord.primitives.Deps;
+import accord.primitives.Keys;
import accord.primitives.PartialDeps;
import accord.primitives.Participants;
import accord.primitives.Range;
@@ -73,22 +84,24 @@ import accord.primitives.Timestamp;
import accord.primitives.Txn.Kind.Kinds;
import accord.primitives.TxnId;
import accord.utils.Invariants;
+import accord.utils.ReducingRangeMap;
import accord.utils.async.AsyncChain;
import accord.utils.async.AsyncChains;
import static accord.local.SafeCommandStore.TestDep.ANY_DEPS;
import static accord.local.SafeCommandStore.TestDep.WITH;
-import static accord.local.SafeCommandStore.TestStatus.ANY_STATUS;
import static accord.local.SafeCommandStore.TestStartedAt.STARTED_BEFORE;
+import static accord.local.SafeCommandStore.TestStatus.ANY_STATUS;
import static accord.local.SaveStatus.Applying;
import static accord.local.SaveStatus.Erased;
import static accord.local.SaveStatus.ErasedOrInvalidOrVestigial;
import static accord.local.SaveStatus.ReadyToExecute;
import static accord.local.Status.Applied;
+import static accord.local.Status.Invalidated;
+import static accord.local.Status.NotDefined;
import static accord.local.Status.PreCommitted;
import static accord.local.Status.Stable;
import static accord.local.Status.Truncated;
-import static accord.local.Status.NotDefined;
import static accord.primitives.Routables.Slice.Minimal;
import static accord.utils.Invariants.illegalState;
import static java.lang.String.format;
@@ -102,10 +115,11 @@ public abstract class InMemoryCommandStore extends CommandStore
final NavigableMap<Timestamp, GlobalCommand> commandsByExecuteAt = new TreeMap<>();
private final NavigableMap<RoutableKey, GlobalTimestampsForKey> timestampsForKey = new TreeMap<>();
private final NavigableMap<RoutableKey, GlobalCommandsForKey> commandsForKey = new TreeMap<>();
- // TODO (find library, efficiency): this is obviously super inefficient, need some range map
+ // TODO (find library, efficiency): this is obviously super inefficient, need some range map
private final TreeMap<TxnId, RangeCommand> rangeCommands = new TreeMap<>();
private final TreeMap<TxnId, Ranges> historicalRangeCommands = new TreeMap<>();
+ // TODO (desired): use `redundantBefore` information instead
protected Timestamp maxRedundant = Timestamp.NONE;
private InMemorySafeStore current;
@@ -149,11 +163,6 @@ public abstract class InMemoryCommandStore extends CommandStore
return historicalRangeCommands;
}
- TreeMap<TxnId, RangeCommand> rangeCommands()
- {
- return rangeCommands;
- }
-
public GlobalCommand commandIfPresent(TxnId txnId)
{
return commands.get(txnId);
@@ -362,7 +371,11 @@ public abstract class InMemoryCommandStore extends CommandStore
public void markShardDurable(SafeCommandStore safeStore, TxnId syncId, Ranges ranges)
{
super.markShardDurable(safeStore, syncId, ranges);
+ markShardDurable(syncId, ranges);
+ }
+ private void markShardDurable(TxnId syncId, Ranges ranges)
+ {
if (!rangeCommands.containsKey(syncId))
historicalRangeCommands.merge(syncId, ranges, Ranges::with);
@@ -408,8 +421,6 @@ public abstract class InMemoryCommandStore extends CommandStore
}, 5L, TimeUnit.SECONDS);
}
-
-
protected InMemorySafeStore createSafeStore(PreLoadContext context, RangesForEpoch ranges,
Map<TxnId, InMemorySafeCommand> commands,
Map<RoutableKey, InMemorySafeTimestampsForKey> timestampsForKey,
@@ -418,15 +429,6 @@ public abstract class InMemoryCommandStore extends CommandStore
return new InMemorySafeStore(this, ranges, context, commands, timestampsForKey, commandsForKeys);
}
- private void loadCommandsForKey(RoutableKey key,
- KeyHistory keyHistory,
- Map<RoutableKey, InMemorySafeTimestampsForKey> timestampsForKey,
- Map<RoutableKey, InMemorySafeCommandsForKey> commandsForKey)
- {
- commandsForKey.put(key, commandsForKey((Key) key).createSafeReference());
- timestampsForKey.put(key, timestampsForKey((Key) key).createSafeReference());
- }
-
protected void validateRead(Command current) {}
protected final InMemorySafeStore createSafeStore(PreLoadContext context, RangesForEpoch ranges)
@@ -475,7 +477,8 @@ public abstract class InMemoryCommandStore extends CommandStore
{
if (current != null)
throw illegalState("Another operation is in progress or it's store was not cleared");
- current = createSafeStore(context, updateRangesForEpoch());
+ current = createSafeStore(context, rangesForEpoch);
+ updateRangesForEpoch(current);
return current;
}
@@ -509,7 +512,7 @@ public abstract class InMemoryCommandStore extends CommandStore
catch (Throwable t)
{
if (isDirectCall) logger.error("Uncaught exception", t);
- throw t;
+ throw new RuntimeException("Caught exception in command store " + this, t);
}
finally
{
@@ -658,7 +661,7 @@ public abstract class InMemoryCommandStore extends CommandStore
protected final Map<TxnId, InMemorySafeCommand> commands;
private final Map<RoutableKey, InMemorySafeTimestampsForKey> timestampsForKey;
private final Map<RoutableKey, InMemorySafeCommandsForKey> commandsForKey;
- private final RangesForEpoch ranges;
+ private RangesForEpoch ranges;
public InMemorySafeStore(InMemoryCommandStore commandStore,
RangesForEpoch ranges,
@@ -672,7 +675,7 @@ public abstract class InMemoryCommandStore extends CommandStore
this.commands = commands;
this.commandsForKey = commandsForKey;
this.timestampsForKey = timestampsForKey;
- this.ranges = Invariants.nonNull(ranges);
+ this.ranges = ranges;
}
@Override
@@ -734,7 +737,7 @@ public abstract class InMemoryCommandStore extends CommandStore
{
if (!commandStore.canExposeUnloaded())
return null;
- GlobalCommandsForKey global = commandStore.commandsForKeyIfPresent((Key) key);
+ GlobalCommandsForKey global = commandStore.commandsForKeyIfPresent(key);
return global != null ? global.createSafeReference() : null;
}
@@ -760,7 +763,6 @@ public abstract class InMemoryCommandStore extends CommandStore
slice = commandStore.redundantBefore().removeShardRedundant(txnId, updated.executeAtOrTxnId(), slice);
commandStore.rangeCommands.computeIfAbsent(txnId, ignore -> new RangeCommand(commandStore.commands.get(txnId)))
.update(((Ranges)keysOrRanges).slice(slice, Minimal));
-
}
@Override
@@ -793,41 +795,11 @@ public abstract class InMemoryCommandStore extends CommandStore
return ranges;
}
- // TODO (desired): this can have protected visibility if under CommandStore, and this is perhaps a better place to put it also
@Override
- public void registerHistoricalTransactions(Deps deps)
- {
- RangesForEpoch rangesForEpoch = commandStore.rangesForEpoch;
- Ranges allRanges = rangesForEpoch.all();
- deps.keyDeps.keys().forEach(allRanges, key -> {
- deps.keyDeps.forEach(key, (txnId, txnIdx) -> {
- // TODO (desired, efficiency): this can be made more efficient by batching by epoch
- if (rangesForEpoch.coordinates(txnId).contains(key))
- return; // already coordinates, no need to replicate
- // TODO (required): check this logic, esp. next line, matches C*
- if (!rangesForEpoch.allAfter(txnId.epoch()).contains(key))
- return;
-
- get(key).registerHistorical(this, txnId);
- });
-
- });
- TreeMap<TxnId, RangeCommand> rangeCommands = commandStore.rangeCommands();
- TreeMap<TxnId, Ranges> historicalRangeCommands = commandStore.historicalRangeCommands();
- deps.rangeDeps.forEachUniqueTxnId(allRanges, null, (ignore, txnId) -> {
-
- if (rangeCommands.containsKey(txnId))
- return;
-
- Ranges ranges = deps.rangeDeps.ranges(txnId);
- if (rangesForEpoch.coordinates(txnId).intersects(ranges))
- return; // already coordinates, no need to replicate
- // TODO (required): check this logic, esp. next line, matches C*
- if (!rangesForEpoch.allAfter(txnId.epoch()).intersects(ranges))
- return;
-
- historicalRangeCommands.merge(txnId, ranges.slice(allRanges), Ranges::with);
- });
+ public void setRangesForEpoch(RangesForEpoch rangesForEpoch)
+ {
+ super.setRangesForEpoch(rangesForEpoch);
+ ranges = rangesForEpoch;
}
@Override
@@ -1338,4 +1310,160 @@ public abstract class InMemoryCommandStore extends CommandStore
logDependencyGraph(commandStore, txnId, true);
}
}
+
+ /**
+ * Replay and loading logic
+ */
+
+ // redundantBefore, durableBefore, newBootstrapBeganAt, safeToRead, rangesForEpoch are
+ // not replayed here. It is assumed that persistence on the application side will ensure
+ // they are brought up to latest values _before_ replay.
+ public void clearForTesting()
+ {
+ Invariants.checkState(current == null);
+ progressLog.clear();
+ commands.clear();
+ commandsByExecuteAt.clear();
+ timestampsForKey.clear();
+ commandsForKey.clear();
+ rangeCommands.clear();
+ historicalRangeCommands.clear();
+ unsafeSetRejectBefore(new ReducingRangeMap<>());
+ }
+
+ public interface Loader
+ {
+ void load(Command next);
+ void apply(Command next);
+ }
+
+ public Loader loader()
+ {
+ return new Loader()
+ {
+ private PreLoadContext context(Command command, KeyHistory keyHistory)
+ {
+ TxnId txnId = command.txnId();
+ Keys keys = null;
+
+ if (CommandsForKey.manages(txnId))
+ keys = (Keys) command.keysOrRanges();
+ else if (!CommandsForKey.managesExecution(txnId) && command.hasBeen(Status.Stable) && !command.hasBeen(Status.Truncated))
+ keys = command.asCommitted().waitingOn.keys;
+
+ if (keys != null)
+ {
+ return PreLoadContext.contextFor(txnId, keys, keyHistory);
+ }
+
+ return PreLoadContext.contextFor(txnId);
+ }
+
+ public void load(Command command)
+ {
+ TxnId txnId = command.txnId();
+
+ executeInContext(InMemoryCommandStore.this,
+ context(command, KeyHistory.COMMANDS),
+ safeStore -> {
+ Command local = command;
+ if (local.status() != Truncated && local.status() != Invalidated)
+ {
+ Cleanup cleanup = Cleanup.shouldCleanup(InMemoryCommandStore.this, local, null, local.route(), false);
+ switch (cleanup)
+ {
+ case NO:
+ break;
+ case INVALIDATE:
+ case TRUNCATE_WITH_OUTCOME:
+ case TRUNCATE:
+ case ERASE:
+ local = Commands.purge(local, local.route(), cleanup);
+ }
+ }
+
+ local = safeStore.unsafeGet(txnId).update(safeStore, local);
+ if (local.status() == Truncated)
+ safeStore.progressLog().clear(local.txnId());
+ return local;
+ });
+
+
+ }
+
+ public void apply(Command command)
+ {
+ TxnId txnId = command.txnId();
+
+ PreLoadContext context = context(command, KeyHistory.TIMESTAMPS);
+ executeInContext(InMemoryCommandStore.this,
+ context,
+ safeStore -> {
+ SafeCommand safeCommand = safeStore.unsafeGet(txnId);
+ Command local = safeCommand.current();
+ if (local.is(Stable) && !local.hasBeen(Applied))
+ Commands.maybeExecute(safeStore, safeCommand, local, true, true);
+ else if (local.saveStatus().compareTo(Applying) >= 0 && !local.is(Invalidated) && !local.is(Truncated))
+ Commands.applyWrites(safeStore, context, local).begin(agent);
+ return null;
+ });
+ }
+ };
+ }
+
+ @VisibleForTesting
+ public void load(Deps loading)
+ {
+ registerHistoricalTransactions(loading,
+ ((key, txnId) -> {
+ GlobalCommandsForKey globalCfk = commandsForKey(key);
+ if (globalCfk.isEmpty())
+ globalCfk.value(new CommandsForKey(key));
+ CommandsForKey cfk = globalCfk.createSafeReference().current();
+ Invariants.checkState(cfk != null);
+ globalCfk.value(cfk.registerHistorical(txnId).cfk());
+ }));
+ }
+
+ @Override
+ protected void registerHistoricalTransactions(Deps deps, SafeCommandStore safeStore)
+ {
+ registerHistoricalTransactions(deps, (key, txnId) -> safeStore.get(key).registerHistorical(safeStore, txnId));
+ }
+
+ private void registerHistoricalTransactions(Deps deps, BiConsumer<Key, TxnId> registerHistorical)
+ {
+ RangesForEpoch rangesForEpoch = this.rangesForEpoch;
+ Ranges allRanges = rangesForEpoch.all();
+ deps.keyDeps.keys().forEach(allRanges, key -> {
+ deps.keyDeps.forEach(key, (txnId, txnIdx) -> {
+ // TODO (desired, efficiency): this can be made more efficient by batching by epoch
+ if (rangesForEpoch.coordinates(txnId).contains(key))
+ return; // already coordinates, no need to replicate
+ // TODO (required): check this logic, esp. next line, matches C*
+ if (!rangesForEpoch.allAfter(txnId.epoch()).contains(key))
+ return;
+
+ registerHistorical.accept(key, txnId);
+ });
+
+ });
+
+ TreeMap<TxnId, RangeCommand> rangeCommands = this.rangeCommands;
+ TreeMap<TxnId, Ranges> historicalRangeCommands = historicalRangeCommands();
+ deps.rangeDeps.forEachUniqueTxnId(allRanges, null, (ignore, txnId) -> {
+
+ if (rangeCommands.containsKey(txnId))
+ return;
+
+ Ranges ranges = deps.rangeDeps.ranges(txnId);
+ if (rangesForEpoch.coordinates(txnId).intersects(ranges))
+ return; // already coordinates, no need to replicate
+ // TODO (required): check this logic, esp. next line, matches C*
+ if (!rangesForEpoch.allAfter(txnId.epoch()).intersects(ranges))
+ return;
+
+ historicalRangeCommands.merge(txnId, ranges.slice(allRanges), Ranges::with);
+ });
+ }
}
diff --git a/accord-core/src/main/java/accord/impl/TimestampsForKeys.java b/accord-core/src/main/java/accord/impl/TimestampsForKeys.java
index 7dcfa6d4..d46d3513 100644
--- a/accord-core/src/main/java/accord/impl/TimestampsForKeys.java
+++ b/accord-core/src/main/java/accord/impl/TimestampsForKeys.java
@@ -43,7 +43,7 @@ public class TimestampsForKeys
{
if (!commandStore.safeToReadAt(executeAt).contains(tfk.key().toUnseekable()))
return current;
- throw new IllegalArgumentException(String.format("%s is less than the most recent write timestamp %s", executeAt, lastWrite));
+ throw new IllegalArgumentException(String.format("%s is less than the most recent write timestamp %s for %s", executeAt, lastWrite, tfk.key()));
}
Timestamp lastExecuted = current.lastExecutedTimestamp();
diff --git a/accord-core/src/main/java/accord/impl/progresslog/CallbackInvoker.java b/accord-core/src/main/java/accord/impl/progresslog/CallbackInvoker.java
index afd6aea1..0170cc11 100644
--- a/accord-core/src/main/java/accord/impl/progresslog/CallbackInvoker.java
+++ b/accord-core/src/main/java/accord/impl/progresslog/CallbackInvoker.java
@@ -100,5 +100,11 @@ class CallbackInvoker<P, V> implements BiConsumer<V, Throwable>
{
return txnId.hashCode();
}
+
+ @Override
+ public String toString()
+ {
+ return txnId + (isHome ? ":Home:" : ":Waiting:") + owner.commandStore;
+ }
}
diff --git a/accord-core/src/main/java/accord/impl/progresslog/DefaultProgressLog.java b/accord-core/src/main/java/accord/impl/progresslog/DefaultProgressLog.java
index 752a8684..0932d92e 100644
--- a/accord-core/src/main/java/accord/impl/progresslog/DefaultProgressLog.java
+++ b/accord-core/src/main/java/accord/impl/progresslog/DefaultProgressLog.java
@@ -79,8 +79,8 @@ public class DefaultProgressLog implements ProgressLog, Runnable
private Object[] stateMap = BTree.empty();
private Object[] progressTokenMap = BTree.empty();
- private final LogGroupTimers<TxnState> timers = new LogGroupTimers<>(MICROSECONDS);
- private final LogGroupTimers<TxnState>.Scheduling timerScheduling;
+ final LogGroupTimers<TxnState> timers = new LogGroupTimers<>(MICROSECONDS);
+ final LogGroupTimers<TxnState>.Scheduling timerScheduling;
/**
* A collection of active callbacks (waiting remote replies) or submitted run invocations
@@ -278,6 +278,23 @@ public class DefaultProgressLog implements ProgressLog, Runnable
clear(state);
}
+ public void clear()
+ {
+ timers.clear();
+ timerScheduling.clear();
+
+ stateMap = BTree.empty();
+ progressTokenMap = BTree.empty();
+
+ activeWaiting.clear();
+ activeHome.clear();
+ if (deleted != null)
+ deleted.clear();
+
+ runBuffer = EMPTY_RUN_BUFFER;
+ runBufferCount = 0;
+ }
+
private void clear(TxnState state)
{
state.setHomeDone(this);
@@ -461,7 +478,7 @@ public class DefaultProgressLog implements ProgressLog, Runnable
if (!deregisterActive(runKind, this))
return; // we've been cancelled
- Invariants.checkState(get(run.txnId) == run);
+ Invariants.checkState(get(run.txnId) == run, "Transaction state for %s does not match expected one %s", run.txnId, run);
Invariants.checkState(run.scheduledTimer() != runKind, "We are actively executing %s, but we are also scheduled to run this same TxnState later. This should not happen.", runKind);
Invariants.checkState(run.pendingTimer() != runKind, "We are actively executing %s, but we also have a pending scheduled task to run this same TxnState later. This should not happen.", runKind);
diff --git a/accord-core/src/main/java/accord/impl/progresslog/WaitingState.java b/accord-core/src/main/java/accord/impl/progresslog/WaitingState.java
index 792eae23..d60f2a28 100644
--- a/accord-core/src/main/java/accord/impl/progresslog/WaitingState.java
+++ b/accord-core/src/main/java/accord/impl/progresslog/WaitingState.java
@@ -303,7 +303,8 @@ abstract class WaitingState extends BaseTxnState
BlockedUntil blockedUntil = blockedUntil();
Command command = safeCommand.current();
Invariants.checkState(!owner.hasActive(Waiting, txnId));
- Invariants.checkState(command.saveStatus().compareTo(blockedUntil.minSaveStatus) < 0, "Command has met desired criteria but progress log entry has not been cancelled");
+ Invariants.checkState(command.saveStatus().compareTo(blockedUntil.minSaveStatus) < 0,
+ "Command has met desired criteria (%s) but progress log entry has not been cancelled: %s", blockedUntil.minSaveStatus, command);
set(safeStore, owner, blockedUntil, Querying);
TxnId txnId = safeCommand.txnId();
diff --git a/accord-core/src/main/java/accord/local/Bootstrap.java b/accord-core/src/main/java/accord/local/Bootstrap.java
index 08bf14a2..d77df6f5 100644
--- a/accord-core/src/main/java/accord/local/Bootstrap.java
+++ b/accord-core/src/main/java/accord/local/Bootstrap.java
@@ -41,7 +41,6 @@ import accord.utils.Invariants;
import accord.utils.async.AsyncResult;
import accord.utils.async.AsyncResults;
-import static accord.local.PreLoadContext.contextFor;
import static accord.local.PreLoadContext.empty;
import static accord.primitives.Routables.Slice.Minimal;
import static accord.primitives.Txn.Kind.ExclusiveSyncPoint;
@@ -96,7 +95,7 @@ class Bootstrap
}
// an attempt to fetch some portion of the range we are bootstrapping
- class Attempt implements FetchRanges, BiConsumer<Void, Throwable>
+ class Attempt implements FetchRanges, BiConsumer<Object, Throwable>
{
final List<SafeToRead> states = new ArrayList<>();
Runnable cancel;
@@ -117,7 +116,7 @@ class Bootstrap
this.valid = ranges;
}
- void start(SafeCommandStore safeStore0)
+ void start(SafeCommandStore safeStore)
{
if (valid.isEmpty())
{
@@ -142,23 +141,16 @@ class Bootstrap
// we fix here the ranges we use for the synthetic command, even though we may end up only finishing a subset
// of these ranges as part of this attempt
Ranges commitRanges = valid;
- store.markBootstrapping(safeStore0, globalSyncId, valid);
- CoordinateSyncPoint.exclusive(node, globalSyncId, commitRanges)
- // ATM all known implementations store ranges in-memory, but this will not be true soon, so this will need to be addressed
- .flatMap(syncPoint -> node.withEpoch(epoch, () -> store.submit(contextFor(localSyncId, syncPoint.waitFor.keyDeps.keys(), KeyHistory.COMMANDS), safeStore1 -> {
- if (valid.isEmpty()) // we've lost ownership of the range
- return AsyncResults.success(Ranges.EMPTY);
-
- Commands.createBootstrapCompleteMarkerTransaction(safeStore1, localSyncId, valid);
- safeStore1.registerHistoricalTransactions(syncPoint.waitFor);
- return fetch = safeStore1.dataStore().fetch(node, safeStore1, valid, syncPoint, this);
- })))
- .flatMap(i -> i)
- .flatMap(ranges -> store.execute(contextFor(localSyncId), safeStore -> {
- if (!ranges.isEmpty())
- Commands.markBootstrapComplete(safeStore, localSyncId, ranges);
- }))
- .begin(this);
+ safeStore = safeStore;
+ // we submit a separate execution so that we know markBootstrapping is durable before we initiate the fetch
+ safeStore.commandStore().submit(empty(), safeStore0 -> {
+ store.markBootstrapping(safeStore0, globalSyncId, commitRanges);
+ return CoordinateSyncPoint.exclusive(node, globalSyncId, commitRanges);
+ }).flatMap(i -> i).flatMap(syncPoint -> node.withEpoch(epoch, () -> store.submit(empty(), safeStore1 -> {
+ if (valid.isEmpty()) // we've lost ownership of the range
+ return AsyncResults.success(Ranges.EMPTY);
+ return fetch = safeStore1.dataStore().fetch(node, safeStore1, valid, syncPoint, this);
+ }))).flatMap(i -> i).begin(this);
}
// we no longer want to fetch these ranges (perhaps we no longer own them)
@@ -379,7 +371,7 @@ class Bootstrap
}
@Override
- public void accept(Void success, Throwable failure)
+ public void accept(Object success, Throwable failure)
{
if (completed)
return;
diff --git a/accord-core/src/main/java/accord/local/Cleanup.java b/accord-core/src/main/java/accord/local/Cleanup.java
index baf96cb0..578912a6 100644
--- a/accord-core/src/main/java/accord/local/Cleanup.java
+++ b/accord-core/src/main/java/accord/local/Cleanup.java
@@ -68,6 +68,16 @@ public enum Cleanup
}
public static Cleanup shouldCleanup(SafeCommandStore safeStore, Command command, EpochSupplier toEpoch, Unseekables<?> maybeFullRoute)
+ {
+ return shouldCleanup(safeStore.commandStore(), command, toEpoch, maybeFullRoute);
+ }
+
+ public static Cleanup shouldCleanup(CommandStore commandStore, Command command, EpochSupplier toEpoch, Unseekables<?> maybeFullRoute)
+ {
+ return shouldCleanup(commandStore, command, toEpoch, maybeFullRoute, true);
+ }
+
+ public static Cleanup shouldCleanup(CommandStore commandStore, Command command, EpochSupplier toEpoch, Unseekables<?> maybeFullRoute, boolean enforceInvariants)
{
if (command.saveStatus() == Erased)
return Cleanup.NO; // once erased we no longer have executeAt, and may consider that a transaction is owned by us that should not be (as its txnId is an earlier epoch than we adopted a range)
@@ -85,7 +95,7 @@ public enum Cleanup
toEpoch = executeAt;
return shouldCleanup(command.txnId(), command.status(), command.durability(), toEpoch, route,
- safeStore.commandStore().redundantBefore(), safeStore.commandStore().durableBefore());
+ commandStore.redundantBefore(), commandStore.durableBefore(), enforceInvariants);
}
public static Cleanup shouldCleanup(TxnId txnId, Status status, Durability durability, EpochSupplier toEpoch, Route<?> route, RedundantBefore redundantBefore, DurableBefore durableBefore)
@@ -124,7 +134,7 @@ public enum Cleanup
// - we can impose additional validations here IF we receive an epoch upper bound
// - we should be more robust to the presence/absence of executeAt
// - be cognisant of future epochs that participated only for PreAccept/Accept, but where txn was not committed to execute in the epoch (this is why we provide null toEpoch here)
- illegalState("Command " + txnId + " that is being loaded is not owned by this shard on route " + route);
+ illegalState("Command %s that is being loaded is not owned by this shard on route %s. Redundant before: %s", txnId, route, redundantBefore);
}
}
switch (redundant)
diff --git a/accord-core/src/main/java/accord/local/Command.java b/accord-core/src/main/java/accord/local/Command.java
index 47b1c51c..ddc4b5a8 100644
--- a/accord-core/src/main/java/accord/local/Command.java
+++ b/accord-core/src/main/java/accord/local/Command.java
@@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
+import javax.annotation.Nullable;
import accord.api.Key;
import accord.api.Result;
@@ -54,8 +55,6 @@ import accord.utils.IndexedTriConsumer;
import accord.utils.Invariants;
import accord.utils.SimpleBitSet;
-import javax.annotation.Nullable;
-
import static accord.local.Command.AbstractCommand.validate;
import static accord.local.SaveStatus.AcceptedInvalidate;
import static accord.local.SaveStatus.ErasedOrInvalidOrVestigial;
@@ -341,7 +340,7 @@ public abstract class Command implements CommonAttributes
Invariants.checkState(result != null, "Result is null");
break;
case Invalidated:
- Invariants.checkState(validate.durability().isMaybeInvalidated());
+ Invariants.checkState(validate.durability().isMaybeInvalidated(), "%s is not invalidated", validate.durability());
case Unknown:
Invariants.checkState(validate.durability() != Local);
case Erased:
diff --git a/accord-core/src/main/java/accord/local/CommandStore.java b/accord-core/src/main/java/accord/local/CommandStore.java
index f993974b..6c15cbc7 100644
--- a/accord-core/src/main/java/accord/local/CommandStore.java
+++ b/accord-core/src/main/java/accord/local/CommandStore.java
@@ -18,29 +18,6 @@
package accord.local;
-import accord.api.LocalListeners;
-import accord.api.ProgressLog;
-import accord.api.DataStore;
-import accord.api.VisibleForImplementationTesting;
-import accord.coordinate.CollectCalculatedDeps;
-import accord.local.Command.WaitingOn;
-
-import javax.annotation.Nullable;
-import accord.api.Agent;
-
-import accord.local.CommandStores.RangesForEpoch;
-import accord.primitives.KeyDeps;
-import accord.primitives.Keys;
-import accord.primitives.Range;
-import accord.primitives.Routables;
-import accord.utils.async.AsyncChain;
-
-import accord.api.ConfigurationService.EpochReady;
-import accord.utils.DeterministicIdentitySet;
-import accord.utils.Invariants;
-import accord.utils.ReducingRangeMap;
-import accord.utils.async.AsyncResult;
-
import java.util.AbstractMap.SimpleImmutableEntry;
import java.util.Collections;
import java.util.HashMap;
@@ -54,17 +31,39 @@ import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.Supplier;
+import javax.annotation.Nullable;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableSortedMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import accord.api.Agent;
+import accord.api.ConfigurationService.EpochReady;
+import accord.api.DataStore;
+import accord.api.LocalListeners;
+import accord.api.ProgressLog;
+import accord.coordinate.CollectCalculatedDeps;
+import accord.local.Command.WaitingOn;
+import accord.local.CommandStores.RangesForEpoch;
+import accord.primitives.Deps;
import accord.primitives.FullRoute;
+import accord.primitives.KeyDeps;
+import accord.primitives.Keys;
import accord.primitives.Participants;
+import accord.primitives.Range;
import accord.primitives.RangeDeps;
import accord.primitives.Ranges;
+import accord.primitives.Routables;
import accord.primitives.Seekables;
import accord.primitives.Timestamp;
import accord.primitives.TxnId;
import accord.primitives.Unseekables;
+import accord.utils.DeterministicIdentitySet;
+import accord.utils.Invariants;
+import accord.utils.ReducingRangeMap;
+import accord.utils.async.AsyncChain;
+import accord.utils.async.AsyncResult;
import accord.utils.async.AsyncResults;
import org.agrona.collections.Int2ObjectHashMap;
@@ -75,6 +74,7 @@ import static accord.local.PreLoadContext.empty;
import static accord.primitives.AbstractRanges.UnionMode.MERGE_ADJACENT;
import static accord.primitives.Routables.Slice.Minimal;
import static accord.primitives.Txn.Kind.ExclusiveSyncPoint;
+import static accord.utils.Invariants.checkState;
import static accord.utils.Invariants.illegalState;
/**
@@ -82,6 +82,8 @@ import static accord.utils.Invariants.illegalState;
*/
public abstract class CommandStore implements AgentExecutor
{
+ public static final Logger logger = LoggerFactory.getLogger(CommandStore.class);
+
static class EpochUpdate
{
final RangesForEpoch newRangesForEpoch;
@@ -110,13 +112,13 @@ public abstract class CommandStore implements AgentExecutor
// TODO (desired): can better encapsulate by accepting only the newRangesForEpoch and deriving the add/remove ranges
public void add(long epoch, RangesForEpoch newRangesForEpoch, Ranges addRanges)
{
- RedundantBefore addRedundantBefore = RedundantBefore.create(addRanges, epoch, Long.MAX_VALUE, TxnId.NONE, TxnId.NONE, TxnId.minForEpoch(epoch));
+ RedundantBefore addRedundantBefore = RedundantBefore.create(addRanges, epoch, Long.MAX_VALUE, TxnId.NONE, TxnId.NONE, TxnId.NONE, TxnId.minForEpoch(epoch));
update(newRangesForEpoch, addRedundantBefore);
}
public void remove(long epoch, RangesForEpoch newRangesForEpoch, Ranges removeRanges)
{
- RedundantBefore addRedundantBefore = RedundantBefore.create(removeRanges, Long.MIN_VALUE, epoch, TxnId.NONE, TxnId.NONE, TxnId.NONE);
+ RedundantBefore addRedundantBefore = RedundantBefore.create(removeRanges, Long.MIN_VALUE, epoch, TxnId.NONE, TxnId.NONE, TxnId.NONE, TxnId.NONE);
update(newRangesForEpoch, addRedundantBefore);
}
@@ -150,9 +152,9 @@ public abstract class CommandStore implements AgentExecutor
protected final LocalListeners listeners;
protected final EpochUpdateHolder epochUpdateHolder;
- // TODO (expected): schedule regular pruning of these collections
- // bootstrapBeganAt and shardDurableAt are both canonical data sets mostly used for debugging / constructing
- private NavigableMap<TxnId, Ranges> bootstrapBeganAt = ImmutableSortedMap.of(TxnId.NONE, Ranges.EMPTY); // additive (i.e. once inserted, rolled-over until invalidated, and the floor entry contains additions)
+ // Used in markShardStale to make sure the staleness includes in progresss bootstraps
+ private transient NavigableMap<TxnId, Ranges> bootstrapBeganAt = ImmutableSortedMap.of(TxnId.NONE, Ranges.EMPTY); // additive (i.e. once inserted, rolled-over until invalidated, and the floor entry contains additions)
+
private RedundantBefore redundantBefore = RedundantBefore.EMPTY;
// TODO (expected): store this only once per node
private DurableBefore durableBefore = DurableBefore.EMPTY;
@@ -179,7 +181,13 @@ public abstract class CommandStore implements AgentExecutor
private final Set<Bootstrap> bootstraps = Collections.synchronizedSet(new DeterministicIdentitySet<>());
@Nullable private ReducingRangeMap<Timestamp> rejectBefore;
- protected CommandStore(int id, NodeTimeService time, Agent agent, DataStore store, ProgressLog.Factory progressLogFactory, LocalListeners.Factory listenersFactory, EpochUpdateHolder epochUpdateHolder)
+ protected CommandStore(int id,
+ NodeTimeService time,
+ Agent agent,
+ DataStore store,
+ ProgressLog.Factory progressLogFactory,
+ LocalListeners.Factory listenersFactory,
+ EpochUpdateHolder epochUpdateHolder)
{
this.id = id;
this.time = time;
@@ -201,20 +209,19 @@ public abstract class CommandStore implements AgentExecutor
return agent;
}
- public RangesForEpoch updateRangesForEpoch()
+ public void updateRangesForEpoch(SafeCommandStore safeStore)
{
EpochUpdate update = epochUpdateHolder.get();
if (update == null)
- return rangesForEpoch;
+ return;
update = epochUpdateHolder.getAndSet(null);
if (!update.addGlobalRanges.isEmpty())
- setDurableBefore(DurableBefore.merge(durableBefore, DurableBefore.create(update.addGlobalRanges, TxnId.NONE, TxnId.NONE)));
+ safeStore.upsertDurableBefore(DurableBefore.create(update.addGlobalRanges, TxnId.NONE, TxnId.NONE));
if (update.addRedundantBefore.size() > 0)
- setRedundantBefore(RedundantBefore.merge(redundantBefore, update.addRedundantBefore));
+ safeStore.upsertRedundantBefore(update.addRedundantBefore);
if (update.newRangesForEpoch != null)
- rangesForEpoch = update.newRangesForEpoch;
- return rangesForEpoch;
+ safeStore.setRangesForEpoch(update.newRangesForEpoch);
}
public RangesForEpoch unsafeRangesForEpoch()
@@ -222,6 +229,11 @@ public abstract class CommandStore implements AgentExecutor
return rangesForEpoch;
}
+ protected void unsafeSetRangesForEpoch(RangesForEpoch newRangesForEpoch)
+ {
+ rangesForEpoch = newRangesForEpoch;
+ }
+
public abstract boolean inStore();
public void maybeExecuteImmediately(Runnable task)
@@ -235,43 +247,45 @@ public abstract class CommandStore implements AgentExecutor
public abstract <T> AsyncChain<T> submit(PreLoadContext context, Function<? super SafeCommandStore, T> apply);
public abstract void shutdown();
- // implementations are expected to override this for persistence
- protected void setRejectBefore(ReducingRangeMap<Timestamp> newRejectBefore)
+ protected abstract void registerHistoricalTransactions(Deps deps, SafeCommandStore safeStore);
+
+ protected void upsertDurableBefore(DurableBefore addDurableBefore)
+ {
+ durableBefore = DurableBefore.merge(durableBefore, addDurableBefore);
+ }
+
+ protected void unsafeSetRejectBefore(ReducingRangeMap<Timestamp> newRejectBefore)
{
this.rejectBefore = newRejectBefore;
}
- /**
- * To be overridden by implementations, to ensure the new state is persisted
- *
- * TODO (required): consider handling asynchronicity of persistence
- * (could leave to impls to call this parent method once persisted)
- * TODO (desired): compact Ranges, merging overlaps
- */
- protected void setBootstrapBeganAt(NavigableMap<TxnId, Ranges> newBootstrapBeganAt)
+ // Should be called _only_ via safe command store
+ protected void upsertRedundantBefore(RedundantBefore addRedundantBefore)
{
- this.bootstrapBeganAt = newBootstrapBeganAt;
+ redundantBefore = RedundantBefore.merge(redundantBefore, addRedundantBefore);
}
- public DurableBefore durableBefore()
+ protected void unsafeSetDurableBefore(DurableBefore newDurableBefore)
{
- return durableBefore;
+ durableBefore = newDurableBefore;
}
- /**
- * To be overridden by implementations, to ensure the new state is persisted.
- */
- public void setDurableBefore(DurableBefore durableBefore)
+ protected void unsafeSetRedundantBefore(RedundantBefore newRedundantBefore)
{
- this.durableBefore = durableBefore;
+ redundantBefore = newRedundantBefore;
}
/**
- * To be overridden by implementations, to ensure the new state is persisted.
+ * This method may be invoked on a non-CommandStore thread
*/
- protected void setRedundantBefore(RedundantBefore newRedundantBefore)
+ protected synchronized void unsafeSetSafeToRead(NavigableMap<Timestamp, Ranges> newSafeToRead)
+ {
+ this.safeToRead = newSafeToRead;
+ }
+
+ protected void unsafeSetBootstrapBeganAt(NavigableMap<TxnId, Ranges> newBootstrapBeganAt)
{
- this.redundantBefore = newRedundantBefore;
+ this.bootstrapBeganAt = newBootstrapBeganAt;
}
/**
@@ -293,29 +307,21 @@ public abstract class CommandStore implements AgentExecutor
setMaxConflicts(maxConflicts.update(keysOrRanges, executeAt));
}
- /**
- * This method may be invoked on a non-CommandStore thread
- */
- protected synchronized void setSafeToRead(NavigableMap<Timestamp, Ranges> newSafeToRead)
- {
- this.safeToRead = newSafeToRead;
- }
-
public final void markExclusiveSyncPoint(SafeCommandStore safeStore, TxnId txnId, Ranges ranges)
{
// TODO (desired): narrow ranges to those that are owned
Invariants.checkArgument(txnId.kind() == ExclusiveSyncPoint);
ReducingRangeMap<Timestamp> newRejectBefore = rejectBefore != null ? rejectBefore : new ReducingRangeMap<>();
newRejectBefore = ReducingRangeMap.add(newRejectBefore, ranges, txnId, Timestamp::max);
- setRejectBefore(newRejectBefore);
+ unsafeSetRejectBefore(newRejectBefore);
}
public final void markExclusiveSyncPointLocallyApplied(SafeCommandStore safeStore, TxnId txnId, Ranges ranges)
{
// TODO (desired): narrow ranges to those that are owned
Invariants.checkArgument(txnId.kind() == ExclusiveSyncPoint);
- RedundantBefore newRedundantBefore = RedundantBefore.merge(redundantBefore, RedundantBefore.create(ranges, txnId, TxnId.NONE, TxnId.NONE));
- setRedundantBefore(newRedundantBefore);
+ RedundantBefore newRedundantBefore = RedundantBefore.merge(redundantBefore, RedundantBefore.create(ranges, txnId, TxnId.NONE, TxnId.NONE, TxnId.NONE));
+ unsafeSetRedundantBefore(newRedundantBefore);
updatedRedundantBefore(safeStore, txnId, ranges);
}
@@ -334,10 +340,7 @@ public abstract class CommandStore implements AgentExecutor
return time.uniqueNow(txnId).asRejected();
if (txnId.kind() == ExclusiveSyncPoint)
- {
- markExclusiveSyncPoint(safeStore, txnId, (Ranges)keys);
return txnId;
- }
// TODO (expected): reject if any transaction exists with a higher timestamp OR a higher epoch
// this permits us to agree fast path decisions across epoch changes
@@ -475,7 +478,7 @@ public abstract class CommandStore implements AgentExecutor
}
else
{
- // TODO (correcness) : PreLoadContext only works with Seekables, which doesn't allow mixing Keys and Ranges... But Deps has both Keys AND Ranges!
+ // TODO (correctness) : PreLoadContext only works with Seekables, which doesn't allow mixing Keys and Ranges... But Deps has both Keys AND Ranges!
// ATM all known implementations store ranges in-memory, but this will not be true soon, so this will need to be addressed
execute(contextFor(null, deps.txnIds(), deps.keyDeps.keys(), COMMANDS), safeStore -> {
safeStore.registerHistoricalTransactions(deps);
@@ -511,23 +514,36 @@ public abstract class CommandStore implements AgentExecutor
final void markBootstrapping(SafeCommandStore safeStore, TxnId globalSyncId, Ranges ranges)
{
- setBootstrapBeganAt(bootstrap(globalSyncId, ranges, bootstrapBeganAt));
- RedundantBefore addRedundantBefore = RedundantBefore.create(ranges, Long.MIN_VALUE, Long.MAX_VALUE, TxnId.NONE, TxnId.NONE, globalSyncId);
- setRedundantBefore(RedundantBefore.merge(redundantBefore, addRedundantBefore));
- DurableBefore addDurableBefore = DurableBefore.create(ranges, TxnId.NONE, TxnId.NONE);
- setDurableBefore(DurableBefore.merge(durableBefore, addDurableBefore));
+ safeStore.setBootstrapBeganAt(bootstrap(globalSyncId, ranges, bootstrapBeganAt));
+ RedundantBefore addRedundantBefore = RedundantBefore.create(ranges, Long.MIN_VALUE, Long.MAX_VALUE, TxnId.NONE, TxnId.NONE, TxnId.NONE, globalSyncId);
+ safeStore.upsertRedundantBefore(addRedundantBefore);
+ safeStore.upsertDurableBefore(DurableBefore.create(ranges, TxnId.NONE, TxnId.NONE));
+ // TODO: can we use `upsert` for notifications?
updatedRedundantBefore(safeStore, globalSyncId, ranges);
}
// TODO (expected): we can immediately truncate dependencies locally once an exclusiveSyncPoint applies, we don't need to wait for the whole shard
- public void markShardDurable(SafeCommandStore safeStore, TxnId globalSyncId, Ranges ranges)
- {
- ranges = ranges.slice(safeStore.ranges().allUntil(globalSyncId.epoch()), Minimal);
- RedundantBefore addRedundantBefore = RedundantBefore.create(ranges, Long.MIN_VALUE, Long.MAX_VALUE, TxnId.NONE, globalSyncId, TxnId.NONE);
- setRedundantBefore(RedundantBefore.merge(redundantBefore, addRedundantBefore));
- DurableBefore addDurableBefore = DurableBefore.create(ranges, globalSyncId, globalSyncId);
- setDurableBefore(DurableBefore.merge(durableBefore, addDurableBefore));
- updatedRedundantBefore(safeStore, globalSyncId, ranges);
+ public void markShardDurable(SafeCommandStore safeStore, TxnId globalSyncId, Ranges durableRanges)
+ {
+ final Ranges slicedRanges = durableRanges.slice(safeStore.ranges().allUntil(globalSyncId.epoch()), Minimal);
+ RedundantBefore addShardRedundant = RedundantBefore.create(slicedRanges, Long.MIN_VALUE, Long.MAX_VALUE, TxnId.NONE, globalSyncId, TxnId.NONE, TxnId.NONE);
+ safeStore.upsertRedundantBefore(addShardRedundant);
+ DurableBefore addDurableBefore = DurableBefore.create(slicedRanges, globalSyncId, globalSyncId);
+ safeStore.upsertDurableBefore(addDurableBefore);
+ updatedRedundantBefore(safeStore, globalSyncId, slicedRanges);
+ safeStore = safeStore; // make unusable in lambda
+ safeStore.dataStore().snapshot(slicedRanges, globalSyncId).begin((success, fail) -> {
+ if (fail != null)
+ {
+ logger.error("Unsuccessful dataStore snapshot; unable to update GC markers", fail);
+ return;
+ }
+
+ execute(PreLoadContext.empty(), safeStore0 -> {
+ RedundantBefore addGc = RedundantBefore.create(slicedRanges, Long.MIN_VALUE, Long.MAX_VALUE, TxnId.NONE, TxnId.NONE, globalSyncId, TxnId.NONE);
+ safeStore0.upsertRedundantBefore(addGc);
+ });
+ });
}
protected void updatedRedundantBefore(SafeCommandStore safeStore, TxnId syncId, Ranges ranges)
@@ -552,21 +568,39 @@ public abstract class CommandStore implements AgentExecutor
}
agent.onStale(staleSince, ranges);
- RedundantBefore addRedundantBefore = RedundantBefore.create(ranges, TxnId.NONE, TxnId.NONE, TxnId.NONE, staleUntilAtLeast);
- setRedundantBefore(RedundantBefore.merge(redundantBefore, addRedundantBefore));
+ RedundantBefore addRedundantBefore = RedundantBefore.create(ranges, TxnId.NONE, TxnId.NONE, TxnId.NONE, TxnId.NONE, staleUntilAtLeast);
+ safeStore.upsertRedundantBefore(addRedundantBefore);
// find which ranges need to bootstrap, subtracting those already in progress that cover the id
markUnsafeToRead(ranges);
}
// MUST be invoked before CommandStore reference leaks to anyone
+ // The integration may have already loaded persisted values for these fields before this is called
+ // so it must be a merge for each field with the initialization values. These starting values don't need to be
+ // persisted since we can synthesize them at startup every time
+ // TODO (review): This needs careful thought about not persisting and that purgeAndInsert is doing the right thing
+ // with safeToRead
Supplier<EpochReady> initialise(long epoch, Ranges ranges)
{
- DurableBefore addDurableBefore = DurableBefore.create(ranges, TxnId.NONE, TxnId.NONE);
- setDurableBefore(DurableBefore.merge(durableBefore, addDurableBefore));
- setBootstrapBeganAt(ImmutableSortedMap.of(TxnId.NONE, ranges));
- setSafeToRead(ImmutableSortedMap.of(Timestamp.NONE, ranges));
- return () -> new EpochReady(epoch, DONE, DONE, DONE, DONE);
+ return () -> {
+ AsyncResult<Void> done = execute(empty(), (safeStore) -> {
+ // Merge in a base for any ranges that needs to be covered
+ DurableBefore addDurableBefore = DurableBefore.create(ranges, TxnId.NONE, TxnId.NONE);
+ safeStore.upsertDurableBefore(addDurableBefore);
+ // TODO (review): Convoluted check to not overwrite existing bootstraps with TxnId.NONE
+ // If loading from disk didn't finish before this then we might initialize the range at TxnId.NONE?
+ // Does CommandStores.topology ensure that doesn't happen? Is it fine if it does because it will get superseded?
+ Ranges newBootstrapRanges = ranges;
+ for (Ranges existing : bootstrapBeganAt.values())
+ newBootstrapRanges = newBootstrapRanges.without(existing);
+ if (!newBootstrapRanges.isEmpty())
+ bootstrapBeganAt = bootstrap(TxnId.NONE, newBootstrapRanges, bootstrapBeganAt);
+ safeStore.setSafeToRead(purgeAndInsert(safeToRead, TxnId.NONE, ranges));
+ }).beginAsResult();
+
+ return new EpochReady(epoch, DONE, DONE, DONE, DONE);
+ };
}
public final Ranges safeToReadAt(Timestamp at)
@@ -585,10 +619,15 @@ public abstract class CommandStore implements AgentExecutor
return redundantBefore;
}
- @VisibleForImplementationTesting
- public NavigableMap<TxnId, Ranges> bootstrapBeganAt() { return bootstrapBeganAt; }
+ public DurableBefore durableBefore()
+ {
+ return durableBefore;
+ }
+
+ @VisibleForTesting
+ public final NavigableMap<TxnId, Ranges> bootstrapBeganAt() { return bootstrapBeganAt; }
- @VisibleForImplementationTesting
+ @VisibleForTesting
public NavigableMap<Timestamp, Ranges> safeToRead() { return safeToRead; }
public final boolean isRejectedIfNotPreAccepted(TxnId txnId, Unseekables<?> participants)
@@ -739,21 +778,30 @@ public abstract class CommandStore implements AgentExecutor
return redundantBefore.status(minimumDependencyId, executeAt, participantsOfWaitingTxn).compareTo(RedundantStatus.PARTIALLY_PRE_BOOTSTRAP_OR_STALE) >= 0;
}
- final synchronized void markUnsafeToRead(Ranges ranges)
+ final void markUnsafeToRead(Ranges ranges)
{
if (safeToRead.values().stream().anyMatch(r -> r.intersects(ranges)))
- setSafeToRead(purgeHistory(safeToRead, ranges));
+ {
+ execute(empty(), safeStore -> {
+ safeStore.setSafeToRead(purgeHistory(safeToRead, ranges));
+ }).beginAsResult();
+ }
}
final synchronized void markSafeToRead(Timestamp forBootstrapAt, Timestamp at, Ranges ranges)
{
- Ranges validatedSafeToRead = redundantBefore.validateSafeToRead(forBootstrapAt, ranges);
- setSafeToRead(purgeAndInsert(safeToRead, at, validatedSafeToRead));
+ execute(empty(), safeStore -> {
+ Ranges validatedSafeToRead = redundantBefore.validateSafeToRead(forBootstrapAt, ranges);
+ safeStore.setSafeToRead(purgeAndInsert(safeToRead, at, validatedSafeToRead));
+ }).beginAsResult();
}
- private static <T extends Timestamp> ImmutableSortedMap<T, Ranges> bootstrap(T at, Ranges ranges, NavigableMap<T, Ranges> bootstrappedAt)
+ public static ImmutableSortedMap<TxnId, Ranges> bootstrap(TxnId at, Ranges ranges, NavigableMap<TxnId, Ranges> bootstrappedAt)
{
- Invariants.checkArgument(bootstrappedAt.lastKey().compareTo(at) < 0);
+ Invariants.checkArgument(bootstrappedAt.lastKey().compareTo(at) < 0 || at == TxnId.NONE);
+ if (at == TxnId.NONE)
+ for (Ranges rs : bootstrappedAt.values())
+ checkState(!ranges.intersects(rs));
Invariants.checkArgument(!ranges.isEmpty());
// if we're bootstrapping these ranges, then any period we previously owned the ranges for is effectively invalidated
return purgeAndInsert(bootstrappedAt, at, ranges);
diff --git a/accord-core/src/main/java/accord/local/CommandStores.java b/accord-core/src/main/java/accord/local/CommandStores.java
index 9d6660ff..545a7843 100644
--- a/accord-core/src/main/java/accord/local/CommandStores.java
+++ b/accord-core/src/main/java/accord/local/CommandStores.java
@@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.function.BiFunction;
import java.util.function.Consumer;
import java.util.function.Function;
@@ -29,8 +30,11 @@ import java.util.function.Predicate;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
+import javax.annotation.Nonnull;
import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import accord.api.Agent;
import accord.api.ConfigurationService.EpochReady;
@@ -56,14 +60,9 @@ import accord.utils.MapReduceConsume;
import accord.utils.RandomSource;
import accord.utils.async.AsyncChain;
import accord.utils.async.AsyncChains;
-import javax.annotation.Nonnull;
-
import org.agrona.collections.Hashing;
import org.agrona.collections.Int2ObjectHashMap;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import static accord.api.ConfigurationService.EpochReady.done;
import static accord.local.PreLoadContext.empty;
import static accord.primitives.Routables.Slice.Minimal;
@@ -143,6 +142,33 @@ public abstract class CommandStores
// TODO (expected): merge with RedundantBefore, and standardise executeRanges() to treat removing stale ranges the same as adding new epoch ranges
public static class RangesForEpoch
{
+ public static class Snapshot
+ {
+ public final long[] epochs;
+ public final Ranges[] ranges;
+
+ public Snapshot(long[] epochs, Ranges[] ranges)
+ {
+ this.epochs = epochs;
+ this.ranges = ranges;
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ Snapshot snapshot = (Snapshot) o;
+ return Objects.deepEquals(epochs, snapshot.epochs) && Objects.deepEquals(ranges, snapshot.ranges);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(Arrays.hashCode(epochs), Arrays.hashCode(ranges));
+ }
+ }
+
final long[] epochs;
final Ranges[] ranges;
final CommandStore store;
@@ -161,6 +187,11 @@ public abstract class CommandStores
this.store = store;
}
+ public Snapshot snapshot()
+ {
+ return new Snapshot(epochs, ranges);
+ }
+
public RangesForEpoch withRanges(long epoch, Ranges latestRanges)
{
Invariants.checkArgument(epochs.length == 0 || epochs[epochs.length - 1] <= epoch);
@@ -690,6 +721,15 @@ public abstract class CommandStores
return shards[supplier.random.nextInt(shards.length)].store;
}
+ public CommandStore[] all()
+ {
+ ShardHolder[] shards = current.shards;
+ CommandStore[] all = new CommandStore[shards.length];
+ for (int i = 0; i < shards.length; i++)
+ all[i] = shards[i].store;
+ return all;
+ }
+
public CommandStore forId(int id)
{
Snapshot snapshot = current;
diff --git a/accord-core/src/main/java/accord/local/Commands.java b/accord-core/src/main/java/accord/local/Commands.java
index 43b8d77f..a20d81ba 100644
--- a/accord-core/src/main/java/accord/local/Commands.java
+++ b/accord-core/src/main/java/accord/local/Commands.java
@@ -75,9 +75,7 @@ import static accord.local.Status.PreApplied;
import static accord.local.Status.PreCommitted;
import static accord.local.Status.Stable;
import static accord.local.Status.Truncated;
-import static accord.primitives.Routables.Slice.Minimal;
import static accord.primitives.Route.isFullRoute;
-import static accord.primitives.Txn.Kind.ExclusiveSyncPoint;
import static accord.utils.Invariants.illegalState;
public class Commands
@@ -547,16 +545,23 @@ public class Commands
return safeStore.ranges().applyRanges(executeAt);
}
- private static AsyncChain<Void> applyChain(SafeCommandStore safeStore, PreLoadContext context, TxnId txnId)
+ public static AsyncChain<Void> applyChain(SafeCommandStore safeStore, PreLoadContext context, TxnId txnId)
{
Command.Executed command = safeStore.get(txnId).current().asExecuted();
if (command.hasBeen(Applied))
return AsyncChains.success(null);
+ return apply(safeStore, context, txnId);
+ }
+ public static AsyncChain<Void> apply(SafeCommandStore safeStore, PreLoadContext context, TxnId txnId)
+ {
+ Command.Executed command = safeStore.get(txnId).current().asExecuted();
// TODO (required): make sure we are correctly handling (esp. C* side with validation logic) executing a transaction
// that was pre-bootstrap for some range (so redundant and we may have gone ahead of), but had to be executed locally
// for another range
CommandStore unsafeStore = safeStore.commandStore();
+ // TODO (required, API): do we care about tracking the write persistence latency, when this is just a memtable write?
+ // the only reason it will be slow is because Memtable flushes are backed-up (which will be reported elsewhere)
long t0 = safeStore.time().now();
return command.writes().apply(safeStore, applyRanges(safeStore, command.executeAt()), command.partialTxn())
.flatMap(unused -> unsafeStore.submit(context, ss -> {
@@ -568,7 +573,23 @@ public class Commands
}));
}
- private static void apply(SafeCommandStore safeStore, Command.Executed command)
+ public static AsyncChain<Void> applyWrites(SafeCommandStore safeStore, PreLoadContext context, Command command)
+ {
+ CommandStore unsafeStore = safeStore.commandStore();
+ Ranges executeRanges = executeRanges(safeStore, command.executeAt());
+ Command.Executed executed = command.asExecuted();
+ boolean intersects = executed.writes().keys.intersects(executeRanges);
+ if (intersects)
+ return command.writes().apply(safeStore, applyRanges(safeStore, command.executeAt()), command.partialTxn())
+ .flatMap(unused -> unsafeStore.submit(context, ss -> {
+ postApply(ss, command.txnId());
+ return null;
+ }));
+ else
+ return AsyncChains.success(null);
+ }
+
+ public static void apply(SafeCommandStore safeStore, Command command)
{
CommandStore unsafeStore = safeStore.commandStore();
TxnId txnId = command.txnId();
@@ -588,9 +609,13 @@ public class Commands
}
}
- private static boolean maybeExecute(SafeCommandStore safeStore, SafeCommand safeCommand, boolean alwaysNotifyListeners, boolean notifyWaitingOn)
+ public static boolean maybeExecute(SafeCommandStore safeStore, SafeCommand safeCommand, boolean alwaysNotifyListeners, boolean notifyWaitingOn)
+ {
+ return maybeExecute(safeStore, safeCommand, safeCommand.current(), alwaysNotifyListeners, notifyWaitingOn);
+ }
+
+ public static boolean maybeExecute(SafeCommandStore safeStore, SafeCommand safeCommand, Command command, boolean alwaysNotifyListeners, boolean notifyWaitingOn)
{
- final Command command = safeCommand.current();
if (logger.isTraceEnabled())
logger.trace("{}: Maybe executing with status {}. Will notify listeners on noop: {}", command.txnId(), command.status(), alwaysNotifyListeners);
@@ -614,7 +639,6 @@ public class Commands
// TODO (required): slice our execute ranges based on any pre-bootstrap state
// FIXME: need to communicate to caller that we didn't execute if we take one of the above paths
-
switch (command.status())
{
case Stable:
@@ -647,12 +671,6 @@ public class Commands
// but: if we later support transitive dependency elision this could be dangerous
logger.trace("{}: applying no-op", command.txnId());
safeCommand.applied(safeStore);
- if (command.txnId().kind() == ExclusiveSyncPoint)
- {
- Ranges ranges = safeStore.ranges().allAt(command.txnId().epoch());
- ranges = command.route().slice(ranges, Minimal).participants().toRanges();
- safeStore.commandStore().markExclusiveSyncPointLocallyApplied(safeStore, command.txnId(), ranges);
- }
safeStore.notifyListeners(safeCommand, command);
return true;
}
@@ -851,6 +869,16 @@ public class Commands
|| (command.route() == null || Infer.safeToCleanup(safeStore, command, command.route(), command.executeAt()) || safeStore.isFullyPreBootstrapOrStale(command, command.route().participants()))
, "Command %s could not be truncated", command);
+ Command result = purge(command, maybeFullRoute, cleanup);
+ safeCommand.update(safeStore, result);
+ safeStore.progressLog().clear(safeCommand.txnId());
+ if (notifyListeners)
+ safeStore.notifyListeners(safeCommand, result);
+ return result;
+ }
+
+ public static Command purge(Command command, @Nullable Unseekables<?> maybeFullRoute, Cleanup cleanup)
+ {
Command result;
switch (cleanup)
{
@@ -861,14 +889,14 @@ public class Commands
break;
case TRUNCATE_WITH_OUTCOME:
- Invariants.checkArgument(!command.hasBeen(Truncated));
+ Invariants.checkArgument(!command.hasBeen(Truncated), "%s", command);
Invariants.checkState(command.hasBeen(PreApplied));
result = truncatedApplyWithOutcome(command.asExecuted());
break;
case TRUNCATE:
- Invariants.checkState(command.saveStatus().compareTo(TruncatedApply) < 0);
- Invariants.checkState(command.hasBeen(PreApplied));
+ Invariants.checkState(command.saveStatus().compareTo(TruncatedApply) < 0, "%s should not have been in %s but is %s", command, TruncatedApply, command.saveStatus());
+ Invariants.checkState(command.hasBeen(PreApplied),"%s should have been %s", command, PreApplied);
result = truncatedApply(command, Route.tryCastToFullRoute(maybeFullRoute));
break;
@@ -877,11 +905,6 @@ public class Commands
result = erased(command);
break;
}
-
- safeCommand.update(safeStore, result);
- safeStore.progressLog().clear(safeCommand.txnId());
- if (notifyListeners)
- safeStore.notifyListeners(safeCommand, command);
return result;
}
@@ -1160,7 +1183,7 @@ public class Commands
public static Command updateRouteOrParticipants(SafeCommandStore safeStore, SafeCommand safeCommand, Unseekables<?> participants)
{
Command current = safeCommand.current();
- if (current.hasBeen(Invalidated))
+ if (current.saveStatus().compareTo(Erased) >= 0)
return current;
CommonAttributes updated = current;
diff --git a/accord-core/src/main/java/accord/local/DurableBefore.java b/accord-core/src/main/java/accord/local/DurableBefore.java
index 6e485f38..aab40559 100644
--- a/accord-core/src/main/java/accord/local/DurableBefore.java
+++ b/accord-core/src/main/java/accord/local/DurableBefore.java
@@ -42,6 +42,11 @@ public class DurableBefore extends ReducingRangeMap<DurableBefore.Entry>
{
public static DurableBefore create(boolean inclusiveEnds, RoutingKey[] ends, Entry[] values)
{
+ if (values.length == 0)
+ {
+ Invariants.checkState(ends.length == 1 && ends[0] == null);
+ return DurableBefore.EMPTY;
+ }
return new DurableBefore(inclusiveEnds, ends, values);
}
}
diff --git a/accord-core/src/main/java/accord/local/PreLoadContext.java b/accord-core/src/main/java/accord/local/PreLoadContext.java
index 25c68efd..d5153a79 100644
--- a/accord-core/src/main/java/accord/local/PreLoadContext.java
+++ b/accord-core/src/main/java/accord/local/PreLoadContext.java
@@ -138,23 +138,58 @@ public interface PreLoadContext
static PreLoadContext contextFor(TxnId primary, Collection<TxnId> additional, Seekables<?, ?> keys, KeyHistory keyHistory)
{
Invariants.checkState(!additional.contains(primary));
- return new PreLoadContext()
+ return new Standard(primary, additional, keys, keyHistory);
+ }
+
+ class Standard implements PreLoadContext
+ {
+ private final TxnId primary;
+ private final Collection<TxnId> additional;
+ private final Seekables<?, ?> keys;
+ private final KeyHistory keyHistory;
+
+ public Standard(TxnId primary, Collection<TxnId> additional, Seekables<?, ?> keys, KeyHistory keyHistory)
{
- @Override
- public TxnId primaryTxnId()
- {
- return primary;
- }
+ this.primary = primary;
+ this.additional = additional;
+ this.keys = keys;
+ this.keyHistory = keyHistory;
+ }
- @Override
- public Collection<TxnId> additionalTxnIds() { return additional; }
+ public String toString()
+ {
+ return "PreLoadContext{" +
+ "primary=" + primary +
+ ", additional=" + additional +
+ ", keys=" + keys +
+ ", keyHistory=" + keyHistory +
+ '}';
+ }
- @Override
- public Seekables<?, ?> keys() { return keys; }
+ @Override
+ @Nullable
+ public TxnId primaryTxnId()
+ {
+ return primary;
+ }
- @Override
- public KeyHistory keyHistory() { return keyHistory; }
- };
+ @Override
+ public Collection<TxnId> additionalTxnIds()
+ {
+ return additional;
+ }
+
+ @Override
+ public Seekables<?, ?> keys()
+ {
+ return keys;
+ }
+
+ @Override
+ public KeyHistory keyHistory()
+ {
+ return keyHistory;
+ }
}
static PreLoadContext contextFor(TxnId primary, Collection<TxnId> additional, Seekables<?, ?> keys)
diff --git a/accord-core/src/main/java/accord/local/RedundantBefore.java b/accord-core/src/main/java/accord/local/RedundantBefore.java
index 26942bf6..a39cdf98 100644
--- a/accord-core/src/main/java/accord/local/RedundantBefore.java
+++ b/accord-core/src/main/java/accord/local/RedundantBefore.java
@@ -95,6 +95,12 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
*/
public final @Nonnull TxnId shardAppliedOrInvalidatedBefore;
+ /**
+ * Represents the maximum TxnId we know to have fully executed until across all healthy replicas for the range in question.
+ * Unless we are stale or pre-bootstrap, in which case no such guarantees can be made.
+ */
+ public final @Nonnull TxnId gcBefore;
+
/**
* bootstrappedAt defines the txnId bounds we expect to maintain data for locally.
*
@@ -116,12 +122,12 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
*/
public final @Nullable Timestamp staleUntilAtLeast;
- public Entry(Range range, long startEpoch, long endEpoch, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId bootstrappedAt, @Nullable Timestamp staleUntilAtLeast)
+ public Entry(Range range, long startEpoch, long endEpoch, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId gcBefore, @Nonnull TxnId bootstrappedAt, @Nullable Timestamp staleUntilAtLeast)
{
- this(range, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, bootstrappedAt, staleUntilAtLeast);
+ this(range, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, gcBefore, bootstrappedAt, staleUntilAtLeast);
}
- public Entry(Range range, long startEpoch, long endEpoch, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId locallyDecidedAndAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId bootstrappedAt, @Nullable Timestamp staleUntilAtLeast)
+ public Entry(Range range, long startEpoch, long endEpoch, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId locallyDecidedAndAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId gcBefore, @Nonnull TxnId bootstrappedAt, @Nullable Timestamp staleUntilAtLeast)
{
this.range = range;
this.startEpoch = startEpoch;
@@ -129,11 +135,14 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
this.locallyAppliedOrInvalidatedBefore = locallyAppliedOrInvalidatedBefore;
this.locallyDecidedAndAppliedOrInvalidatedBefore = locallyDecidedAndAppliedOrInvalidatedBefore;
this.shardAppliedOrInvalidatedBefore = shardAppliedOrInvalidatedBefore;
+ this.gcBefore = gcBefore;
this.bootstrappedAt = bootstrappedAt;
this.staleUntilAtLeast = staleUntilAtLeast;
Invariants.checkArgument(locallyAppliedOrInvalidatedBefore.equals(TxnId.NONE) || locallyAppliedOrInvalidatedBefore.domain().isRange());
Invariants.checkArgument(locallyDecidedAndAppliedOrInvalidatedBefore.equals(TxnId.NONE) || locallyDecidedAndAppliedOrInvalidatedBefore.domain().isRange());
Invariants.checkArgument(shardAppliedOrInvalidatedBefore.equals(TxnId.NONE) || shardAppliedOrInvalidatedBefore.domain().isRange());
+ Invariants.checkArgument(gcBefore.equals(TxnId.NONE) || gcBefore.domain().isRange());
+ Invariants.checkArgument(gcBefore.compareTo(shardAppliedOrInvalidatedBefore) <= 0);
}
public static Entry reduce(Entry a, Entry b)
@@ -154,6 +163,7 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
int cl = cur.locallyAppliedOrInvalidatedBefore.compareTo(add.locallyAppliedOrInvalidatedBefore);
int cd = cur.locallyDecidedAndAppliedOrInvalidatedBefore.compareTo(add.locallyDecidedAndAppliedOrInvalidatedBefore);
int cs = cur.shardAppliedOrInvalidatedBefore.compareTo(add.shardAppliedOrInvalidatedBefore);
+ int cg = cur.gcBefore.compareTo(add.gcBefore);
int cb = cur.bootstrappedAt.compareTo(add.bootstrappedAt);
int csu = compareStaleUntilAtLeast(cur.staleUntilAtLeast, add.staleUntilAtLeast);
@@ -165,6 +175,7 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
TxnId locallyAppliedOrInvalidatedBefore = cl >= 0 ? cur.locallyAppliedOrInvalidatedBefore : add.locallyAppliedOrInvalidatedBefore;
TxnId locallyDecidedAndAppliedOrInvalidatedBefore = cd >= 0 ? cur.locallyDecidedAndAppliedOrInvalidatedBefore : add.locallyDecidedAndAppliedOrInvalidatedBefore;
TxnId shardAppliedOrInvalidatedBefore = cs >= 0 ? cur.shardAppliedOrInvalidatedBefore : add.shardAppliedOrInvalidatedBefore;
+ TxnId gcBefore = cg >= 0 ? cur.gcBefore : add.gcBefore;
TxnId bootstrappedAt = cb >= 0 ? cur.bootstrappedAt : add.bootstrappedAt;
Timestamp staleUntilAtLeast = csu >= 0 ? cur.staleUntilAtLeast : add.staleUntilAtLeast;
@@ -182,7 +193,7 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
if (staleUntilAtLeast != null && bootstrappedAt.compareTo(staleUntilAtLeast) >= 0)
staleUntilAtLeast = null;
- return new Entry(range, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, locallyDecidedAndAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, bootstrappedAt, staleUntilAtLeast);
+ return new Entry(range, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, locallyDecidedAndAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, gcBefore, bootstrappedAt, staleUntilAtLeast);
}
static @Nonnull RedundantStatus getAndMerge(Entry entry, @Nonnull RedundantStatus prev, TxnId txnId, EpochSupplier executeAt)
@@ -305,7 +316,7 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
Entry withRange(Range range)
{
- return new Entry(range, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, bootstrappedAt, staleUntilAtLeast);
+ return new Entry(range, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, gcBefore, bootstrappedAt, staleUntilAtLeast);
}
public boolean equals(Object that)
@@ -376,27 +387,27 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
return Ranges.ofSortedAndDeoverlapped(staleRanges).mergeTouching();
}
- public static RedundantBefore create(Ranges ranges, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId bootstrappedAt)
+ public static RedundantBefore create(Ranges ranges, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId gcBefore, @Nonnull TxnId bootstrappedAt)
{
- return create(ranges, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, bootstrappedAt, null);
+ return create(ranges, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, gcBefore, bootstrappedAt, null);
}
- public static RedundantBefore create(Ranges ranges, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId bootstrappedAt, @Nullable Timestamp staleUntilAtLeast)
+ public static RedundantBefore create(Ranges ranges, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId gcBefore, @Nonnull TxnId bootstrappedAt, @Nullable Timestamp staleUntilAtLeast)
{
- return create(ranges, Long.MIN_VALUE, Long.MAX_VALUE, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, bootstrappedAt, staleUntilAtLeast);
+ return create(ranges, Long.MIN_VALUE, Long.MAX_VALUE, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, gcBefore, bootstrappedAt, staleUntilAtLeast);
}
- public static RedundantBefore create(Ranges ranges, long startEpoch, long endEpoch, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId bootstrappedAt)
+ public static RedundantBefore create(Ranges ranges, long startEpoch, long endEpoch, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId gcBefore, @Nonnull TxnId bootstrappedAt)
{
- return create(ranges, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, bootstrappedAt, null);
+ return create(ranges, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, gcBefore, bootstrappedAt, null);
}
- public static RedundantBefore create(Ranges ranges, long startEpoch, long endEpoch, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId bootstrappedAt, @Nullable Timestamp staleUntilAtLeast)
+ public static RedundantBefore create(Ranges ranges, long startEpoch, long endEpoch, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId gcBefore, @Nonnull TxnId bootstrappedAt, @Nullable Timestamp staleUntilAtLeast)
{
if (ranges.isEmpty())
return new RedundantBefore();
- Entry entry = new Entry(null, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, bootstrappedAt, staleUntilAtLeast);
+ Entry entry = new Entry(null, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, gcBefore, bootstrappedAt, staleUntilAtLeast);
Builder builder = new Builder(ranges.get(0).endInclusive(), ranges.size() * 2);
for (int i = 0 ; i < ranges.size() ; ++i)
{
@@ -509,7 +520,7 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
if (v.range.start().equals(start) && v.range.end().equals(end))
return v;
- return new Entry(v.range.newRange(start, end), v.startEpoch, v.endEpoch, v.locallyAppliedOrInvalidatedBefore, v.shardAppliedOrInvalidatedBefore, v.bootstrappedAt, v.staleUntilAtLeast);
+ return new Entry(v.range.newRange(start, end), v.startEpoch, v.endEpoch, v.locallyAppliedOrInvalidatedBefore, v.shardAppliedOrInvalidatedBefore, v.gcBefore, v.bootstrappedAt, v.staleUntilAtLeast);
}
@Override
@@ -528,7 +539,7 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
return new Entry(a.range.newRange(
a.range.start().compareTo(b.range.start()) <= 0 ? a.range.start() : b.range.start(),
a.range.end().compareTo(b.range.end()) >= 0 ? a.range.end() : b.range.end()
- ), a.startEpoch, a.endEpoch, a.locallyAppliedOrInvalidatedBefore, a.shardAppliedOrInvalidatedBefore, a.bootstrappedAt, a.staleUntilAtLeast);
+ ), a.startEpoch, a.endEpoch, a.locallyAppliedOrInvalidatedBefore, a.shardAppliedOrInvalidatedBefore, a.gcBefore, a.bootstrappedAt, a.staleUntilAtLeast);
}
@Override
diff --git a/accord-core/src/main/java/accord/local/SafeCommandStore.java b/accord-core/src/main/java/accord/local/SafeCommandStore.java
index e74e29af..cc51f7c8 100644
--- a/accord-core/src/main/java/accord/local/SafeCommandStore.java
+++ b/accord-core/src/main/java/accord/local/SafeCommandStore.java
@@ -18,6 +18,7 @@
package accord.local;
+import java.util.NavigableMap;
import javax.annotation.Nullable;
import accord.api.Agent;
@@ -34,10 +35,12 @@ import accord.primitives.EpochSupplier;
import accord.primitives.Keys;
import accord.primitives.Participants;
import accord.primitives.Ranges;
+import accord.primitives.Routable.Domain;
import accord.primitives.Routables;
import accord.primitives.Seekable;
import accord.primitives.Seekables;
import accord.primitives.Timestamp;
+import accord.primitives.Txn.Kind;
import accord.primitives.Txn.Kind.Kinds;
import accord.primitives.TxnId;
import accord.primitives.Unseekables;
@@ -46,8 +49,10 @@ import accord.utils.Invariants;
import static accord.local.Cleanup.NO;
import static accord.local.KeyHistory.COMMANDS;
import static accord.local.RedundantBefore.PreBootstrapOrStale.FULLY;
+import static accord.local.SaveStatus.Applied;
import static accord.local.SaveStatus.Erased;
import static accord.local.SaveStatus.ErasedOrInvalidOrVestigial;
+import static accord.primitives.Routables.Slice.Minimal;
import static accord.primitives.Route.isFullRoute;
/**
@@ -206,9 +211,32 @@ public abstract class SafeCommandStore
{
updateMaxConflicts(prev, updated);
updateCommandsForKey(prev, updated);
+ updateExclusiveSyncPoint(prev, updated);
}
- private void updateMaxConflicts(Command prev, Command updated)
+ public void updateExclusiveSyncPoint(Command prev, Command updated)
+ {
+ if (updated.txnId().kind() != Kind.ExclusiveSyncPoint || updated.txnId().domain() != Domain.Range) return;
+ if (updated.route() == null) return;
+
+ SaveStatus oldSaveStatus = prev == null ? SaveStatus.Uninitialised : prev.saveStatus();
+ SaveStatus newSaveStatus = updated.saveStatus();
+
+ TxnId txnId = updated.txnId();
+ if (newSaveStatus.known.isDefinitionKnown() && !oldSaveStatus.known.isDefinitionKnown())
+ {
+ Ranges ranges = updated.route().slice(ranges().all(), Minimal).toRanges();
+ commandStore().markExclusiveSyncPoint(this, txnId, ranges);
+ }
+
+ if (newSaveStatus == Applied && oldSaveStatus != Applied)
+ {
+ Ranges ranges = updated.route().slice(ranges().all(), Minimal).toRanges();
+ commandStore().markExclusiveSyncPointLocallyApplied(this, txnId, ranges);
+ }
+ }
+
+ public void updateMaxConflicts(Command prev, Command updated)
{
SaveStatus oldSaveStatus = prev == null ? SaveStatus.Uninitialised : prev.saveStatus();
SaveStatus newSaveStatus = updated.saveStatus();
@@ -222,7 +250,41 @@ public abstract class SafeCommandStore
commandStore().updateMaxConflicts(prev, updated);
}
- private void updateCommandsForKey(Command prev, Command next)
+ /**
+ * Methods that implementors can use to capture changes to auxiliary collections:
+ */
+
+ public void upsertRedundantBefore(RedundantBefore addRedundantBefore)
+ {
+ commandStore().upsertRedundantBefore(addRedundantBefore);
+ }
+
+ public void upsertDurableBefore(DurableBefore addDurableBefore)
+ {
+ commandStore().upsertDurableBefore(addDurableBefore);
+ }
+
+ public void setBootstrapBeganAt(NavigableMap<TxnId, Ranges> newBootstrapBeganAt)
+ {
+ commandStore().unsafeSetBootstrapBeganAt(newBootstrapBeganAt);
+ }
+
+ public void setSafeToRead(NavigableMap<Timestamp, Ranges> newSafeToRead)
+ {
+ commandStore().unsafeSetSafeToRead(newSafeToRead);
+ }
+
+ public void setRangesForEpoch(CommandStores.RangesForEpoch rangesForEpoch)
+ {
+ commandStore().unsafeSetRangesForEpoch(rangesForEpoch);
+ }
+
+ protected void registerHistoricalTransactions(Deps deps)
+ {
+ commandStore().registerHistoricalTransactions(deps, this);
+ }
+
+ public void updateCommandsForKey(Command prev, Command next)
{
if (!CommandsForKey.needsUpdate(prev, next))
return;
@@ -311,8 +373,6 @@ public abstract class SafeCommandStore
public abstract NodeTimeService time();
public abstract CommandStores.RangesForEpoch ranges();
- public abstract void registerHistoricalTransactions(Deps deps);
-
public boolean isTruncated(Command command)
{
return Cleanup.shouldCleanup(this, command, null, null) != NO;
diff --git a/accord-core/src/main/java/accord/local/cfk/CommandsForKey.java b/accord-core/src/main/java/accord/local/cfk/CommandsForKey.java
index eb5137b7..7cc2bb84 100644
--- a/accord-core/src/main/java/accord/local/cfk/CommandsForKey.java
+++ b/accord-core/src/main/java/accord/local/cfk/CommandsForKey.java
@@ -193,6 +193,7 @@ public class CommandsForKey extends CommandsForKeyUpdate implements CommandsSumm
{
private static final Logger logger = LoggerFactory.getLogger(CommandsForKey.class);
+ private static boolean reportLinearizabilityViolations = true;
private static final boolean ELIDE_TRANSITIVE_DEPENDENCIES = true;
public static final TxnInfo NO_INFO = new TxnInfo(TxnId.NONE, HISTORICAL, TxnId.NONE);
@@ -1577,7 +1578,7 @@ public class CommandsForKey extends CommandsForKeyUpdate implements CommandsSumm
for (int i = maxAppliedWriteByExecuteAt ; i >= 0 ; --i)
{
TxnInfo txn = committedByExecuteAt[i];
- if (newInfo == txn)
+ if (newInfo == txn && CommandsForKey.reportLinearizabilityViolations())
{
// we haven't found anything pre-bootstrap that follows this command, so log a linearizability violation
// TODO (expected): this should be a rate-limited logger; need to integrate with Cassandra
@@ -1693,4 +1694,19 @@ public class CommandsForKey extends CommandsForKeyUpdate implements CommandsSumm
}
}
}
+
+ public static boolean reportLinearizabilityViolations()
+ {
+ return reportLinearizabilityViolations;
+ }
+
+ public static void enableLinearizabilityViolationsReporting()
+ {
+ reportLinearizabilityViolations = true;
+ }
+
+ public static void disableLinearizabilityViolationsReporting()
+ {
+ reportLinearizabilityViolations = false;
+ }
}
diff --git a/accord-core/src/main/java/accord/local/cfk/NotifySink.java b/accord-core/src/main/java/accord/local/cfk/NotifySink.java
index 91909849..5a306890 100644
--- a/accord-core/src/main/java/accord/local/cfk/NotifySink.java
+++ b/accord-core/src/main/java/accord/local/cfk/NotifySink.java
@@ -39,8 +39,6 @@ interface NotifySink
{
void notWaiting(SafeCommandStore safeStore, TxnId txnId, Key key);
- void notWaiting(SafeCommandStore safeStore, SafeCommand safeCommand, Key key);
-
void waitingOn(SafeCommandStore safeStore, TxnInfo txn, Key key, SaveStatus waitingOnStatus, BlockedUntil blockedUntil, boolean notifyCfk);
class DefaultNotifySink implements NotifySink
@@ -60,8 +58,7 @@ interface NotifySink
}
}
- @Override
- public void notWaiting(SafeCommandStore safeStore, SafeCommand safeCommand, Key key)
+ private void notWaiting(SafeCommandStore safeStore, SafeCommand safeCommand, Key key)
{
Commands.removeWaitingOnKeyAndMaybeExecute(safeStore, safeCommand, key);
}
diff --git a/accord-core/src/main/java/accord/local/cfk/Updating.java b/accord-core/src/main/java/accord/local/cfk/Updating.java
index 2939b5c7..fdf8eb9d 100644
--- a/accord-core/src/main/java/accord/local/cfk/Updating.java
+++ b/accord-core/src/main/java/accord/local/cfk/Updating.java
@@ -53,6 +53,7 @@ import static accord.local.cfk.CommandsForKey.InternalStatus.TRANSITIVELY_KNOWN;
import static accord.local.cfk.CommandsForKey.managesExecution;
import static accord.local.cfk.CommandsForKey.Unmanaged.Pending.APPLY;
import static accord.local.cfk.CommandsForKey.Unmanaged.Pending.COMMIT;
+import static accord.local.cfk.CommandsForKey.reportLinearizabilityViolations;
import static accord.local.cfk.Pruning.loadPruned;
import static accord.local.cfk.Pruning.loadingPrunedFor;
import static accord.local.cfk.Utils.insertMissing;
@@ -615,7 +616,7 @@ class Updating
{
for (int i = pos; i <= maxAppliedWriteByExecuteAt; ++i)
{
- if (committedByExecuteAt[pos].kind().witnesses(newInfo))
+ if (committedByExecuteAt[pos].kind().witnesses(newInfo) && reportLinearizabilityViolations())
logger.error("Linearizability violation on key {}: {} is committed to execute (at {}) before {} that should witness it but has already applied (at {})", cfk.key, newInfo.plainTxnId(), newInfo.plainExecuteAt(), committedByExecuteAt[i].plainTxnId(), committedByExecuteAt[i].plainExecuteAt());
}
}
@@ -673,7 +674,7 @@ class Updating
TxnInfo[] committedByExecuteAt = cfk.committedByExecuteAt;
for (int i = cfk.maxAppliedWriteByExecuteAt + 1; i < appliedPos ; ++i)
{
- if (committedByExecuteAt[i].status != APPLIED && appliedKind.witnesses(committedByExecuteAt[i]))
+ if (committedByExecuteAt[i].status != APPLIED && appliedKind.witnesses(committedByExecuteAt[i]) && reportLinearizabilityViolations())
logger.error("Linearizability violation on key {}: {} is committed to execute (at {}) before {} that should witness it but has already applied (at {})", cfk.key, committedByExecuteAt[i].plainTxnId(), committedByExecuteAt[i].plainExecuteAt(), applied.plainTxnId(), applied.plainExecuteAt());
}
}
diff --git a/accord-core/src/main/java/accord/messages/PreAccept.java b/accord-core/src/main/java/accord/messages/PreAccept.java
index 10af97c2..7d7d514f 100644
--- a/accord-core/src/main/java/accord/messages/PreAccept.java
+++ b/accord-core/src/main/java/accord/messages/PreAccept.java
@@ -115,6 +115,7 @@ public class PreAccept extends WithUnsynced<PreAccept.PreAcceptReply> implements
// must consider cleanup though. (alternative is to make recovery more complicated)
Ranges ranges = safeStore.ranges().allBetween(minEpoch, txnId);
if (txnId.kind() == ExclusiveSyncPoint)
+ // TODO (required): this must be removed for correctly restoring rejectBefore (so journal can update it). This is expected to happen in a follow-up patch soon after.
safeStore.commandStore().markExclusiveSyncPoint(safeStore, txnId, ranges);
return new PreAcceptOk(txnId, txnId, calculatePartialDeps(safeStore, txnId, partialTxn.keys(), scope, EpochSupplier.constant(minEpoch), txnId, ranges));
}
diff --git a/accord-core/src/main/java/accord/messages/SetGloballyDurable.java b/accord-core/src/main/java/accord/messages/SetGloballyDurable.java
index 3fa39321..7d912435 100644
--- a/accord-core/src/main/java/accord/messages/SetGloballyDurable.java
+++ b/accord-core/src/main/java/accord/messages/SetGloballyDurable.java
@@ -18,8 +18,8 @@
package accord.messages;
-import accord.local.PreLoadContext;
import accord.local.DurableBefore;
+import accord.local.PreLoadContext;
import accord.local.SafeCommandStore;
import accord.primitives.TxnId;
@@ -51,7 +51,8 @@ public class SetGloballyDurable extends AbstractEpochRequest<SimpleReply>
{
DurableBefore cur = safeStore.commandStore().durableBefore();
DurableBefore upd = DurableBefore.merge(durableBefore, cur);
- safeStore.commandStore().setDurableBefore(upd);
+ // This is done asynchronously
+ safeStore.upsertDurableBefore(upd);
return Ok;
}
diff --git a/accord-core/src/main/java/accord/primitives/Writes.java b/accord-core/src/main/java/accord/primitives/Writes.java
index e470d074..cdc76efb 100644
--- a/accord-core/src/main/java/accord/primitives/Writes.java
+++ b/accord-core/src/main/java/accord/primitives/Writes.java
@@ -77,6 +77,10 @@ public class Writes
accumulate.add(write.apply(key, safeStore, executeAt, safeStore.dataStore(), txn));
return accumulate;
}, new ArrayList<>());
+
+ if (futures.isEmpty())
+ return SUCCESS;
+
return AsyncChains.reduce(futures, (l, r) -> null);
}
diff --git a/accord-core/src/main/java/accord/topology/Topologies.java b/accord-core/src/main/java/accord/topology/Topologies.java
index 95fce506..2d4a3494 100644
--- a/accord-core/src/main/java/accord/topology/Topologies.java
+++ b/accord-core/src/main/java/accord/topology/Topologies.java
@@ -18,12 +18,15 @@
package accord.topology;
+import java.util.AbstractCollection;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
+import java.util.Iterator;
import java.util.List;
import java.util.Set;
-import com.google.common.collect.Sets;
+import com.google.common.collect.Iterators;
import accord.api.TopologySorter;
import accord.local.Node;
@@ -74,7 +77,29 @@ public interface Topologies extends TopologySorter
// note this can be expensive to evaluate
SortedList<Id> nodes();
- Set<Node.Id> nonStaleNodes();
+ default Collection<Node.Id> nonStaleNodes()
+ {
+ Topology currentTopology = current();
+ Set<Id> staleIds = currentTopology.staleIds();
+ SortedList<Id> nodes = nodes();
+ if (staleIds.isEmpty())
+ return nodes;
+
+ return new AbstractCollection<>()
+ {
+ @Override
+ public Iterator<Id> iterator()
+ {
+ return Iterators.filter(nodes.iterator(), id -> !staleIds.contains(id));
+ }
+
+ @Override
+ public int size()
+ {
+ return Iterators.size(iterator());
+ }
+ };
+ }
int estimateUniqueNodes();
@@ -216,12 +241,6 @@ public interface Topologies extends TopologySorter
return topology.nodes();
}
- @Override
- public Set<Node.Id> nonStaleNodes()
- {
- return topology.nonStaleNodes();
- }
-
@Override
public int estimateUniqueNodes()
{
@@ -402,13 +421,6 @@ public interface Topologies extends TopologySorter
return result;
}
- @Override
- public Set<Node.Id> nonStaleNodes()
- {
- Topology currentTopology = current();
- return Sets.filter(nodes(), id -> !currentTopology.staleIds().contains(id));
- }
-
@Override
public Ranges computeRangesForNode(Id node)
{
diff --git a/accord-core/src/main/java/accord/topology/Topology.java b/accord-core/src/main/java/accord/topology/Topology.java
index f1fc988f..fca2618b 100644
--- a/accord-core/src/main/java/accord/topology/Topology.java
+++ b/accord-core/src/main/java/accord/topology/Topology.java
@@ -34,9 +34,6 @@ import java.util.stream.IntStream;
import javax.annotation.Nullable;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Sets;
-import org.agrona.collections.Int2ObjectHashMap;
-import org.agrona.collections.IntArrayList;
import accord.api.RoutingKey;
import accord.local.Node.Id;
@@ -53,6 +50,8 @@ import accord.utils.IndexedTriFunction;
import accord.utils.SimpleBitSet;
import accord.utils.SortedArrays.SortedArrayList;
import accord.utils.Utils;
+import org.agrona.collections.Int2ObjectHashMap;
+import org.agrona.collections.IntArrayList;
import static accord.utils.Invariants.illegalArgument;
import static accord.utils.SortedArrays.Search.FLOOR;
@@ -558,11 +557,6 @@ public class Topology
return nodeIds;
}
- public Set<Id> nonStaleNodes()
- {
- return Sets.filter(nodes(), id -> !staleIds.contains(id));
- }
-
public Ranges ranges()
{
return subsetOfRanges;
diff --git a/accord-core/src/main/java/accord/utils/LogGroupTimers.java b/accord-core/src/main/java/accord/utils/LogGroupTimers.java
index 1245c03c..1c3251b6 100644
--- a/accord-core/src/main/java/accord/utils/LogGroupTimers.java
+++ b/accord-core/src/main/java/accord/utils/LogGroupTimers.java
@@ -132,6 +132,15 @@ public class LogGroupTimers<T extends LogGroupTimers.Timer>
if (scheduled.isDone() || newDeadline < scheduledAt - schedulerPreciseLateTolerance)
ensureScheduled(now);
}
+
+ public void clear()
+ {
+ if (!scheduled.isDone())
+ scheduled.cancel();
+ scheduled = Scheduler.CANCELLED;
+ lastNow = 0;
+ scheduledAt = Long.MAX_VALUE;
+ }
}
static class Bucket<T extends Timer> extends IntrusivePriorityHeap<T> implements Comparable<Bucket<T>>
@@ -604,4 +613,16 @@ public class LogGroupTimers<T extends LogGroupTimers.Timer>
for (int i = bucketsStart + 1 ; i < bucketsEnd ; ++i)
Invariants.checkState(buckets[i - 1].end() == buckets[i].epoch);
}
+
+ public void clear()
+ {
+ while (bucketsStart < bucketsEnd)
+ {
+ buckets[bucketsStart].clear();
+ buckets[bucketsStart++] = null;
+ }
+ bucketsStart = bucketsEnd = 0;
+ curEpoch = 0;
+ addFinger = null;
+ }
}
diff --git a/accord-core/src/test/java/accord/impl/basic/PendingQueue.java b/accord-core/src/main/java/accord/utils/TriConsumer.java
similarity index 72%
copy from accord-core/src/test/java/accord/impl/basic/PendingQueue.java
copy to accord-core/src/main/java/accord/utils/TriConsumer.java
index fd4932d4..72d1ec03 100644
--- a/accord-core/src/test/java/accord/impl/basic/PendingQueue.java
+++ b/accord-core/src/main/java/accord/utils/TriConsumer.java
@@ -16,17 +16,9 @@
* limitations under the License.
*/
-package accord.impl.basic;
+package accord.utils;
-import java.util.concurrent.TimeUnit;
-
-public interface PendingQueue
+public interface TriConsumer<P1, P2, P3>
{
- void add(Pending item);
- void addNoDelay(Pending item);
- void add(Pending item, long delay, TimeUnit units);
- boolean remove(Pending item);
- Pending poll();
- int size();
- long nowInMillis();
+ void consume(P1 p1, P2 p2, P3 p3);
}
diff --git a/accord-core/src/main/java/accord/utils/async/AsyncChain.java b/accord-core/src/main/java/accord/utils/async/AsyncChain.java
index 66f97f16..a642321b 100644
--- a/accord-core/src/main/java/accord/utils/async/AsyncChain.java
+++ b/accord-core/src/main/java/accord/utils/async/AsyncChain.java
@@ -21,6 +21,7 @@ package accord.utils.async;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.function.BiConsumer;
+import java.util.function.BooleanSupplier;
import java.util.function.Consumer;
import java.util.function.Function;
@@ -48,6 +49,16 @@ public interface AsyncChain<V>
return AsyncChains.flatMap(this, mapper, executor);
}
+ default <T> AsyncChain<T> map(Function<? super V, ? extends T> mapper, BooleanSupplier inExecutor, Executor executor)
+ {
+ return flatMap(input -> {
+ if (inExecutor.getAsBoolean())
+ return AsyncChains.success(mapper.apply(input));
+ else
+ return AsyncChains.ofCallable(executor, () -> mapper.apply(input));
+ });
+ }
+
/**
* When the chain has failed, this allows the chain to attempt to recover if possible. The provided function may return a {@code null} to represent
* that recovery was not possible and that the original exception should propgate.
diff --git a/accord-core/src/test/java/accord/impl/RemoteListenersTest.java b/accord-core/src/test/java/accord/impl/RemoteListenersTest.java
index 19f18426..664f025a 100644
--- a/accord-core/src/test/java/accord/impl/RemoteListenersTest.java
+++ b/accord-core/src/test/java/accord/impl/RemoteListenersTest.java
@@ -31,7 +31,6 @@ import java.util.function.Function;
import java.util.function.IntSupplier;
import java.util.function.Supplier;
import java.util.stream.Stream;
-
import javax.annotation.Nullable;
import org.junit.jupiter.api.Assertions;
@@ -384,7 +383,14 @@ public class RemoteListenersTest
protected TestCommandStore(int id)
{
- super(id, null, null, null, ignore -> new ProgressLog.NoOpProgressLog(), ignore -> new DefaultLocalListeners(new DefaultRemoteListeners((a, b, c, d, e)->{}), DefaultLocalListeners.DefaultNotifySink.INSTANCE), new EpochUpdateHolder());
+ super(id,
+ null,
+ null,
+ null,
+ ignore -> new ProgressLog.NoOpProgressLog(),
+ ignore -> new DefaultLocalListeners(new DefaultRemoteListeners((a, b, c, d, e)->{}),
+ DefaultLocalListeners.DefaultNotifySink.INSTANCE),
+ new EpochUpdateHolder());
this.storeId = id;
}
@@ -392,6 +398,8 @@ public class RemoteListenersTest
@Override public AsyncChain<Void> execute(PreLoadContext context, Consumer<? super SafeCommandStore> consumer) { return null; }
@Override public <T> AsyncChain<T> submit(PreLoadContext context, Function<? super SafeCommandStore, T> apply) { return null; }
@Override public void shutdown() {}
+ @Override protected void registerHistoricalTransactions(Deps deps, SafeCommandStore safeStore) {}
+
@Override public <T> AsyncChain<T> submit(Callable<T> task) { return null; }
}
@@ -418,6 +426,5 @@ public class RemoteListenersTest
@Override public ProgressLog progressLog() { return null; }
@Override public NodeTimeService time() { return null; }
@Override public CommandStores.RangesForEpoch ranges() { return null; }
- @Override public void registerHistoricalTransactions(Deps deps) { }
}
}
diff --git a/accord-core/src/test/java/accord/impl/basic/Cluster.java b/accord-core/src/test/java/accord/impl/basic/Cluster.java
index b24f161a..a938052e 100644
--- a/accord-core/src/test/java/accord/impl/basic/Cluster.java
+++ b/accord-core/src/test/java/accord/impl/basic/Cluster.java
@@ -23,6 +23,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
@@ -41,7 +42,6 @@ import java.util.function.IntSupplier;
import java.util.function.LongSupplier;
import java.util.function.Predicate;
import java.util.function.Supplier;
-
import javax.annotation.Nullable;
import org.junit.jupiter.api.Assertions;
@@ -50,12 +50,12 @@ import org.slf4j.LoggerFactory;
import accord.api.BarrierType;
import accord.api.Key;
+import accord.api.LocalConfig;
import accord.api.MessageSink;
import accord.api.Scheduler;
import accord.burn.BurnTestConfigurationService;
import accord.burn.TopologyUpdates;
import accord.burn.random.FrequentLargeRange;
-import accord.api.LocalConfig;
import accord.coordinate.Barrier;
import accord.coordinate.CoordinationAdapter;
import accord.coordinate.Exhausted;
@@ -64,22 +64,23 @@ import accord.coordinate.Preempted;
import accord.coordinate.Timeout;
import accord.coordinate.Truncated;
import accord.impl.CoordinateDurabilityScheduling;
+import accord.impl.DefaultLocalListeners;
+import accord.impl.DefaultRemoteListeners;
import accord.impl.DefaultRequestTimeouts;
import accord.impl.InMemoryCommandStore.GlobalCommand;
import accord.impl.MessageListener;
import accord.impl.PrefixedIntHashKey;
-import accord.impl.DefaultLocalListeners;
-import accord.impl.progresslog.DefaultProgressLogs;
-import accord.impl.DefaultRemoteListeners;
import accord.impl.SizeOfIntersectionSorter;
import accord.impl.TopologyFactory;
import accord.impl.basic.DelayedCommandStores.DelayedCommandStore;
import accord.impl.list.ListAgent;
import accord.impl.list.ListStore;
+import accord.impl.progresslog.DefaultProgressLogs;
import accord.local.AgentExecutor;
import accord.local.Command;
-import accord.local.Node.Id;
+import accord.local.CommandStore;
import accord.local.Node;
+import accord.local.Node.Id;
import accord.local.NodeTimeService;
import accord.local.SaveStatus;
import accord.local.ShardDistributor;
@@ -219,21 +220,54 @@ public class Cluster implements Scheduler
processNext(next);
}
+ boolean hasNonRecurring()
+ {
+ boolean hasNonRecurring = false;
+ for (Pending p : pending)
+ {
+ if (!(p instanceof RecurringPendingRunnable))
+ continue;
+ RecurringPendingRunnable r = (RecurringPendingRunnable) p;
+ if (r.requeue.hasNonRecurring())
+ {
+ hasNonRecurring = true;
+ break;
+ }
+ }
+
+ return hasNonRecurring;
+ }
+
public boolean processPending()
{
checkFailures.run();
- if (pending.size() == recurring)
+ // All remaining tasks are recurring
+ if (recurring > 0 && pending.size() == recurring && !hasNonRecurring())
return false;
- Object next = pending.poll();
+ Pending next = pending.poll();
if (next == null)
return false;
processNext(next);
+
checkFailures.run();
return true;
}
+ /**
+ * Drain tasks that match predicate.
+ *
+ * Returns whether any tasks were processed
+ */
+ public boolean drain(Predicate<Pending> process)
+ {
+ List<Pending> pending = this.pending.drain(process);
+ for (Pending p : pending)
+ processNext(p);
+ return !pending.isEmpty();
+ }
+
private void processNext(Object next)
{
if (next instanceof Packet)
@@ -257,7 +291,8 @@ public class Cluster implements Scheduler
else callback.success(deliver.src, reply);
}
}
- else journalLookup.apply(deliver.dst).handle((Request) deliver.message, deliver.src, deliver);
+
+ else on.receive((Request) deliver.message, deliver.src, deliver);
}
else
{
@@ -274,21 +309,27 @@ public class Cluster implements Scheduler
@Override
public Scheduled recurring(Runnable run, long delay, TimeUnit units)
{
- RecurringPendingRunnable result = new RecurringPendingRunnable(pending, run, delay, units);
- ++recurring;
- result.onCancellation(() -> --recurring);
- pending.add(result, delay, units);
- return result;
+ return recurring(run, () -> delay, units);
}
@Override
public Scheduled once(Runnable run, long delay, TimeUnit units)
{
- RecurringPendingRunnable result = new RecurringPendingRunnable(null, run, delay, units);
+ RecurringPendingRunnable result = new RecurringPendingRunnable(null, run, () -> delay, units);
pending.add(result, delay, units);
return result;
}
+ public Scheduled recurring(Runnable run, LongSupplier delay, TimeUnit units)
+ {
+ RecurringPendingRunnable result = new RecurringPendingRunnable(pending, run, delay, units);
+ ++recurring;
+ result.onCancellation(() -> --recurring);
+ pending.add(result, delay.getAsLong(), units);
+ return result;
+
+ }
+
public void onDone(Runnable run)
{
onDone.add(run);
@@ -422,12 +463,12 @@ public class Cluster implements Scheduler
BiConsumer<Timestamp, Ranges> onStale = (sinceAtLeast, ranges) -> configRandomizer.onStale(id, sinceAtLeast, ranges);
AgentExecutor nodeExecutor = nodeExecutorSupplier.apply(id, onStale);
executorMap.put(id, nodeExecutor);
- Journal journal = new Journal();
+ Journal journal = new Journal(id);
journalMap.put(id, journal);
BurnTestConfigurationService configService = new BurnTestConfigurationService(id, nodeExecutor, randomSupplier, topology, nodeMap::get, topologyUpdates);
BooleanSupplier isLoadedCheck = Gens.supplier(Gens.bools().mixedDistribution().next(random), random);
Node node = new Node(id, messageSink, configService, nowSupplier, NodeTimeService.elapsedWrapperFromNonMonotonicSource(TimeUnit.MILLISECONDS, nowSupplier),
- () -> new ListStore(id), new ShardDistributor.EvenSplit<>(8, ignore -> new PrefixedIntHashKey.Splitter()),
+ () -> new ListStore(sinks, random, id), new ShardDistributor.EvenSplit<>(8, ignore -> new PrefixedIntHashKey.Splitter()),
nodeExecutor.agent(),
randomSupplier.get(), sinks, SizeOfIntersectionSorter.SUPPLIER, DefaultRemoteListeners::new, DefaultRequestTimeouts::new,
DefaultProgressLogs::new, DefaultLocalListeners.Factory::new, DelayedCommandStores.factory(sinks.pending, isLoadedCheck, journal), new CoordinationAdapter.DefaultFactory(),
@@ -461,8 +502,6 @@ public class Cluster implements Scheduler
updateDurabilityRate.run();
schemaApply.onUpdate(topology);
- // startup
- journalMap.entrySet().forEach(e -> e.getValue().start(nodeMap.get(e.getKey())));
AsyncResult<?> startup = AsyncChains.reduce(nodeMap.values().stream().map(Node::unsafeStart).collect(toList()), (a, b) -> null).beginAsResult();
while (sinks.processPending());
Assertions.assertTrue(startup.isDone());
@@ -475,11 +514,54 @@ public class Cluster implements Scheduler
}, 5L, SECONDS);
Scheduled reconfigure = sinks.recurring(configRandomizer::maybeUpdateTopology, 1, SECONDS);
+
+ Scheduled purge = sinks.recurring(() -> {
+ trace.debug("Triggering purge.");
+ int numNodes = random.nextInt(1, nodeMap.size());
+ for (int i = 0; i < numNodes; i++)
+ {
+ Id id = random.pick(nodes);
+ Node node = nodeMap.get(id);
+
+ Journal journal = journalMap.get(node.id());
+ CommandStore[] stores = nodeMap.get(node.id()).commandStores().all();
+ journal.purge((j) -> stores[j]);
+ }
+ }, () -> random.nextInt(1, 10), SECONDS);
+
+ Scheduled restart = sinks.recurring(() -> {
+ Id id = random.pick(nodes);
+ CommandStore[] stores = nodeMap.get(id).commandStores().all();
+ Predicate<Pending> pred = getPendingPredicate(stores);
+ while (sinks.drain(pred));
+
+ // Journal cleanup is a rough equivalent of a node restart.
+ trace.debug("Triggering journal cleanup for node " + id);
+ CommandsForKey.disableLinearizabilityViolationsReporting();
+ ListStore listStore = (ListStore) nodeMap.get(id).commandStores().dataStore();
+ listStore.clear();
+ listStore.restoreFromSnapshot();
+
+ Journal journal = journalMap.get(id);
+ for (CommandStore s : stores)
+ {
+ DelayedCommandStores.DelayedCommandStore store = (DelayedCommandStores.DelayedCommandStore) s;
+ store.clearForTesting();
+ journal.reconstructAll(store.loader(), store.id());
+ journal.loadHistoricalTransactions(store::load, store.id());
+ }
+ while (sinks.drain(pred));
+ CommandsForKey.enableLinearizabilityViolationsReporting();
+ trace.debug("Done with cleanup.");
+ }, () -> random.nextInt(1, 10), SECONDS);
+
durabilityScheduling.forEach(CoordinateDurabilityScheduling::start);
services.forEach(Service::start);
noMoreWorkSignal.accept(() -> {
reconfigure.cancel();
+ purge.cancel();
+ restart.cancel();
durabilityScheduling.forEach(CoordinateDurabilityScheduling::stop);
services.forEach(Service::close);
});
@@ -517,11 +599,24 @@ public class Cluster implements Scheduler
}
finally
{
- journalMap.values().forEach(Journal::shutdown);
nodeMap.values().forEach(Node::shutdown);
}
}
+ private static Predicate<Pending> getPendingPredicate(CommandStore[] stores)
+ {
+ Set<CommandStore> nodeStores = new HashSet<>(Arrays.asList(stores));
+ return item -> {
+ if (item instanceof DelayedCommandStore.DelayedTask)
+ {
+ DelayedCommandStore.DelayedTask<?> task = (DelayedCommandStore.DelayedTask<?>) item;
+ if (nodeStores.contains(task.parent()))
+ return true;
+ }
+ return false;
+ };
+ }
+
private interface Service extends AutoCloseable
{
void start();
diff --git a/accord-core/src/test/java/accord/impl/basic/DelayedCommandStores.java b/accord-core/src/test/java/accord/impl/basic/DelayedCommandStores.java
index 6e4de714..0de6debc 100644
--- a/accord-core/src/test/java/accord/impl/basic/DelayedCommandStores.java
+++ b/accord-core/src/test/java/accord/impl/basic/DelayedCommandStores.java
@@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Queue;
import java.util.concurrent.Callable;
import java.util.function.BiConsumer;
@@ -32,11 +33,13 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import accord.api.Agent;
import accord.api.DataStore;
import accord.api.LocalListeners;
import accord.api.ProgressLog;
-import accord.api.Result;
import accord.impl.InMemoryCommandStore;
import accord.impl.InMemoryCommandStores;
import accord.impl.InMemorySafeCommand;
@@ -44,7 +47,6 @@ import accord.impl.InMemorySafeCommandsForKey;
import accord.impl.InMemorySafeTimestampsForKey;
import accord.impl.PrefixedIntHashKey;
import accord.impl.basic.TaskExecutorService.Task;
-import accord.impl.mock.MockStore;
import accord.local.Command;
import accord.local.CommandStore;
import accord.local.CommandStores;
@@ -53,6 +55,7 @@ import accord.local.NodeTimeService;
import accord.local.PreLoadContext;
import accord.local.SafeCommandStore;
import accord.local.ShardDistributor;
+import accord.primitives.Deps;
import accord.primitives.Range;
import accord.primitives.RoutableKey;
import accord.primitives.Txn;
@@ -71,6 +74,8 @@ import static accord.utils.Invariants.ParanoiaCostFactor.HIGH;
public class DelayedCommandStores extends InMemoryCommandStores.SingleThread
{
+ private static final Logger logger = LoggerFactory.getLogger(DelayedCommandStores.class);
+
//TODO (correctness): remove once we have a Journal integration that does not change the values of a command...
// There are known issues due to Journal, so exclude known problamtic fields
private static class ValidationHack
@@ -138,13 +143,18 @@ public class DelayedCommandStores extends InMemoryCommandStores.SingleThread
public static class DelayedCommandStore extends InMemoryCommandStore
{
- private class DelayedTask<T> extends Task<T>
+ public class DelayedTask<T> extends Task<T>
{
private DelayedTask(Callable<T> fn)
{
super(fn);
}
+ public DelayedCommandStore parent()
+ {
+ return DelayedCommandStore.this;
+ }
+
@Override
public void run()
{
@@ -175,11 +185,8 @@ public class DelayedCommandStores extends InMemoryCommandStores.SingleThread
if (current.txnId().kind() == Txn.Kind.EphemeralRead)
return;
- Result result = current.result();
- if (result == null)
- result = MockStore.RESULT;
// Journal will not have result persisted. This part is here for test purposes and ensuring that we have strict object equality.
- Command reconstructed = journal.reconstruct(id, current.txnId(), result);
+ Command reconstructed = journal.reconstruct(id, current.txnId());
List<Difference<?>> diff = ReflectionUtils.recursiveEquals(current, reconstructed);
List<String> filteredDiff = diff.stream().filter(d -> !DelayedCommandStores.hasKnownIssue(d.path)).map(Object::toString).collect(Collectors.toList());
Invariants.checkState(filteredDiff.isEmpty(), "Commands did not match: expected %s, given %s, node %s, store %d, diff %s", current, reconstructed, time, id(), new LazyToString(() -> String.join("\n", filteredDiff)));
@@ -272,6 +279,13 @@ public class DelayedCommandStores extends InMemoryCommandStores.SingleThread
{
return new DelayedSafeStore(this, ranges, context, commands, timestampsForKey, commandsForKeys);
}
+
+ @Override
+ protected void registerHistoricalTransactions(Deps deps, SafeCommandStore safeStore)
+ {
+ journal.registerHistoricalTransactions(id(), deps);
+ super.registerHistoricalTransactions(deps, safeStore);
+ }
}
public static class DelayedSafeStore extends InMemoryCommandStore.InMemorySafeStore
@@ -286,16 +300,16 @@ public class DelayedCommandStores extends InMemoryCommandStores.SingleThread
@Override
public void postExecute()
{
- super.postExecute();
commands.entrySet().forEach(e -> {
InMemorySafeCommand safe = e.getValue();
if (!safe.isModified()) return;
Command before = safe.original();
Command after = safe.current();
- commandStore.journal.onExecute(commandStore.id(), before, after, context.primaryTxnId().equals(after.txnId()));
+ commandStore.journal.onExecute(commandStore.id(), before, after, Objects.equals(context.primaryTxnId(), after.txnId()));
commandStore.validateRead(safe.current());
});
+ super.postExecute();
}
}
diff --git a/accord-core/src/test/java/accord/impl/basic/Journal.java b/accord-core/src/test/java/accord/impl/basic/Journal.java
index 28e37b99..e11a8dec 100644
--- a/accord-core/src/test/java/accord/impl/basic/Journal.java
+++ b/accord-core/src/test/java/accord/impl/basic/Journal.java
@@ -18,26 +18,33 @@
package accord.impl.basic;
-import java.util.ArrayDeque;
import java.util.ArrayList;
+import java.util.Comparator;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.NavigableMap;
import java.util.Objects;
-import java.util.Queue;
-import java.util.concurrent.TimeUnit;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.function.Consumer;
import java.util.function.Function;
+import java.util.function.IntFunction;
+import java.util.stream.Collectors;
import accord.api.Result;
+import accord.impl.InMemoryCommandStore;
+import accord.local.Cleanup;
import accord.local.Command;
+import accord.local.CommandStore;
+import accord.local.Commands;
import accord.local.CommonAttributes;
import accord.local.Node;
import accord.local.SaveStatus;
import accord.local.Status;
-import accord.messages.Message;
-import accord.messages.ReplyContext;
-import accord.messages.Request;
import accord.primitives.Ballot;
+import accord.primitives.Deps;
import accord.primitives.PartialDeps;
import accord.primitives.PartialTxn;
import accord.primitives.Route;
@@ -47,121 +54,132 @@ import accord.primitives.TxnId;
import accord.primitives.Writes;
import accord.utils.Invariants;
import org.agrona.collections.Long2ObjectHashMap;
-import org.agrona.collections.LongArrayList;
+import static accord.local.Status.Invalidated;
+import static accord.local.Status.Truncated;
import static accord.utils.Invariants.illegalState;
-
-public class Journal implements Runnable
+public class Journal
{
- private final Queue<RequestContext> unframedRequests = new ArrayDeque<>();
- private final LongArrayList waitForEpochs = new LongArrayList();
- private final Long2ObjectHashMap<ArrayList<RequestContext>> delayedRequests = new Long2ObjectHashMap<>();
- private final Map<Key, List<Diff>> diffs = new HashMap<>();
- private Node node;
- boolean isScheduled;
-
- public Journal()
- {
- }
+ private final Long2ObjectHashMap<NavigableMap<TxnId, List<Diff>>> diffsPerCommandStore = new Long2ObjectHashMap<>();
+ private final Map<Integer, List<Deps>> historicalTransactions = new HashMap<>();
- public void start(Node node)
- {
- this.node = node;
- }
+ private final Node.Id id;
- private void ensureScheduled()
+ public Journal(Node.Id id)
{
- if (isScheduled) return;
- node.scheduler().once(this, 1, TimeUnit.MILLISECONDS);
- isScheduled = true;
+ this.id = id;
}
- public void shutdown()
+ public void purge(IntFunction<CommandStore> storeSupplier)
{
- this.node = null;
- }
-
- public void handle(Request request, Node.Id from, ReplyContext replyContext)
- {
- ensureScheduled();
- if (request.type() != null && request.type().hasSideEffects())
+ for (Map.Entry<Long, NavigableMap<TxnId, List<Diff>>> e : diffsPerCommandStore.entrySet())
{
- // enqueue
- unframedRequests.add(new RequestContext(request, request.waitForEpoch(), () -> node.receive(request, from, replyContext)));
- return;
+ int commandStoreId = e.getKey().intValue();
+ Map<TxnId, List<Diff>> localJournal = e.getValue();
+ CommandStore store = storeSupplier.apply(commandStoreId);
+
+ Map<TxnId, List<Diff>> updates = new HashMap<>();
+ List<TxnId> removals = new ArrayList<>();
+ for (Map.Entry<TxnId, List<Diff>> e2 : localJournal.entrySet())
+ {
+ TxnId txnId = e2.getKey();
+ List<Diff> diffs = e2.getValue();
+ Command command = reconstruct(diffs, Reconstruct.Last).get(0);
+ if (command.status() == Truncated || command.status() == Invalidated)
+ continue; // Already truncated
+ Cleanup cleanup = Cleanup.shouldCleanup(store, command, null, command.route(), false);
+ switch (cleanup)
+ {
+ case NO:
+ break;
+ case INVALIDATE:
+ case TRUNCATE_WITH_OUTCOME:
+ case TRUNCATE:
+ command = Commands.purge(command, command.route(), cleanup);
+ Invariants.checkState(command.saveStatus() != SaveStatus.Uninitialised);
+ List<Diff> arr = new ArrayList<>();
+ arr.add(diff(null, command));
+ updates.put(txnId, arr);
+ break;
+ case ERASE:
+ removals.add(txnId);
+ break;
+ }
+ }
+
+ for (TxnId removal : removals)
+ localJournal.remove(removal);
+ localJournal.putAll(updates);
}
- node.receive(request, from, replyContext);
}
- @Override
- public void run()
+ // TODO (required): this might be a good first approximation, but maybe we need to make a better distinction between
+ // when we want to produce side-effects.
+ private boolean loading = false;
+
+ public void reconstructAll(InMemoryCommandStore.Loader loader, int commandStoreId)
{
- isScheduled = false;
- if (this.node == null)
+ Map<TxnId, List<Diff>> diffs = diffsPerCommandStore.get(commandStoreId);
+
+ // Nothing to do here, journal is empty for this command store
+ if (diffs == null)
return;
+
+ loading = true;
try
{
- doRun();
+ List<Command> toApply = new ArrayList<>();
+ for (TxnId txnId : diffs.keySet())
+ {
+ Command command = reconstruct(commandStoreId, txnId);
+ if (command.saveStatus().compareTo(SaveStatus.Applying) >= 0 && !command.is(Invalidated) && !command.is(Truncated))
+ toApply.add(command);
+ loader.load(command);
+ }
+
+ toApply.sort(Comparator.comparing(Command::executeAt));
+ for (Command command : toApply)
+ loader.apply(command);
}
- catch (Throwable t)
+ finally
{
- node.agent().onUncaughtException(t);
+ loading = false;
}
}
- private void doRun()
+ private enum Reconstruct
{
- ArrayList<RequestContext> requests = null;
- // check to see if any pending epochs are in
- waitForEpochs.sort(null);
- for (int i = 0; i < waitForEpochs.size(); i++)
- {
- long waitForEpoch = waitForEpochs.getLong(i);
- if (!node.topology().hasEpoch(waitForEpoch))
- break;
- List<RequestContext> delayed = delayedRequests.remove(waitForEpoch);
- if (null == requests) requests = new ArrayList<>(delayed.size());
- requests.addAll(delayed);
- }
- waitForEpochs.removeIfLong(epoch -> !delayedRequests.containsKey(epoch));
+ Each,
+ Last
+ }
- // for anything queued, put into the pending epochs or schedule
- RequestContext request;
- while (null != (request = unframedRequests.poll()))
- {
- long waitForEpoch = request.waitForEpoch;
- if (waitForEpoch != 0 && !node.topology().hasEpoch(waitForEpoch))
- {
- delayedRequests.computeIfAbsent(waitForEpoch, ignore -> new ArrayList<>()).add(request);
- if (!waitForEpochs.containsLong(waitForEpoch))
- waitForEpochs.addLong(waitForEpoch);
- }
- else
- {
- if (null == requests) requests = new ArrayList<>();
- requests.add(request);
- }
- }
+ public void loadHistoricalTransactions(Consumer<Deps> consumer, int commandStoreId)
+ {
+ List<Deps> depsList = historicalTransactions.get(commandStoreId);
+ if (depsList == null)
+ return;
+ for (Deps deps : depsList)
+ consumer.accept(deps);
+ }
- // schedule
- if (requests != null)
- {
- requests.forEach(Runnable::run);
- }
+ public Command reconstruct(int commandStoreId, TxnId txnId)
+ {
+ List<Diff> diffs = this.diffsPerCommandStore.get(commandStoreId).get(txnId);
+ return reconstruct(diffs, Reconstruct.Last).get(0);
}
- public Command reconstruct(int commandStoreId, TxnId txnId, Result result)
+ private List<Command> reconstruct(List<Diff> diffs, Reconstruct reconstruct)
{
- Key key = new Key(txnId, commandStoreId);
- List<Diff> diffs = this.diffs.get(key);
- if (diffs == null || diffs.isEmpty())
- return null;
+ Invariants.checkState(diffs != null && !diffs.isEmpty());
+
+ List<Command> results = new ArrayList<>();
+ TxnId txnId = null;
Timestamp executeAt = null;
Timestamp executesAtLeast = null;
SaveStatus saveStatus = null;
- Status.Durability durability = null;
+ Status.Durability durability = Status.Durability.NotDurable;
Ballot acceptedOrCommitted = Ballot.ZERO;
Ballot promised = Ballot.ZERO;
@@ -173,94 +191,138 @@ public class Journal implements Runnable
Command.WaitingOn waitingOn = null;
Writes writes = null;
+ Result result = null;
- for (Diff diff : diffs)
+ for (int i = 0; i < diffs.size(); i++)
{
+ Diff diff = diffs.get(i);
if (diff.txnId != null)
- txnId = diff.txnId;
+ txnId = diff.txnId.get();
if (diff.executeAt != null)
- executeAt = diff.executeAt;
+ executeAt = diff.executeAt.get();
if (diff.executesAtLeast != null)
- executesAtLeast = diff.executesAtLeast;
+ executesAtLeast = diff.executesAtLeast.get();
if (diff.saveStatus != null)
- saveStatus = diff.saveStatus;
+ {
+ Set<SaveStatus> allowed = new HashSet<>();
+ allowed.add(SaveStatus.TruncatedApply);
+ allowed.add(SaveStatus.TruncatedApplyWithOutcome);
+
+ saveStatus = diff.saveStatus.get();
+ }
if (diff.durability != null)
- durability = diff.durability;
+ durability = diff.durability.get();
if (diff.acceptedOrCommitted != null)
- acceptedOrCommitted = diff.acceptedOrCommitted;
+ acceptedOrCommitted = diff.acceptedOrCommitted.get();
if (diff.promised != null)
- promised = diff.promised;
+ promised = diff.promised.get();
if (diff.route != null)
- route = diff.route;
+ route = diff.route.get();
if (diff.partialTxn != null)
- partialTxn = diff.partialTxn;
+ partialTxn = diff.partialTxn.get();
if (diff.partialDeps != null)
- partialDeps = diff.partialDeps;
+ partialDeps = diff.partialDeps.get();
if (diff.additionalKeysOrRanges != null)
- additionalKeysOrRanges = diff.additionalKeysOrRanges;
+ additionalKeysOrRanges = diff.additionalKeysOrRanges.get();
if (diff.waitingOn != null)
- waitingOn = diff.waitingOn;
+ waitingOn = diff.waitingOn.get();
if (diff.writes != null)
- writes = diff.writes;
- }
+ writes = diff.writes.get();
- if (!txnId.kind().awaitsOnlyDeps())
- executesAtLeast = null;
+ if (diff.result != null)
+ result = diff.result.get();
- switch (saveStatus.known.outcome)
- {
- case Erased:
- case WasApply:
- writes = null;
- result = null;
- break;
- }
+ try
+ {
+ if (!txnId.kind().awaitsOnlyDeps())
+ executesAtLeast = null;
+ }
+ catch (Throwable t)
+ {
+ t.printStackTrace();
+ }
+ switch (saveStatus.known.outcome)
+ {
+ case Erased:
+ case WasApply:
+ writes = null;
+ result = null;
+ break;
+ }
- CommonAttributes.Mutable attrs = new CommonAttributes.Mutable(txnId);
- if (partialTxn != null)
- attrs.partialTxn(partialTxn);
- if (durability != null)
- attrs.durability(durability);
- if (route != null)
- attrs.route(route);
-
- // TODO (desired): we can simplify this logic if, instead of diffing, we will infer the diff from the status
- if (partialDeps != null &&
- (saveStatus.known.deps != Status.KnownDeps.NoDeps &&
- saveStatus.known.deps != Status.KnownDeps.DepsErased &&
- saveStatus.known.deps != Status.KnownDeps.DepsUnknown))
- attrs.partialDeps(partialDeps);
- if (additionalKeysOrRanges != null)
- attrs.additionalKeysOrRanges(additionalKeysOrRanges);
- Invariants.checkState(saveStatus != null,
- "Save status is null after applying %s", diffs);
-
- switch (saveStatus.status)
- {
- case NotDefined:
- return saveStatus == SaveStatus.Uninitialised ? Command.NotDefined.uninitialised(attrs.txnId())
- : Command.NotDefined.notDefined(attrs, promised);
- case PreAccepted:
- return Command.PreAccepted.preAccepted(attrs, executeAt, promised);
- case AcceptedInvalidate:
- case Accepted:
- case PreCommitted:
- return Command.Accepted.accepted(attrs, saveStatus, executeAt, promised, acceptedOrCommitted);
- case Committed:
- case Stable:
- return Command.Committed.committed(attrs, saveStatus, executeAt, promised, acceptedOrCommitted, waitingOn);
- case PreApplied:
- case Applied:
- return Command.Executed.executed(attrs, saveStatus, executeAt, promised, acceptedOrCommitted, waitingOn, writes, result);
- case Invalidated:
- case Truncated:
- return truncated(attrs, saveStatus, executeAt, executesAtLeast, writes, result);
- default:
- throw new IllegalStateException("Do not know " + saveStatus.status + " " + saveStatus);
+ CommonAttributes.Mutable attrs = new CommonAttributes.Mutable(txnId);
+ if (partialTxn != null)
+ attrs.partialTxn(partialTxn);
+ if (durability != null)
+ attrs.durability(durability);
+ if (route != null)
+ attrs.route(route);
+
+ // TODO (desired): we can simplify this logic if, instead of diffing, we will infer the diff from the status
+ if (partialDeps != null &&
+ (saveStatus.known.deps != Status.KnownDeps.NoDeps &&
+ saveStatus.known.deps != Status.KnownDeps.DepsErased &&
+ saveStatus.known.deps != Status.KnownDeps.DepsUnknown))
+ attrs.partialDeps(partialDeps);
+ if (additionalKeysOrRanges != null)
+ attrs.additionalKeysOrRanges(additionalKeysOrRanges);
+ Invariants.checkState(saveStatus != null,
+ "Save status is null after applying %s", diffs);
+
+ try
+ {
+ if (reconstruct == Reconstruct.Each ||
+ (reconstruct == Reconstruct.Last && i == diffs.size() - 1))
+ {
+ Command current;
+ switch (saveStatus.status)
+ {
+ case NotDefined:
+ current = saveStatus == SaveStatus.Uninitialised ? Command.NotDefined.uninitialised(attrs.txnId())
+ : Command.NotDefined.notDefined(attrs, promised);
+ break;
+ case PreAccepted:
+ current = Command.PreAccepted.preAccepted(attrs, executeAt, promised);
+ break;
+ case AcceptedInvalidate:
+ case Accepted:
+ case PreCommitted:
+ if (saveStatus == SaveStatus.AcceptedInvalidate)
+ current = Command.AcceptedInvalidateWithoutDefinition.acceptedInvalidate(attrs, promised, acceptedOrCommitted);
+ else
+ current = Command.Accepted.accepted(attrs, saveStatus, executeAt, promised, acceptedOrCommitted);
+ break;
+ case Committed:
+ case Stable:
+ current = Command.Committed.committed(attrs, saveStatus, executeAt, promised, acceptedOrCommitted, waitingOn);
+ break;
+ case PreApplied:
+ case Applied:
+ current = Command.Executed.executed(attrs, saveStatus, executeAt, promised, acceptedOrCommitted, waitingOn, writes, result);
+ break;
+ case Invalidated:
+ case Truncated:
+ current = truncated(attrs, saveStatus, executeAt, executesAtLeast, writes, result);
+ break;
+ default:
+ throw new IllegalStateException("Do not know " + saveStatus.status + " " + saveStatus);
+ }
+
+ results.add(current);
+ }
+ }
+ catch (Throwable t)
+ {
+
+ throw new RuntimeException("Can not reconstruct from diff:\n" + diffs.stream().map(o -> o.toString())
+ .collect(Collectors.joining("\n")),
+ t);
+ }
}
+ return results;
}
private static Command.Truncated truncated(CommonAttributes.Mutable attrs, SaveStatus status, Timestamp executeAt, Timestamp executesAtLeast, Writes writes, Result result)
@@ -282,77 +344,74 @@ public class Journal implements Runnable
}
}
- public void onExecute(int commandStoreId, Command before, Command after, boolean isPrimary)
+ public void registerHistoricalTransactions(int commandStoreId, Deps deps)
{
- if (before == null && after == null)
- return;
- Diff diff = diff(before, after);
- if (!isPrimary)
- diff = diff.asNonprimary();
-
- if (diff != null)
- {
- Key key = new Key(after.txnId(), commandStoreId);
- diffs.computeIfAbsent(key, (k_) -> new ArrayList<>()).add(diff);
- }
+ this.historicalTransactions.computeIfAbsent(commandStoreId, (k) -> new ArrayList<>()).add(deps);
}
- private static class RequestContext implements Runnable
+ public void onExecute(int commandStoreId, Command before, Command after, boolean isPrimary)
{
- final long waitForEpoch;
- final Message message;
- final Runnable fn;
+ if (loading || (before == null && after == null))
+ return;
- protected RequestContext(Message request, long waitForEpoch, Runnable fn)
+ if (after.saveStatus() == SaveStatus.Erased)
{
- this.waitForEpoch = waitForEpoch;
- this.message = request;
- this.fn = fn;
+ diffsPerCommandStore.computeIfAbsent(commandStoreId, (k) -> new TreeMap<>())
+ .remove(after.txnId());
+ return;
}
+ Diff diff = diff(before, after);
+ if (!isPrimary)
+ diff = diff.asNonPrimary();
- @Override
- public void run()
+ if (diff != null)
{
- fn.run();
+ diffsPerCommandStore.computeIfAbsent(commandStoreId, (k) -> new TreeMap<>())
+ .computeIfAbsent(after.txnId(), (k_) -> new ArrayList<>())
+ .add(diff);
}
}
private static class Diff
{
- public final TxnId txnId;
+ public final NewValue<TxnId> txnId;
- public final Timestamp executeAt;
- public final Timestamp executesAtLeast;
- public final SaveStatus saveStatus;
- public final Status.Durability durability;
+ public final NewValue<Timestamp> executeAt;
+ public final NewValue<Timestamp> executesAtLeast;
+ public final NewValue<SaveStatus> saveStatus;
+ public final NewValue<Status.Durability> durability;
- public final Ballot acceptedOrCommitted;
- public final Ballot promised;
+ public final NewValue<Ballot> acceptedOrCommitted;
+ public final NewValue<Ballot> promised;
- public final Route<?> route;
- public final PartialTxn partialTxn;
- public final PartialDeps partialDeps;
+ public final NewValue<Route<?>> route;
+ public final NewValue<PartialTxn> partialTxn;
+ public final NewValue<PartialDeps> partialDeps;
- public final Writes writes;
- public final Command.WaitingOn waitingOn;
- public final Seekables<?, ?> additionalKeysOrRanges;
+ public final NewValue<Writes> writes;
+ public final NewValue<Command.WaitingOn> waitingOn;
+ public final NewValue<Seekables<?, ?>> additionalKeysOrRanges;
- public Diff(TxnId txnId,
- Timestamp executeAt,
- Timestamp executesAtLeast,
- SaveStatus saveStatus,
- Status.Durability durability,
+ public final NewValue<Result> result; // temporarily here for sakes for reloads
- Ballot acceptedOrCommitted,
- Ballot promised,
+ public Diff(NewValue<TxnId> txnId,
+ NewValue<Timestamp> executeAt,
+ NewValue<Timestamp> executesAtLeast,
+ NewValue<SaveStatus> saveStatus,
+ NewValue<Status.Durability> durability,
- Route<?> route,
- PartialTxn partialTxn,
- PartialDeps partialDeps,
- Command.WaitingOn waitingOn,
+ NewValue<Ballot> acceptedOrCommitted,
+ NewValue<Ballot> promised,
- Writes writes,
- Seekables<?, ?> additionalKeysOrRanges)
+ NewValue<Route<?>> route,
+ NewValue<PartialTxn> partialTxn,
+ NewValue<PartialDeps> partialDeps,
+ NewValue<Command.WaitingOn> waitingOn,
+
+ NewValue<Writes> writes,
+ NewValue<Seekables<?, ?>> additionalKeysOrRanges,
+
+ NewValue<Result> result)
{
this.txnId = txnId;
this.executeAt = executeAt;
@@ -370,12 +429,13 @@ public class Journal implements Runnable
this.writes = writes;
this.waitingOn = waitingOn;
this.additionalKeysOrRanges = additionalKeysOrRanges;
+ this.result = result;
}
- // We allow only save status, waitingOn, and listeners to be updated by non-primary transactions
- public Diff asNonprimary()
+ // We allow only save status, and waitingOn to be updated by non-primary transactions
+ public Diff asNonPrimary()
{
- return new Diff(null, null, null, saveStatus, null, null, null, null, null, null, waitingOn, null, null);
+ return new Diff(txnId, null, null, saveStatus, null, null, null, null, null, null, waitingOn, null, additionalKeysOrRanges, null);
}
public boolean allNulls()
@@ -393,13 +453,14 @@ public class Journal implements Runnable
if (writes != null) return false;
if (waitingOn != null) return false;
if (additionalKeysOrRanges != null) return false;
+ if (result != null) return false;
return true;
}
@Override
public String toString()
{
- StringBuilder builder = new StringBuilder("SavedDiff{");
+ StringBuilder builder = new StringBuilder("Diff{");
if (txnId != null)
builder.append("txnId = ").append(txnId).append(" ");
if (executeAt != null)
@@ -426,6 +487,8 @@ public class Journal implements Runnable
builder.append("waitingOn = ").append(waitingOn).append(" ");
if (additionalKeysOrRanges != null)
builder.append("additionalKeysOrRanges = ").append(additionalKeysOrRanges).append(" ");
+ if (result != null)
+ builder.append("result = ").append(result).append(" ");
builder.append("}");
return builder.toString();
}
@@ -450,7 +513,9 @@ public class Journal implements Runnable
ifNotEqual(before, after, Command::partialDeps, false),
ifNotEqual(before, after, Journal::getWaitingOn, true),
ifNotEqual(before, after, Command::writes, false),
- ifNotEqual(before, after, Command::additionalKeysOrRanges, false));
+ ifNotEqual(before, after, Command::additionalKeysOrRanges, false),
+ ifNotEqual(before, after, Command::result, false));
+
if (diff.allNulls())
return null;
@@ -465,7 +530,7 @@ public class Journal implements Runnable
return null;
}
- private static <OBJ, VAL> VAL ifNotEqual(OBJ lo, OBJ ro, Function<OBJ, VAL> convert, boolean allowClassMismatch)
+ private static <OBJ, VAL> NewValue<VAL> ifNotEqual(OBJ lo, OBJ ro, Function<OBJ, VAL> convert, boolean allowClassMismatch)
{
VAL l = null;
VAL r = null;
@@ -473,39 +538,42 @@ public class Journal implements Runnable
if (ro != null) r = convert.apply(ro);
if (l == r)
- return null;
+ return null; // null here means there was no change
+
if (l == null || r == null)
- return r;
+ return NewValue.of(r);
+
assert allowClassMismatch || l.getClass() == r.getClass() : String.format("%s != %s", l.getClass(), r.getClass());
if (l.equals(r))
return null;
- return r;
+ return NewValue.of(r);
}
- public static class Key
+ private static class NewValue<T>
{
- final TxnId timestamp;
- final int commandStoreId;
+ final T value;
- public Key(TxnId timestamp, int commandStoreId)
+ private NewValue(T value)
{
- this.timestamp = timestamp;
- this.commandStoreId = commandStoreId;
+ this.value = value;
}
- public boolean equals(Object o)
+ public T get()
{
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
- Key key = (Key) o;
- return commandStoreId == key.commandStoreId && Objects.equals(timestamp, key.timestamp);
+ return value;
}
- public int hashCode()
+ public static <T> NewValue<T> of(T value)
{
- return Objects.hash(timestamp, commandStoreId);
+ return new NewValue<>(value);
+ }
+
+ public String toString()
+ {
+ return "" + value;
}
}
+
}
\ No newline at end of file
diff --git a/accord-core/src/test/java/accord/impl/basic/NodeSink.java b/accord-core/src/test/java/accord/impl/basic/NodeSink.java
index e3a95775..bfa4b3c9 100644
--- a/accord-core/src/test/java/accord/impl/basic/NodeSink.java
+++ b/accord-core/src/test/java/accord/impl/basic/NodeSink.java
@@ -22,6 +22,9 @@ import java.util.LinkedHashMap;
import java.util.Map;
import java.util.function.Function;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import accord.api.MessageSink;
import accord.impl.basic.Cluster.Link;
import accord.local.AgentExecutor;
@@ -41,6 +44,8 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS;
public class NodeSink implements MessageSink
{
+ private static final Logger logger = LoggerFactory.getLogger(NodeSink.class);
+
private static final boolean DEBUG = false;
public enum Action { DELIVER, DROP, DELIVER_WITH_FAILURE, FAILURE }
@@ -150,8 +155,8 @@ public class NodeSink implements MessageSink
}
@Override
- public void replyWithUnknownFailure(Id replyingToNode, ReplyContext replyContext, Throwable failure)
+ public void replyWithUnknownFailure(Id replyToNode, ReplyContext replyContext, Throwable failure)
{
- reply(replyingToNode, replyContext, new FailureReply(failure));
+ reply(replyToNode, replyContext, new FailureReply(failure));
}
}
diff --git a/accord-core/src/test/java/accord/impl/basic/PendingQueue.java b/accord-core/src/test/java/accord/impl/basic/PendingQueue.java
index fd4932d4..15c75143 100644
--- a/accord-core/src/test/java/accord/impl/basic/PendingQueue.java
+++ b/accord-core/src/test/java/accord/impl/basic/PendingQueue.java
@@ -18,15 +18,20 @@
package accord.impl.basic;
+import java.util.List;
import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
-public interface PendingQueue
+public interface PendingQueue extends Iterable<Pending>
{
void add(Pending item);
void addNoDelay(Pending item);
void add(Pending item, long delay, TimeUnit units);
boolean remove(Pending item);
Pending poll();
+ List<Pending> drain(Predicate<Pending> toDrain);
int size();
long nowInMillis();
+
+ boolean hasNonRecurring();
}
diff --git a/accord-core/src/test/java/accord/impl/basic/PropagatingPendingQueue.java b/accord-core/src/test/java/accord/impl/basic/PropagatingPendingQueue.java
index bd25ac8f..430adbb1 100644
--- a/accord-core/src/test/java/accord/impl/basic/PropagatingPendingQueue.java
+++ b/accord-core/src/test/java/accord/impl/basic/PropagatingPendingQueue.java
@@ -18,8 +18,10 @@
package accord.impl.basic;
+import java.util.Iterator;
import java.util.List;
import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
public class PropagatingPendingQueue implements PendingQueue
{
@@ -63,6 +65,12 @@ public class PropagatingPendingQueue implements PendingQueue
return wrapped.poll();
}
+ @Override
+ public List<Pending> drain(Predicate<Pending> toDrain)
+ {
+ return wrapped.drain(toDrain);
+ }
+
public void checkFailures()
{
if (!failures.isEmpty())
@@ -99,4 +107,16 @@ public class PropagatingPendingQueue implements PendingQueue
{
return wrapped.nowInMillis();
}
+
+ @Override
+ public boolean hasNonRecurring()
+ {
+ return wrapped.hasNonRecurring();
+ }
+
+ @Override
+ public Iterator<Pending> iterator()
+ {
+ return wrapped.iterator();
+ }
}
diff --git a/accord-core/src/test/java/accord/impl/basic/RandomDelayQueue.java b/accord-core/src/test/java/accord/impl/basic/RandomDelayQueue.java
index 1468719b..e568e5c4 100644
--- a/accord-core/src/test/java/accord/impl/basic/RandomDelayQueue.java
+++ b/accord-core/src/test/java/accord/impl/basic/RandomDelayQueue.java
@@ -18,14 +18,18 @@
package accord.impl.basic;
-import accord.burn.random.FrequentLargeRange;
-import accord.utils.RandomSource;
-
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
import java.util.PriorityQueue;
import java.util.concurrent.TimeUnit;
import java.util.function.LongSupplier;
+import java.util.function.Predicate;
import java.util.function.Supplier;
+import accord.burn.random.FrequentLargeRange;
+import accord.utils.RandomSource;
+
import static accord.utils.Invariants.illegalArgument;
public class RandomDelayQueue implements PendingQueue
@@ -94,6 +98,7 @@ public class RandomDelayQueue implements PendingQueue
private final LongSupplier jitterMillis;
long now;
int seq;
+ int recurring;
public RandomDelayQueue(RandomSource random)
{
@@ -115,6 +120,8 @@ public class RandomDelayQueue implements PendingQueue
public void addNoDelay(Pending item)
{
queue.add(new Item(now, seq++, item));
+ if (item instanceof RecurringPendingRunnable)
+ ++recurring;
}
@Override
@@ -123,11 +130,15 @@ public class RandomDelayQueue implements PendingQueue
if (delay < 0)
throw illegalArgument("Delay must be positive or 0, but given " + delay);
queue.add(new Item(now + units.toMillis(delay) + jitterMillis.getAsLong(), seq++, item));
+ if (item instanceof RecurringPendingRunnable)
+ ++recurring;
}
@Override
public boolean remove(Pending item)
{
+ if (item instanceof RecurringPendingRunnable)
+ --recurring;
return queue.remove(item);
}
@@ -137,10 +148,35 @@ public class RandomDelayQueue implements PendingQueue
Item item = queue.poll();
if (item == null)
return null;
+
+ if (item.item instanceof RecurringPendingRunnable)
+ --recurring;
+
now = item.time;
return item.item;
}
+ @Override
+ public List<Pending> drain(Predicate<Pending> toDrain)
+ {
+ List<Item> items = new ArrayList<>(queue);
+ queue.clear();
+ recurring = 0;
+ List<Pending> ret = new ArrayList<>();
+ for (Item item : items)
+ {
+ if (toDrain.test(item.item))
+ ret.add(item.item);
+ else
+ {
+ queue.add(item);
+ if (item.item instanceof RecurringPendingRunnable)
+ ++recurring;
+ }
+ }
+ return ret;
+ }
+
@Override
public int size()
{
@@ -152,4 +188,16 @@ public class RandomDelayQueue implements PendingQueue
{
return now;
}
+
+ @Override
+ public boolean hasNonRecurring()
+ {
+ return recurring != queue.size();
+ }
+
+ @Override
+ public Iterator<Pending> iterator()
+ {
+ return queue.stream().map(i -> i.item).iterator();
+ }
}
diff --git a/accord-core/src/test/java/accord/impl/basic/RecurringPendingRunnable.java b/accord-core/src/test/java/accord/impl/basic/RecurringPendingRunnable.java
index 8dd2e3d2..d8b876a0 100644
--- a/accord-core/src/test/java/accord/impl/basic/RecurringPendingRunnable.java
+++ b/accord-core/src/test/java/accord/impl/basic/RecurringPendingRunnable.java
@@ -19,18 +19,19 @@
package accord.impl.basic;
import java.util.concurrent.TimeUnit;
+import java.util.function.LongSupplier;
import accord.api.Scheduler.Scheduled;
class RecurringPendingRunnable implements PendingRunnable, Scheduled
{
final PendingQueue requeue;
- final long delay;
+ final LongSupplier delay;
final TimeUnit units;
Runnable run;
Runnable onCancellation;
- RecurringPendingRunnable(PendingQueue requeue, Runnable run, long delay, TimeUnit units)
+ RecurringPendingRunnable(PendingQueue requeue, Runnable run, LongSupplier delay, TimeUnit units)
{
this.requeue = requeue;
this.run = run;
@@ -44,11 +45,16 @@ class RecurringPendingRunnable implements PendingRunnable, Scheduled
if (run != null)
{
run.run();
- if (requeue != null) requeue.add(this, delay, units);
- else run = null;
+ maybeRequeue();
}
}
+ public void maybeRequeue()
+ {
+ if (requeue != null) requeue.add(this, delay.getAsLong(), units);
+ else run = null;
+ }
+
@Override
public void cancel()
{
diff --git a/accord-core/src/test/java/accord/impl/list/ListFetchCoordinator.java b/accord-core/src/test/java/accord/impl/list/ListFetchCoordinator.java
index a4687514..c7ffc2c6 100644
--- a/accord-core/src/test/java/accord/impl/list/ListFetchCoordinator.java
+++ b/accord-core/src/test/java/accord/impl/list/ListFetchCoordinator.java
@@ -65,7 +65,7 @@ public class ListFetchCoordinator extends AbstractFetchCoordinator
ListData listData = (ListData) data;
persisting.add(commandStore.execute(PreLoadContext.empty(), safeStore -> {
listData.forEach((key, value) -> listStore.data.merge(key, value, Timestamped::merge));
- }).addCallback((ignore, fail) -> {
+ }).flatMap(ignore -> listStore.snapshot(received, syncPoint.syncId)).addCallback((success, fail) -> {
if (fail == null) success(from, received);
else fail(from, received, fail);
}).beginAsResult());
diff --git a/accord-core/src/test/java/accord/impl/list/ListStore.java b/accord-core/src/test/java/accord/impl/list/ListStore.java
index cbcdd89c..71aecfde 100644
--- a/accord-core/src/test/java/accord/impl/list/ListStore.java
+++ b/accord-core/src/test/java/accord/impl/list/ListStore.java
@@ -18,8 +18,10 @@
package accord.impl.list;
+import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Deque;
import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
@@ -30,6 +32,7 @@ import java.util.stream.Collectors;
import accord.api.DataStore;
import accord.api.Key;
+import accord.api.Scheduler;
import accord.coordinate.CoordinateSyncPoint;
import accord.coordinate.ExecuteSyncPoint.SyncPointErased;
import accord.coordinate.Invalidated;
@@ -54,9 +57,12 @@ import accord.primitives.Timestamp;
import accord.primitives.TxnId;
import accord.topology.Topologies;
import accord.topology.Topology;
+import accord.utils.Invariants;
+import accord.utils.RandomSource;
import accord.utils.Timestamped;
import accord.utils.async.AsyncChain;
import accord.utils.async.AsyncChains;
+import accord.utils.async.AsyncResult;
import accord.utils.async.AsyncResults;
import org.agrona.collections.Int2ObjectHashMap;
import org.agrona.collections.LongArrayList;
@@ -136,6 +142,9 @@ public class ListStore implements DataStore
static final Timestamped<int[]> EMPTY = new Timestamped<>(Timestamp.NONE, new int[0], Arrays::toString);
final NavigableMap<RoutableKey, Timestamped<int[]>> data = new TreeMap<>();
+ final Scheduler scheduler;
+ final RandomSource random;
+
private final List<ChangeAt> addedAts = new ArrayList<>();
private final List<ChangeAt> removedAts = new ArrayList<>();
private final List<PurgeAt> purgedAts = new ArrayList<>();
@@ -148,11 +157,104 @@ public class ListStore implements DataStore
// when out of order epochs are detected, this holds the callbacks to try again
private final List<Runnable> onRemovalDone = new ArrayList<>();
+ private static final class Snapshot
+ {
+ private final NavigableMap<RoutableKey, Timestamped<int[]>> data;
+ private final List<ChangeAt> addedAts;
+ private final List<ChangeAt> removedAts;
+ private final List<PurgeAt> purgedAts;
+ private final List<FetchComplete> fetchCompletes;
+ private final LongArrayList pendingRemoves;
+
+ private Snapshot(NavigableMap<RoutableKey, Timestamped<int[]>> data, List<ChangeAt> addedAts, List<ChangeAt> removedAts, List<PurgeAt> purgedAts, List<FetchComplete> fetchCompletes, LongArrayList pendingRemoves)
+ {
+ this.data = new TreeMap<>(data);
+ this.addedAts = new ArrayList<>(addedAts);
+ this.removedAts = new ArrayList<>(removedAts);
+ this.purgedAts = new ArrayList<>(purgedAts);
+ this.fetchCompletes = new ArrayList<>(fetchCompletes);
+ this.pendingRemoves = new LongArrayList();
+ this.pendingRemoves.addAll(pendingRemoves);
+ }
+ }
+
+ private static final class PendingSnapshot
+ {
+ final long delay;
+ final Runnable runnable;
+
+ private PendingSnapshot(long delay, Runnable runnable)
+ {
+ this.delay = delay;
+ this.runnable = runnable;
+ }
+ }
+
+ private Snapshot snapshot;
+ private final Deque<PendingSnapshot> pendingSnapshots = new ArrayDeque<>();
+ private long pendingDelay = 0;
+
+ public AsyncResult<Void> snapshot(Ranges ranges, TxnId before)
+ {
+ Snapshot snapshot = new Snapshot(data, addedAts, removedAts, purgedAts, fetchCompletes, pendingRemoves);
+ AsyncResult.Settable<Void> result = new AsyncResults.SettableResult<>();
+ long delay = Math.max(1, random.nextBiasedLong(100, 1000, 5000) - pendingDelay);
+ pendingDelay += delay;
+ pendingSnapshots.add(new PendingSnapshot(delay, () -> {
+ this.snapshot = snapshot;
+ result.setSuccess(null);
+ }));
+
+ if (pendingSnapshots.size() == 1)
+ scheduleRunSnapshot();
+ return result;
+ }
+
+ private void scheduleRunSnapshot()
+ {
+ Invariants.checkState(!pendingSnapshots.isEmpty());
+ scheduler.once(() -> {
+ if (pendingSnapshots.isEmpty())
+ return;
+
+ PendingSnapshot pendingSnapshot = pendingSnapshots.pollFirst();
+ pendingSnapshot.runnable.run();
+ pendingDelay -= pendingSnapshot.delay;
+ if (!pendingSnapshots.isEmpty())
+ scheduleRunSnapshot();
+ }, pendingSnapshots.peekFirst().delay, TimeUnit.MILLISECONDS);
+ }
+
+ public void restoreFromSnapshot()
+ {
+ if (snapshot == null)
+ return;
+
+ data.putAll(snapshot.data);
+ addedAts.addAll(snapshot.addedAts);
+ removedAts.addAll(snapshot.removedAts);
+ purgedAts.addAll(snapshot.purgedAts);
+ fetchCompletes.addAll(snapshot.fetchCompletes);
+ pendingRemoves.addAll(snapshot.pendingRemoves);
+ }
+
+ public void clear()
+ {
+ data.clear();
+ addedAts.clear();
+ removedAts.clear();
+ purgedAts.clear();
+ fetchCompletes.clear();
+ pendingRemoves.clear();
+ }
+
// adding here to help trace burn test queries
public final Node.Id node;
- public ListStore(Node.Id node)
+ public ListStore(Scheduler scheduler, RandomSource random, Node.Id node)
{
+ this.scheduler = scheduler;
+ this.random = random;
this.node = node;
}
diff --git a/accord-core/src/test/java/accord/impl/mock/MockStore.java b/accord-core/src/test/java/accord/impl/mock/MockStore.java
index e3613fb2..803c7efc 100644
--- a/accord-core/src/test/java/accord/impl/mock/MockStore.java
+++ b/accord-core/src/test/java/accord/impl/mock/MockStore.java
@@ -33,9 +33,11 @@ import accord.primitives.Seekable;
import accord.primitives.Seekables;
import accord.primitives.SyncPoint;
import accord.primitives.Timestamp;
+import accord.primitives.TxnId;
import accord.primitives.Writes;
import accord.utils.async.AsyncChain;
import accord.utils.async.AsyncChains;
+import accord.utils.async.AsyncResult;
import accord.utils.async.AsyncResults;
public class MockStore implements DataStore
@@ -149,4 +151,10 @@ public class MockStore implements DataStore
callback.fetched(ranges);
return new ImmediateFetchFuture(ranges);
}
+
+ @Override
+ public AsyncResult<Void> snapshot(Ranges ranges, TxnId before)
+ {
+ return AsyncResults.success(null);
+ }
}
diff --git a/accord-core/src/test/java/accord/local/BootstrapLocalTxnTest.java b/accord-core/src/test/java/accord/local/BootstrapLocalTxnTest.java
deleted file mode 100644
index 4d598d9e..00000000
--- a/accord-core/src/test/java/accord/local/BootstrapLocalTxnTest.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package accord.local;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.function.Consumer;
-
-import org.junit.jupiter.api.Test;
-
-import accord.impl.PrefixedIntHashKey;
-import accord.impl.basic.Cluster;
-import accord.impl.basic.DelayedCommandStores.DelayedCommandStore;
-import accord.messages.MessageType;
-import accord.messages.ReplyContext;
-import accord.messages.Request;
-import accord.primitives.Deps;
-import accord.primitives.FullRoute;
-import accord.primitives.Ranges;
-import accord.primitives.Routable;
-import accord.primitives.SyncPoint;
-import accord.primitives.Txn;
-import accord.primitives.TxnId;
-import accord.topology.Topology;
-import accord.topology.TopologyUtils;
-import accord.utils.AccordGens;
-import accord.utils.Gen;
-import accord.utils.Gens;
-import accord.utils.Invariants;
-import org.assertj.core.api.Assertions;
-
-import static accord.local.PreLoadContext.contextFor;
-import static accord.utils.Property.qt;
-
-class BootstrapLocalTxnTest
-{
- private static final Gen<Gen<Cleanup>> CLEANUP_DISTRIBUTION = Gens.mixedDistribution(Cleanup.NO, Cleanup.TRUNCATE, Cleanup.TRUNCATE_WITH_OUTCOME, Cleanup.ERASE);
-
- @Test
- public void localOnlyTxnLifeCycle()
- {
- Ranges ranges = Ranges.ofSortedAndDeoverlapped(PrefixedIntHashKey.ranges(0, 1));
- List<Node.Id> nodes = Collections.singletonList(new Node.Id(42));
- Topology t = TopologyUtils.topology(1, nodes, ranges, 2);
- qt().check(rs -> Cluster.run(rs::fork, nodes, t, nodeMap -> new Request()
- {
- @Override
- public void process(Node on, Node.Id from, ReplyContext replyContext)
- {
- Gen<Cleanup> cleanupGen = CLEANUP_DISTRIBUTION.next(rs);
- for (int storeId : on.commandStores().ids())
- {
- DelayedCommandStore store = (DelayedCommandStore) on.commandStores().forId(storeId);
- // this is a bit redudent but here to make the test easier to maintain. Pre/Post execute we validate each command to make sure everything is fine
- // but that logic could be changed and this test has a dependency on validation the command... so to make this dependency explicit
- // the test will call the validation logic within the test even though it will be called again in the background...
- Consumer<Command> validate = store::validateRead;
- TxnId globalSyncId = on.nextTxnId(Txn.Kind.ExclusiveSyncPoint, Routable.Domain.Range);
- TxnId localSyncId = globalSyncId.as(Txn.Kind.LocalOnly);
- TxnId nextGlobalSyncId = on.nextTxnId(Txn.Kind.ExclusiveSyncPoint, Routable.Domain.Range).withEpoch(globalSyncId.epoch() + 1);
- Ranges ranges = AccordGens.rangesInsideRanges(store.updateRangesForEpoch().currentRanges(), (rs2, r) -> rs2.nextInt(1, 4)).next(rs);
-
- FullRoute<?> route = on.computeRoute(globalSyncId, ranges);
- SyncPoint<Ranges> syncPoint = new SyncPoint<>(globalSyncId, Deps.NONE, ranges, route);
- Ranges valid = AccordGens.rangesInsideRanges(ranges, (rs2, r) -> rs2.nextInt(1, 4)).next(rs);
- Invariants.checkArgument(syncPoint.keysOrRanges.containsAll(valid));
- store.execute(contextFor(localSyncId, syncPoint.waitFor.keyDeps.keys(), KeyHistory.COMMANDS), safe -> Commands.createBootstrapCompleteMarkerTransaction(safe, localSyncId, valid))
- .flatMap(ignore -> store.execute(contextFor(localSyncId), safe -> validate.accept(safe.get(localSyncId, route.homeKey()).current())))
- .flatMap(ignore -> store.execute(contextFor(localSyncId), safe -> Commands.markBootstrapComplete(safe, localSyncId, ranges)))
- .flatMap(ignore -> store.execute(contextFor(localSyncId), safe -> validate.accept(safe.get(localSyncId, route.homeKey()).current())))
- // cleanup txn
- .flatMap(ignore -> store.submit(PreLoadContext.empty(), safe -> {
- Cleanup target = cleanupGen.next(rs);
- if (target == Cleanup.NO)
- return Cleanup.NO;
- safe.commandStore().setRedundantBefore(RedundantBefore.create(ranges, Long.MIN_VALUE, Long.MAX_VALUE, nextGlobalSyncId, nextGlobalSyncId, TxnId.NONE));
- switch (target)
- {
- case ERASE:
- safe.commandStore().setDurableBefore(DurableBefore.create(ranges, nextGlobalSyncId, nextGlobalSyncId));
- break;
- case TRUNCATE:
- safe.commandStore().setDurableBefore(DurableBefore.create(ranges, nextGlobalSyncId, globalSyncId));
- break;
- case TRUNCATE_WITH_OUTCOME:
- case INVALIDATE:
- // no update to DurableBefore = TRUNCATE_WITH_OUTCOME
- break;
- default:
- throw new UnsupportedOperationException(target.name());
- }
- return target;
- }))
- // validateRead is called implicitly _on command completion_
- .flatMap(target -> store.execute(contextFor(localSyncId), safe -> {
- SafeCommand cmd = safe.get(localSyncId, route.homeKey());
- Command current = cmd.current();
- Assertions.assertThat(current.saveStatus()).isEqualTo(target == Cleanup.NO ? SaveStatus.Applied : target.appliesIfNot);
- }))
- .begin(on.agent());
- }
- }
-
- @Override
- public MessageType type()
- {
- return null;
- }
- }));
- }
-}
\ No newline at end of file
diff --git a/accord-core/src/test/java/accord/local/cfk/CommandsForKeyTest.java b/accord-core/src/test/java/accord/local/cfk/CommandsForKeyTest.java
index 271037ad..6b7c0b09 100644
--- a/accord-core/src/test/java/accord/local/cfk/CommandsForKeyTest.java
+++ b/accord-core/src/test/java/accord/local/cfk/CommandsForKeyTest.java
@@ -47,10 +47,10 @@ import accord.api.Read;
import accord.api.Result;
import accord.api.RoutingKey;
import accord.api.Update;
-import accord.impl.IntKey;
import accord.impl.DefaultLocalListeners;
import accord.impl.DefaultLocalListeners.DefaultNotifySink;
import accord.impl.DefaultRemoteListeners;
+import accord.impl.IntKey;
import accord.local.Command;
import accord.local.Command.AbstractCommand;
import accord.local.CommandStore;
@@ -226,12 +226,6 @@ public class CommandsForKeyTest
set(prev, next);
}
- @Override
- public void notWaiting(SafeCommandStore safeStore, SafeCommand safeCommand, Key key)
- {
- notWaiting(safeStore, safeCommand.txnId(), key);
- }
-
@Override
public void waitingOn(SafeCommandStore safeStore, TxnInfo txn, Key key, SaveStatus waitingOnStatus, BlockedUntil blockedUntil, boolean notifyCfk)
{
@@ -837,12 +831,6 @@ public class CommandsForKeyTest
{
throw new UnsupportedOperationException();
}
-
- @Override
- public void registerHistoricalTransactions(Deps deps)
- {
- throw new UnsupportedOperationException();
- }
}
private static final class TestRead implements Read
@@ -915,7 +903,13 @@ public class CommandsForKeyTest
protected TestCommandStore(int pruneInterval, int pruneHlcDelta)
{
- super(0, null, null, null, ignore -> new ProgressLog.NoOpProgressLog(), ignore -> new DefaultLocalListeners(new DefaultRemoteListeners((a, b, c, d, e)->{}), DefaultNotifySink.INSTANCE), new EpochUpdateHolder());
+ super(0,
+ null,
+ null,
+ null,
+ ignore -> new ProgressLog.NoOpProgressLog(),
+ ignore -> new DefaultLocalListeners(new DefaultRemoteListeners((a, b, c, d, e)->{}), DefaultNotifySink.INSTANCE),
+ new EpochUpdateHolder());
this.pruneInterval = pruneInterval;
this.pruneHlcDelta = pruneHlcDelta;
}
@@ -961,6 +955,12 @@ public class CommandsForKeyTest
Invariants.checkState(queue.isEmpty());
}
+ @Override
+ protected void registerHistoricalTransactions(Deps deps, SafeCommandStore safeStore)
+ {
+ throw new UnsupportedOperationException();
+ }
+
@Override
public <T> AsyncChain<T> submit(Callable<T> task)
{
diff --git a/accord-core/src/test/java/accord/verify/StrictSerializabilityVerifier.java b/accord-core/src/test/java/accord/verify/StrictSerializabilityVerifier.java
index 4faa4cc2..c0cb6b95 100644
--- a/accord-core/src/test/java/accord/verify/StrictSerializabilityVerifier.java
+++ b/accord-core/src/test/java/accord/verify/StrictSerializabilityVerifier.java
@@ -460,6 +460,8 @@ public class StrictSerializabilityVerifier implements Verifier
.append(", preds:").append(Arrays.toString(maxPredecessors))
.append(unknownStepPeers != null ? ", peers?:" + unknownStepPeers : "")
.append(unknownStepPredecessors != null ? ", preds?:" + unknownStepPredecessors.values() : "")
+ .append(" writtenBefore " + writtenBefore)
+ .append(" writtenAfter " + writtenAfter)
.append("}");
return builder.toString();
}
@@ -709,7 +711,7 @@ public class StrictSerializabilityVerifier implements Verifier
throw new HistoryViolation(prefix, key, "Cycle detected on key " + prefix + ":" + key + ", step " + step.ofStepIndex + " " + Arrays.toString(Arrays.copyOf(sequence, step.ofStepIndex)));
if (step.writtenBefore < step.writtenAfter)
- throw new HistoryViolation(prefix, key, prefix + ":" + key + ": timestamp inconsistency, step " + step.ofStepIndex);
+ throw new HistoryViolation(prefix, key, String.format("%s:%d: timestamp inconsistency, step %d. %s", prefix, key, step.ofStepIndex, step));
if (step.maxPredecessorWrittenAfter > step.writtenBefore)
throw new HistoryViolation(prefix, key, prefix + ":" + key + " must have been written prior to its maximum predecessor in real-time order on step " + step.ofStepIndex);
diff --git a/accord-maelstrom/src/main/java/accord/maelstrom/MaelstromStore.java b/accord-maelstrom/src/main/java/accord/maelstrom/MaelstromStore.java
index 013db36d..777700cc 100644
--- a/accord-maelstrom/src/main/java/accord/maelstrom/MaelstromStore.java
+++ b/accord-maelstrom/src/main/java/accord/maelstrom/MaelstromStore.java
@@ -27,7 +27,10 @@ import accord.local.Node;
import accord.local.SafeCommandStore;
import accord.primitives.Ranges;
import accord.primitives.SyncPoint;
+import accord.primitives.TxnId;
import accord.utils.Timestamped;
+import accord.utils.async.AsyncResult;
+import accord.utils.async.AsyncResults;
import accord.utils.async.AsyncResults.SettableResult;
public class MaelstromStore implements DataStore
@@ -57,4 +60,10 @@ public class MaelstromStore implements DataStore
{
return new ImmediateFetchResult(ranges);
}
+
+ @Override
+ public AsyncResult<Void> snapshot(Ranges ranges, TxnId before)
+ {
+ return AsyncResults.success(null);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org