You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by if...@apache.org on 2025/01/24 16:20:28 UTC
(cassandra) branch cep-15-accord updated (c4c3c27deba -> 93702760153)
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a change to branch cep-15-accord
in repository https://gitbox.apache.org/repos/asf/cassandra.git
from c4c3c27deba Fix deps serializers
new da2fa003044 Fix Simulator Tests
new 93702760153 Fix race condition in AccordExecutor#cancelExclusive
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../cassandra/service/accord/AccordExecutor.java | 134 ++++++++++++++++-----
.../accord/AccordExecutorAbstractLockLoop.java | 3 +-
.../accord/AccordExecutorInfiniteLoops.java | 2 +-
.../cassandra/service/accord/AccordService.java | 2 -
.../cassandra/service/accord/AccordTask.java | 3 +
.../org/apache/cassandra/utils/ExecutorUtils.java | 2 +-
.../paxos/PairOfSequencesAccordSimulation.java | 13 +-
.../cassandra/simulator/paxos/PaxosSimulation.java | 6 +-
.../simulator/systems/InterceptibleThread.java | 3 +-
.../systems/NotInterceptedSyncCondition.java | 5 +-
10 files changed, 135 insertions(+), 38 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
(cassandra) 02/02: Fix race condition in AccordExecutor#cancelExclusive
Posted by if...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch cep-15-accord
in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit 93702760153bfc96c02b5eb2482c06d5cdc38db8
Author: Alex Petrov <ol...@gmail.com>
AuthorDate: Tue Jan 21 17:05:37 2025 +0100
Fix race condition in AccordExecutor#cancelExclusive
Patch by Alex Petrov; reviewed by Benedict Elliott Smith for CASSANDRA-20242
---
.../cassandra/service/accord/AccordExecutor.java | 128 +++++++++++++++++----
.../cassandra/service/accord/AccordTask.java | 3 +
2 files changed, 106 insertions(+), 25 deletions(-)
diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutor.java b/src/java/org/apache/cassandra/service/accord/AccordExecutor.java
index 35fbb35e4cd..0ab6dcb2e38 100644
--- a/src/java/org/apache/cassandra/service/accord/AccordExecutor.java
+++ b/src/java/org/apache/cassandra/service/accord/AccordExecutor.java
@@ -50,10 +50,10 @@ import org.apache.cassandra.metrics.AccordCacheMetrics;
import org.apache.cassandra.utils.concurrent.AsyncPromise;
import org.apache.cassandra.utils.concurrent.Future;
-import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.EVICTED;
import static org.apache.cassandra.service.accord.AccordCache.CommandAdapter.COMMAND_ADAPTER;
import static org.apache.cassandra.service.accord.AccordCache.CommandsForKeyAdapter.CFK_ADAPTER;
import static org.apache.cassandra.service.accord.AccordCache.registerJfrListener;
+import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.EVICTED;
import static org.apache.cassandra.service.accord.AccordTask.State.LOADING;
import static org.apache.cassandra.service.accord.AccordTask.State.SCANNING_RANGES;
import static org.apache.cassandra.service.accord.AccordTask.State.WAITING_TO_LOAD;
@@ -357,7 +357,7 @@ public abstract class AccordExecutor implements CacheSize, AccordCacheEntry.OnLo
case WAITING_TO_RUN:
task.runQueuedAt = nanoTime();
commandStoreQueues.computeIfAbsent(task.commandStore, CommandStoreQueue::new)
- .append(task);
+ .appendOrSetNext(task);
break;
}
}
@@ -371,7 +371,7 @@ public abstract class AccordExecutor implements CacheSize, AccordCacheEntry.OnLo
else
{
commandStoreQueues.computeIfAbsent(task.commandStore, CommandStoreQueue::new)
- .append(task);
+ .appendOrSetNext(task);
}
}
@@ -407,9 +407,12 @@ public abstract class AccordExecutor implements CacheSize, AccordCacheEntry.OnLo
submit(AccordExecutor::loadExclusive, Function.identity(), operation);
}
- public <R> void cancel(AccordTask<R> operation)
+ public <R> void cancel(AccordTask<R> task)
{
- submit(AccordExecutor::cancelExclusive, OnCancel::new, operation);
+ Invariants.checkState(task.commandStore.executor() == this,
+ "%s is a wrong command store for %s, should be %s",
+ this, task, task);
+ submit(AccordExecutor::cancelExclusive, CancelAsync::new, task);
}
public void onScannedRanges(AccordTask<?> task, Throwable fail)
@@ -695,43 +698,61 @@ public abstract class AccordExecutor implements CacheSize, AccordCacheEntry.OnLo
abstract protected void addToQueue(TaskQueue queue);
}
- class CommandStoreQueue extends Task
+ static class CommandStoreQueueTask extends Task
{
- final TaskQueue<Task> queue = new TaskQueue<>(WAITING_TO_RUN);
- Task next;
+ private final CommandStoreQueue queue;
+ private Task task;
- CommandStoreQueue(AccordCommandStore commandStore)
+ CommandStoreQueueTask(CommandStoreQueue queue, AccordCommandStore commandStore)
{
super(commandStore);
+ this.queue = queue;
+ }
+
+ public boolean isSet()
+ {
+ return this.task != null;
+ }
+
+ public void reset()
+ {
+ queuePosition = -1;
+ this.task = null;
+ }
+
+ public void setNext(Task task)
+ {
+ queuePosition = task.queuePosition;
+ this.task = task;
}
@Override
protected void preRunExclusive()
{
- Invariants.checkState(next != null);
+ Invariants.checkState(task != null);
Thread self = Thread.currentThread();
commandStore.setOwner(self, self);
- next.preRunExclusive();
+ task.preRunExclusive();
}
@Override
protected void run()
{
- next.run();
+ task.run();
}
@Override
protected void fail(Throwable t)
{
- next.fail(t);
+ task.fail(t);
}
@Override
protected void cleanupExclusive()
{
- next.cleanupExclusive();
+ task.cleanupExclusive();
commandStore.setOwner(null, Thread.currentThread());
- updateNext(queue.poll());
+ queue.updateNext();
}
@Override
@@ -739,25 +760,80 @@ public abstract class AccordExecutor implements CacheSize, AccordCacheEntry.OnLo
{
throw new UnsupportedOperationException();
}
+ }
+
+ class CommandStoreQueue extends TaskQueue<Task>
+ {
+ final CommandStoreQueueTask next;
- void append(Task task)
- { // TODO (expected): if the new task is higher priority, replace next
- if (next == null) updateNext(task);
- else task.addToQueue(queue);
+ CommandStoreQueue(AccordCommandStore commandStore)
+ {
+ super(WAITING_TO_RUN);
+ this.next = new CommandStoreQueueTask(this, commandStore);
+ }
+
+ void updateNext()
+ {
+ updateNext(super.poll());
}
void updateNext(Task task)
{
- next = task;
+ next.reset();
if (task != null)
+ task.addToQueue(this);
+ }
+
+ public void appendOrSetNext(Task task)
+ {
+ if (!next.isSet())
+ task.addToQueue(this);
+ else
+ super.append(task);
+ }
+
+ @Override
+ public void append(Task task)
+ {
+ Invariants.checkState(!next.isSet());
+ // TODO (expected): if the new task is higher priority, replace next
+ next.setNext(task);
+ waitingToRun.append(next);
+ }
+
+ @Override
+ public void remove(Task remove)
+ {
+ if (next.isSet() && next.task == remove)
{
- queuePosition = task.queuePosition;
- waitingToRun.append(this);
+ next.reset();
+ waitingToRun.remove(next);
+ return;
}
+
+ super.remove(remove);
+ }
+
+ @Override
+ public Task poll()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Task peek()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean contains(Task contains)
+ {
+ throw new UnsupportedOperationException();
}
}
- static final class TaskQueue<T extends Task> extends IntrusivePriorityHeap<T>
+ static class TaskQueue<T extends Task> extends IntrusivePriorityHeap<T>
{
final AccordTask.State kind;
@@ -790,7 +866,9 @@ public abstract class AccordExecutor implements CacheSize, AccordCacheEntry.OnLo
public void remove(T remove)
{
+ Invariants.checkState(super.contains(remove));
super.remove(remove);
+ Invariants.checkState(!super.contains(remove));
}
public boolean contains(T contains)
@@ -908,11 +986,11 @@ public abstract class AccordExecutor implements CacheSize, AccordCacheEntry.OnLo
}
}
- private static class OnCancel<R> extends SubmitAsync
+ private static class CancelAsync<R> extends SubmitAsync
{
final AccordTask<R> cancel;
- private OnCancel(AccordTask<R> cancel)
+ private CancelAsync(AccordTask<R> cancel)
{
this.cancel = cancel;
}
diff --git a/src/java/org/apache/cassandra/service/accord/AccordTask.java b/src/java/org/apache/cassandra/service/accord/AccordTask.java
index db36c425622..1f258fcf501 100644
--- a/src/java/org/apache/cassandra/service/accord/AccordTask.java
+++ b/src/java/org/apache/cassandra/service/accord/AccordTask.java
@@ -895,6 +895,9 @@ public abstract class AccordTask<R> extends Task implements Runnable, Function<S
protected void addToQueue(TaskQueue queue)
{
+ if (state == CANCELLED)
+ return;
+
Invariants.checkState(queue.kind == state || (queue.kind == State.WAITING_TO_LOAD && state == WAITING_TO_SCAN_RANGES), "Invalid queue type: %s vs %s", queue.kind, this, AccordTask::toDescription);
Invariants.checkState(this.queued == null, "Already queued with state: %s", this, AccordTask::toDescription);
queued = queue;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
(cassandra) 01/02: Fix Simulator Tests
Posted by if...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch cep-15-accord
in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit da2fa0030442f5788a0a108fb14b7ad9ed618283
Author: Alex Petrov <ol...@gmail.com>
AuthorDate: Mon Jan 20 14:29:49 2025 +0100
Fix Simulator Tests
Patch by Alex Petrov and Benedict Elliott Smith; reviewed by Alex Petrov and Benedict Elliott Smith for CASSANDRA-20240.
---
.../org/apache/cassandra/service/accord/AccordExecutor.java | 6 ++++--
.../service/accord/AccordExecutorAbstractLockLoop.java | 3 ++-
.../service/accord/AccordExecutorInfiniteLoops.java | 2 +-
.../org/apache/cassandra/service/accord/AccordService.java | 2 --
src/java/org/apache/cassandra/utils/ExecutorUtils.java | 2 +-
.../simulator/paxos/PairOfSequencesAccordSimulation.java | 13 +++++++++++--
.../apache/cassandra/simulator/paxos/PaxosSimulation.java | 6 +++++-
.../cassandra/simulator/systems/InterceptibleThread.java | 3 +--
.../simulator/systems/NotInterceptedSyncCondition.java | 5 ++++-
9 files changed, 29 insertions(+), 13 deletions(-)
diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutor.java b/src/java/org/apache/cassandra/service/accord/AccordExecutor.java
index 3ccbe4cc2ef..35fbb35e4cd 100644
--- a/src/java/org/apache/cassandra/service/accord/AccordExecutor.java
+++ b/src/java/org/apache/cassandra/service/accord/AccordExecutor.java
@@ -39,6 +39,7 @@ import accord.utils.QuadFunction;
import accord.utils.QuintConsumer;
import accord.utils.TriConsumer;
import accord.utils.TriFunction;
+import accord.utils.UnhandledEnum;
import accord.utils.async.Cancellable;
import org.agrona.collections.Object2ObjectHashMap;
import org.apache.cassandra.cache.CacheSize;
@@ -482,7 +483,7 @@ public abstract class AccordExecutor implements CacheSize, AccordCacheEntry.OnLo
{
switch (task.state())
{
- default:
+ default: throw new UnhandledEnum(task.state());
case INITIALIZED:
// we could be cancelled before we even reach the queue
task.cancelExclusive();
@@ -498,6 +499,7 @@ public abstract class AccordExecutor implements CacheSize, AccordCacheEntry.OnLo
task.cancelExclusive();
break;
+ case FAILING:
case RUNNING:
case PERSISTING:
case FINISHED:
@@ -958,7 +960,7 @@ public abstract class AccordExecutor implements CacheSize, AccordCacheEntry.OnLo
}
class PlainRunnable extends Task implements Cancellable
- { // TODO (expected): support cancellation
+ {
final AsyncPromise<Void> result;
final Runnable run;
diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLockLoop.java b/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLockLoop.java
index 323929e2c10..c92dccd0f39 100644
--- a/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLockLoop.java
+++ b/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLockLoop.java
@@ -234,7 +234,8 @@ abstract class AccordExecutorAbstractLockLoop extends AccordExecutor
try { agent.onUncaughtException(t); }
catch (Throwable t2) { /* nothing we can sensibly do after already reporting */ }
}
- pauseExclusive();
+ if (isHeldByExecutor)
+ pauseExclusive();
exitLockExclusive();
throw t;
}
diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorInfiniteLoops.java b/src/java/org/apache/cassandra/service/accord/AccordExecutorInfiniteLoops.java
index 5a19d9aa220..e267e0e3a88 100644
--- a/src/java/org/apache/cassandra/service/accord/AccordExecutorInfiniteLoops.java
+++ b/src/java/org/apache/cassandra/service/accord/AccordExecutorInfiniteLoops.java
@@ -93,7 +93,7 @@ class AccordExecutorInfiniteLoops implements Shutdownable
for (Interruptible loop : loops)
{
long wait = deadline - nanoTime();
- if (!loop.awaitTermination(wait, unit))
+ if (!loop.awaitTermination(wait, TimeUnit.NANOSECONDS))
return false;
}
return true;
diff --git a/src/java/org/apache/cassandra/service/accord/AccordService.java b/src/java/org/apache/cassandra/service/accord/AccordService.java
index 5b14572614b..5b5865d6635 100644
--- a/src/java/org/apache/cassandra/service/accord/AccordService.java
+++ b/src/java/org/apache/cassandra/service/accord/AccordService.java
@@ -245,8 +245,6 @@ public class AccordService implements IAccordService, Shutdownable
public synchronized static void startup(NodeId tcmId)
{
- if (instance != null)
- throw new IllegalStateException("Attempt to call startup multiple times");
if (!DatabaseDescriptor.getAccordTransactionsEnabled())
{
instance = NOOP_SERVICE;
diff --git a/src/java/org/apache/cassandra/utils/ExecutorUtils.java b/src/java/org/apache/cassandra/utils/ExecutorUtils.java
index b37c8ac9627..a28225456c7 100644
--- a/src/java/org/apache/cassandra/utils/ExecutorUtils.java
+++ b/src/java/org/apache/cassandra/utils/ExecutorUtils.java
@@ -105,7 +105,7 @@ public class ExecutorUtils
}
catch (Throwable t)
{
- throw new IllegalStateException("Caught interrupt while shutting down " + executor);
+ throw new IllegalStateException("Caught interrupt while shutting down " + executor, t);
}
}
diff --git a/test/simulator/main/org/apache/cassandra/simulator/paxos/PairOfSequencesAccordSimulation.java b/test/simulator/main/org/apache/cassandra/simulator/paxos/PairOfSequencesAccordSimulation.java
index b8de7f0cd68..f829da06c6e 100644
--- a/test/simulator/main/org/apache/cassandra/simulator/paxos/PairOfSequencesAccordSimulation.java
+++ b/test/simulator/main/org/apache/cassandra/simulator/paxos/PairOfSequencesAccordSimulation.java
@@ -19,9 +19,11 @@
package org.apache.cassandra.simulator.paxos;
import java.nio.ByteBuffer;
+import java.nio.channels.ClosedChannelException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.concurrent.CancellationException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.BiFunction;
import java.util.function.LongSupplier;
@@ -34,6 +36,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import accord.coordinate.CoordinationFailed;
+import accord.coordinate.Invalidated;
import com.carrotsearch.hppc.IntArrayList;
import com.carrotsearch.hppc.IntHashSet;
import com.carrotsearch.hppc.cursors.IntCursor;
@@ -59,6 +62,7 @@ import org.apache.cassandra.simulator.RunnableActionScheduler;
import org.apache.cassandra.simulator.cluster.ClusterActions;
import org.apache.cassandra.simulator.systems.SimulatedSystems;
import org.apache.cassandra.simulator.utils.IntRange;
+import org.apache.cassandra.streaming.StreamReceivedOutOfTokenRangeException;
import static org.apache.cassandra.simulator.paxos.HistoryChecker.fail;
@@ -155,8 +159,13 @@ public class PairOfSequencesAccordSimulation extends AbstractPairOfSequencesPaxo
@Override
protected Class<? extends Throwable>[] expectedExceptions()
{
- return (Class<? extends Throwable>[]) new Class<?>[] { RequestExecutionException.class,
- CoordinationFailed.class };
+ return (Class<? extends Throwable>[]) new Class<?>[] { CancellationException.class,
+ CoordinationFailed.class,
+ ClosedChannelException.class,
+ Invalidated.class,
+ RequestExecutionException.class,
+ StreamReceivedOutOfTokenRangeException.class // should always come in combination with closed channel exception
+ };
}
@Override
diff --git a/test/simulator/main/org/apache/cassandra/simulator/paxos/PaxosSimulation.java b/test/simulator/main/org/apache/cassandra/simulator/paxos/PaxosSimulation.java
index 74ffd4db563..774c44a3a0e 100644
--- a/test/simulator/main/org/apache/cassandra/simulator/paxos/PaxosSimulation.java
+++ b/test/simulator/main/org/apache/cassandra/simulator/paxos/PaxosSimulation.java
@@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.CancellationException;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@@ -76,9 +77,12 @@ public abstract class PaxosSimulation implements Simulation, ClusterActionListen
return (primaryKeys.length == 1 ? Integer.toString(primaryKeys[0]) : Arrays.toString(primaryKeys)) + "/" + id + ": " + idString;
}
+ @SuppressWarnings("unchecked")
protected Class<? extends Throwable>[] expectedExceptions()
{
- return (Class<? extends Throwable>[]) new Class<?>[] { RequestExecutionException.class, Invalidated.class };
+ return (Class<? extends Throwable>[]) new Class<?>[] { RequestExecutionException.class,
+ Invalidated.class,
+ CancellationException.class };
}
abstract class Operation extends SimulatedActionCallable<SimpleQueryResult> implements BiConsumer<SimpleQueryResult, Throwable>
diff --git a/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptibleThread.java b/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptibleThread.java
index 0cb26bf0e5a..1300a0a47d3 100644
--- a/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptibleThread.java
+++ b/test/simulator/main/org/apache/cassandra/simulator/systems/InterceptibleThread.java
@@ -159,8 +159,7 @@ public class InterceptibleThread extends FastThreadLocalThread implements Interc
}
catch (InterruptedException e)
{
- if (!isTriggered()) throw new UncheckedInterruptedException(e);
- else doInterrupt();
+ doInterrupt();
}
}
diff --git a/test/simulator/main/org/apache/cassandra/simulator/systems/NotInterceptedSyncCondition.java b/test/simulator/main/org/apache/cassandra/simulator/systems/NotInterceptedSyncCondition.java
index 6c53a4e981f..04a72b695c5 100644
--- a/test/simulator/main/org/apache/cassandra/simulator/systems/NotInterceptedSyncCondition.java
+++ b/test/simulator/main/org/apache/cassandra/simulator/systems/NotInterceptedSyncCondition.java
@@ -64,7 +64,10 @@ public class NotInterceptedSyncCondition extends Awaitable.AbstractAwaitable imp
if (wait <= 0)
return false;
- monitor.wait((wait + 999999) / 1000000);
+ wait = (wait + 999999) / 1000000;
+ if (wait < 0)
+ wait = Long.MAX_VALUE;
+ monitor.wait(wait);
return true;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org