You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by yc...@apache.org on 2025/01/03 20:43:22 UTC
(cassandra-sidecar) branch trunk updated: CASSSIDECAR-181: Ensure memory consistency from PeriodicTask executio… (#167)
This is an automated email from the ASF dual-hosted git repository.
ycai pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra-sidecar.git
The following commit(s) were added to refs/heads/trunk by this push:
new 58a6bfe1 CASSSIDECAR-181: Ensure memory consistency from PeriodicTask executio… (#167)
58a6bfe1 is described below
commit 58a6bfe140021d21283e45d95ab50c8c132968ed
Author: Yifan Cai <yc...@apache.org>
AuthorDate: Fri Jan 3 12:43:17 2025 -0800
CASSSIDECAR-181: Ensure memory consistency from PeriodicTask executio… (#167)
Patch by Yifan Cai; Reviewed by Francisco Guerrero for CASSSIDECAR-181
---
CHANGES.txt | 1 +
checkstyle.xml | 1 +
server/build.gradle | 2 +
.../sidecar/concurrent/TaskExecutorPool.java | 5 +-
.../sidecar/coordination/ClusterLease.java | 69 ++++-
.../coordination/ClusterLeaseClaimTask.java | 22 +-
.../cassandra/sidecar/job/OperationalJob.java | 3 +-
.../sidecar/restore/RestoreJobDiscoverer.java | 36 ++-
.../sidecar/restore/RestoreProcessor.java | 6 +-
.../sidecar/restore/RingTopologyRefresher.java | 5 +-
.../sidecar/tasks/HealthCheckPeriodicTask.java | 44 +--
.../sidecar/tasks/KeyStoreCheckPeriodicTask.java | 22 +-
.../cassandra/sidecar/tasks/PeriodicTask.java | 23 +-
.../sidecar/tasks/PeriodicTaskExecutor.java | 258 +++++++++++-----
...ionDetermination.java => ScheduleDecision.java} | 20 +-
.../org/apache/cassandra/sidecar/tasks/Task.java | 4 +-
.../ClusterLeaseClaimTaskIntegrationTest.java | 18 +-
.../sidecar/testing/IntegrationTestModule.java | 3 +-
.../sidecar/CassandraSidecarDaemonTest.java | 6 +
.../coordination/ClusterLeaseClaimTaskTest.java | 12 +-
.../cassandra/sidecar/db/SidecarSchemaTest.java | 3 +-
.../sidecar/metrics/SchemaMetricsTest.java | 3 +-
.../sidecar/restore/RestoreJobDiscovererTest.java | 5 +-
.../sidecar/restore/RestoreProcessorTest.java | 3 +-
.../sidecar/tasks/PeriodicTaskExecutorTest.java | 329 +++++++++++++++++++--
25 files changed, 681 insertions(+), 222 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 057500b8..ffbd3cc6 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,6 @@
1.0.0
-----
+ * Ensure memory consistency from PeriodicTask executions and expose richer ScheduleDecision (CASSSIDECAR-181)
* Refactor access to delegate methods to simplify (CASSSIDECAR-182)
* Renaming InstancesConfig to InstancesMetadata (CASSSIDECAR-175)
* Mechanism to have a reduced number of Sidecar instances run operations (CASSSIDECAR-174)
diff --git a/checkstyle.xml b/checkstyle.xml
index 2b4b36e6..0b19a07e 100644
--- a/checkstyle.xml
+++ b/checkstyle.xml
@@ -329,6 +329,7 @@ page at http://checkstyle.sourceforge.net/config.html -->
SL, SLIST, SL_ASSIGN, SR_ASSIGN, STAR, STAR_ASSIGN" />
<property name="allowEmptyConstructors" value="true" />
<property name="allowEmptyMethods" value="true" />
+ <property name="allowEmptyLambdas" value="true" />
<property name="severity" value="error" />
</module>
diff --git a/server/build.gradle b/server/build.gradle
index 565b30a5..d74f55df 100644
--- a/server/build.gradle
+++ b/server/build.gradle
@@ -191,6 +191,8 @@ publishing {
test {
systemProperty "vertxweb.environment", "dev"
systemProperty "vertx.logger-delegate-factory-class-name", "io.vertx.core.logging.SLF4JLogDelegateFactory"
+ // There is no native lib (JNR) for getting time for testing
+ systemProperty "com.datastax.driver.USE_NATIVE_CLOCK", "false"
// ordinarily we don't need integration tests
// see the integrationTest task
useJUnitPlatform()
diff --git a/server/src/main/java/org/apache/cassandra/sidecar/concurrent/TaskExecutorPool.java b/server/src/main/java/org/apache/cassandra/sidecar/concurrent/TaskExecutorPool.java
index 53292ef4..3e58dc7d 100644
--- a/server/src/main/java/org/apache/cassandra/sidecar/concurrent/TaskExecutorPool.java
+++ b/server/src/main/java/org/apache/cassandra/sidecar/concurrent/TaskExecutorPool.java
@@ -167,11 +167,10 @@ public abstract class TaskExecutorPool implements WorkerExecutor
}
@Override
- public <T> Future<T> executeBlocking(Handler<Promise<T>> blockingCodeHandler,
- boolean ordered)
+ public <T> Future<T> executeBlocking(Handler<Promise<T>> blockingCodeHandler, boolean ordered)
{
// TODO: migrate to org.apache.cassandra.sidecar.concurrent.TaskExecutorPool.executeBlocking(java.util.concurrent.Callable<T>, boolean)
- return StopWatch.measureTimeTaken(workerExecutor.executeBlocking(blockingCodeHandler), this::recordTimeTaken);
+ return StopWatch.measureTimeTaken(workerExecutor.executeBlocking(blockingCodeHandler, ordered), this::recordTimeTaken);
}
@Override
diff --git a/server/src/main/java/org/apache/cassandra/sidecar/coordination/ClusterLease.java b/server/src/main/java/org/apache/cassandra/sidecar/coordination/ClusterLease.java
index 2f1c02d2..6b1c1909 100644
--- a/server/src/main/java/org/apache/cassandra/sidecar/coordination/ClusterLease.java
+++ b/server/src/main/java/org/apache/cassandra/sidecar/coordination/ClusterLease.java
@@ -21,7 +21,8 @@ package org.apache.cassandra.sidecar.coordination;
import java.util.Objects;
import com.google.inject.Singleton;
-import org.apache.cassandra.sidecar.tasks.ExecutionDetermination;
+import org.apache.cassandra.sidecar.tasks.ScheduleDecision;
+import org.jetbrains.annotations.VisibleForTesting;
/**
* Holds information about whether this Sidecar instance has claimed the lease, it has not, or it's unable
@@ -30,23 +31,24 @@ import org.apache.cassandra.sidecar.tasks.ExecutionDetermination;
@Singleton
public class ClusterLease
{
- private volatile ExecutionDetermination executionDetermination = ExecutionDetermination.INDETERMINATE;
+ private volatile Ownership leaseOwnership;
public ClusterLease()
{
+ this(Ownership.INDETERMINATE);
}
- public ClusterLease(ExecutionDetermination executionDetermination)
+ public ClusterLease(Ownership leaseOwnership)
{
- this.executionDetermination = executionDetermination;
+ this.leaseOwnership = leaseOwnership;
}
/**
* @return the current execution determination
*/
- public ExecutionDetermination executionDetermination()
+ public ScheduleDecision toScheduleDecision()
{
- return executionDetermination;
+ return leaseOwnership.toScheduleDecision();
}
/**
@@ -54,16 +56,61 @@ public class ClusterLease
*/
public boolean isClaimedByLocalSidecar()
{
- return executionDetermination.shouldExecuteOnLocalSidecar();
+ return leaseOwnership == Ownership.CLAIMED;
}
/**
- * Updates the determination
+ * Updates the ownership of the lease
*
- * @param executionDetermination the new value
+ * @param ownership ownership of the lease
*/
- void setExecutionDetermination(ExecutionDetermination executionDetermination)
+ void setOwnership(Ownership ownership)
{
- this.executionDetermination = Objects.requireNonNull(executionDetermination, "executionDetermination must be provided");
+ this.leaseOwnership = Objects.requireNonNull(ownership, "ownership must be provided");
+ }
+
+ @VisibleForTesting // only use it for testing
+ public void setOwnershipTesting(Ownership ownership)
+ {
+ setOwnership(ownership);
+ }
+
+ /**
+ * Ownership of the cluster-wide lease
+ */
+ public enum Ownership
+ {
+ /**
+ * The cluster lease is claimed by the local Sidecar
+ */
+ CLAIMED,
+ /**
+ * The cluster lease is lost by the local Sidecar. It is claimed by another remote Sidecar.
+ */
+ LOST,
+ /**
+ * The cluster lease is neither claimed by the local Sidecar nor any other remote Sidecar
+ */
+ INDETERMINATE;
+
+ private ScheduleDecision toScheduleDecision()
+ {
+ switch (this)
+ {
+ case CLAIMED:
+ return ScheduleDecision.EXECUTE;
+ case LOST:
+ return ScheduleDecision.SKIP;
+ case INDETERMINATE:
+ default:
+ // When the process is unable to determine whether it is the leaseholder, we want
+ // the task to be rescheduled for a shorter period of time. Assume you have a PeriodicTask that
+ // runs every day. If it happens to run during a period of time when there's no
+ // determination whether task should run or not, we do not want to wait another day for the
+ // task to run, so instead we reschedule it and only wait the initial delay of the task for
+ // the task to try again.
+ return ScheduleDecision.RESCHEDULE;
+ }
+ }
}
}
diff --git a/server/src/main/java/org/apache/cassandra/sidecar/coordination/ClusterLeaseClaimTask.java b/server/src/main/java/org/apache/cassandra/sidecar/coordination/ClusterLeaseClaimTask.java
index 05e09531..77216a05 100644
--- a/server/src/main/java/org/apache/cassandra/sidecar/coordination/ClusterLeaseClaimTask.java
+++ b/server/src/main/java/org/apache/cassandra/sidecar/coordination/ClusterLeaseClaimTask.java
@@ -36,8 +36,8 @@ import org.apache.cassandra.sidecar.config.ServiceConfiguration;
import org.apache.cassandra.sidecar.db.SidecarLeaseDatabaseAccessor;
import org.apache.cassandra.sidecar.metrics.CoordinationMetrics;
import org.apache.cassandra.sidecar.metrics.SidecarMetrics;
-import org.apache.cassandra.sidecar.tasks.ExecutionDetermination;
import org.apache.cassandra.sidecar.tasks.PeriodicTask;
+import org.apache.cassandra.sidecar.tasks.ScheduleDecision;
import org.jetbrains.annotations.VisibleForTesting;
import static org.apache.cassandra.sidecar.server.SidecarServerEvents.ON_SIDECAR_GLOBAL_LEASE_CLAIMED;
@@ -73,8 +73,8 @@ public class ClusterLeaseClaimTask implements PeriodicTask
private final PeriodicTaskConfiguration periodicTaskConfiguration;
private final ServiceConfiguration config;
private final Vertx vertx;
- private volatile String currentLeaseholder;
- private volatile Instant leaseTime;
+ private String currentLeaseholder;
+ private Instant leaseTime;
public ClusterLeaseClaimTask(Vertx vertx,
ServiceConfiguration serviceConfiguration,
@@ -96,7 +96,7 @@ public class ClusterLeaseClaimTask implements PeriodicTask
* {@inheritDoc}
*/
@Override
- public boolean shouldSkip()
+ public ScheduleDecision scheduleDecision()
{
// The Sidecar schema feature is required for this implementation
// so skip when the feature is not enabled
@@ -111,10 +111,10 @@ public class ClusterLeaseClaimTask implements PeriodicTask
}
if (!isEnabled || !isMember)
{
- clusterLease.setExecutionDetermination(ExecutionDetermination.SKIP_EXECUTION);
- return true;
+ clusterLease.setOwnership(ClusterLease.Ownership.LOST);
+ return ScheduleDecision.SKIP;
}
- return false;
+ return ScheduleDecision.EXECUTE;
}
/**
@@ -225,7 +225,7 @@ public class ClusterLeaseClaimTask implements PeriodicTask
}
leaseTime = null;
- clusterLease.setExecutionDetermination(ExecutionDetermination.INDETERMINATE);
+ clusterLease.setOwnership(ClusterLease.Ownership.INDETERMINATE);
}
return;
}
@@ -233,12 +233,12 @@ public class ClusterLeaseClaimTask implements PeriodicTask
if (isCurrentLeaseholder(sidecarHostId))
{
leaseTime = Instant.now();
- clusterLease.setExecutionDetermination(ExecutionDetermination.EXECUTE);
+ clusterLease.setOwnership(ClusterLease.Ownership.CLAIMED);
}
else
{
leaseTime = null;
- clusterLease.setExecutionDetermination(ExecutionDetermination.SKIP_EXECUTION);
+ clusterLease.setOwnership(ClusterLease.Ownership.LOST);
}
}
@@ -314,6 +314,6 @@ public class ClusterLeaseClaimTask implements PeriodicTask
{
currentLeaseholder = null;
leaseTime = null;
- clusterLease.setExecutionDetermination(ExecutionDetermination.INDETERMINATE);
+ clusterLease.setOwnership(ClusterLease.Ownership.INDETERMINATE);
}
}
diff --git a/server/src/main/java/org/apache/cassandra/sidecar/job/OperationalJob.java b/server/src/main/java/org/apache/cassandra/sidecar/job/OperationalJob.java
index 1076b44f..80ec374c 100644
--- a/server/src/main/java/org/apache/cassandra/sidecar/job/OperationalJob.java
+++ b/server/src/main/java/org/apache/cassandra/sidecar/job/OperationalJob.java
@@ -193,11 +193,11 @@ public abstract class OperationalJob implements Task<Void>
{
isExecuting = true;
LOGGER.info("Executing job. jobId={}", jobId);
+ promise.future().onComplete(executionPromise);
try
{
// Blocking call to perform concrete job-specific execution, returning the status
executeInternal();
- executionPromise.tryComplete();
promise.tryComplete();
if (LOGGER.isDebugEnabled())
{
@@ -208,7 +208,6 @@ public abstract class OperationalJob implements Task<Void>
{
OperationalJobException oje = OperationalJobException.wraps(e);
LOGGER.error("Job execution failed. jobId={} reason='{}'", jobId, oje.getMessage(), oje);
- executionPromise.tryFail(oje);
promise.tryFail(oje);
}
}
diff --git a/server/src/main/java/org/apache/cassandra/sidecar/restore/RestoreJobDiscoverer.java b/server/src/main/java/org/apache/cassandra/sidecar/restore/RestoreJobDiscoverer.java
index 1984a1eb..308ff2b0 100644
--- a/server/src/main/java/org/apache/cassandra/sidecar/restore/RestoreJobDiscoverer.java
+++ b/server/src/main/java/org/apache/cassandra/sidecar/restore/RestoreJobDiscoverer.java
@@ -56,6 +56,7 @@ import org.apache.cassandra.sidecar.metrics.RestoreMetrics;
import org.apache.cassandra.sidecar.metrics.SidecarMetrics;
import org.apache.cassandra.sidecar.tasks.PeriodicTask;
import org.apache.cassandra.sidecar.tasks.PeriodicTaskExecutor;
+import org.apache.cassandra.sidecar.tasks.ScheduleDecision;
import org.apache.cassandra.sidecar.utils.InstanceMetadataFetcher;
/**
@@ -133,22 +134,9 @@ public class RestoreJobDiscoverer implements PeriodicTask
}
@Override
- public boolean shouldSkip()
+ public ScheduleDecision scheduleDecision()
{
- boolean shouldSkip = !sidecarSchema.isInitialized();
- if (shouldSkip)
- {
- LOGGER.trace("Skipping restore job discovering due to sidecarSchema not initialized");
- }
- boolean executing = isExecuting.get();
- shouldSkip = shouldSkip || executing;
- if (executing)
- {
- LOGGER.trace("Skipping restore job discovering due to overlapping execution of this task");
- }
-
- // skip the task
- return shouldSkip;
+ return shouldSkip() ? ScheduleDecision.SKIP : ScheduleDecision.EXECUTE;
}
@Override
@@ -189,6 +177,24 @@ public class RestoreJobDiscoverer implements PeriodicTask
}
}
+ private boolean shouldSkip()
+ {
+ boolean shouldSkip = !sidecarSchema.isInitialized();
+ if (shouldSkip)
+ {
+ LOGGER.trace("Skipping restore job discovering due to sidecarSchema not initialized");
+ }
+ boolean executing = isExecuting.get();
+ shouldSkip = shouldSkip || executing;
+ if (executing)
+ {
+ LOGGER.trace("Skipping restore job discovering due to overlapping execution of this task");
+ }
+
+ // skip the task
+ return shouldSkip;
+ }
+
private void executeInternal()
{
Preconditions.checkState(periodicTaskExecutor != null, "Loop executor is not registered");
diff --git a/server/src/main/java/org/apache/cassandra/sidecar/restore/RestoreProcessor.java b/server/src/main/java/org/apache/cassandra/sidecar/restore/RestoreProcessor.java
index 76a0b321..58b4bc77 100644
--- a/server/src/main/java/org/apache/cassandra/sidecar/restore/RestoreProcessor.java
+++ b/server/src/main/java/org/apache/cassandra/sidecar/restore/RestoreProcessor.java
@@ -49,6 +49,7 @@ import org.apache.cassandra.sidecar.exceptions.RestoreJobExceptions;
import org.apache.cassandra.sidecar.metrics.SidecarMetrics;
import org.apache.cassandra.sidecar.metrics.instance.InstanceRestoreMetrics;
import org.apache.cassandra.sidecar.tasks.PeriodicTask;
+import org.apache.cassandra.sidecar.tasks.ScheduleDecision;
import org.apache.cassandra.sidecar.utils.SSTableImporter;
/**
@@ -130,14 +131,15 @@ public class RestoreProcessor implements PeriodicTask
}
@Override
- public boolean shouldSkip()
+ public ScheduleDecision scheduleDecision()
{
boolean shouldSkip = !sidecarSchema().isInitialized();
if (shouldSkip)
{
LOGGER.trace("Skipping restore job processing");
+ return ScheduleDecision.SKIP;
}
- return shouldSkip;
+ return ScheduleDecision.EXECUTE;
}
@Override
diff --git a/server/src/main/java/org/apache/cassandra/sidecar/restore/RingTopologyRefresher.java b/server/src/main/java/org/apache/cassandra/sidecar/restore/RingTopologyRefresher.java
index 53ba5059..27db0081 100644
--- a/server/src/main/java/org/apache/cassandra/sidecar/restore/RingTopologyRefresher.java
+++ b/server/src/main/java/org/apache/cassandra/sidecar/restore/RingTopologyRefresher.java
@@ -42,6 +42,7 @@ import org.apache.cassandra.sidecar.config.SidecarConfiguration;
import org.apache.cassandra.sidecar.db.RestoreJob;
import org.apache.cassandra.sidecar.exceptions.CassandraUnavailableException;
import org.apache.cassandra.sidecar.tasks.PeriodicTask;
+import org.apache.cassandra.sidecar.tasks.ScheduleDecision;
import org.apache.cassandra.sidecar.utils.InstanceMetadataFetcher;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.annotations.VisibleForTesting;
@@ -74,9 +75,9 @@ public class RingTopologyRefresher implements PeriodicTask
}
@Override
- public boolean shouldSkip()
+ public ScheduleDecision scheduleDecision()
{
- return replicaByTokenRangePerKeyspace.isEmpty();
+ return replicaByTokenRangePerKeyspace.isEmpty() ? ScheduleDecision.SKIP : ScheduleDecision.EXECUTE;
}
@Override
diff --git a/server/src/main/java/org/apache/cassandra/sidecar/tasks/HealthCheckPeriodicTask.java b/server/src/main/java/org/apache/cassandra/sidecar/tasks/HealthCheckPeriodicTask.java
index daaeb44f..20e7bac2 100644
--- a/server/src/main/java/org/apache/cassandra/sidecar/tasks/HealthCheckPeriodicTask.java
+++ b/server/src/main/java/org/apache/cassandra/sidecar/tasks/HealthCheckPeriodicTask.java
@@ -30,13 +30,13 @@ import io.vertx.core.Promise;
import io.vertx.core.Vertx;
import io.vertx.core.eventbus.EventBus;
import org.apache.cassandra.sidecar.cluster.InstancesMetadata;
+import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata;
import org.apache.cassandra.sidecar.concurrent.ExecutorPools;
import org.apache.cassandra.sidecar.concurrent.TaskExecutorPool;
import org.apache.cassandra.sidecar.config.SidecarConfiguration;
import org.apache.cassandra.sidecar.metrics.HealthMetrics;
import org.apache.cassandra.sidecar.metrics.SidecarMetrics;
-import static org.apache.cassandra.sidecar.server.SidecarServerEvents.ON_SERVER_START;
import static org.apache.cassandra.sidecar.server.SidecarServerEvents.ON_SERVER_STOP;
/**
@@ -67,7 +67,6 @@ public class HealthCheckPeriodicTask implements PeriodicTask
@Override
public void registerPeriodicTaskExecutor(PeriodicTaskExecutor executor)
{
- eventBus.localConsumer(ON_SERVER_START.address(), message -> executor.schedule(this));
eventBus.localConsumer(ON_SERVER_STOP.address(), message -> executor.unschedule(this));
}
@@ -92,30 +91,12 @@ public class HealthCheckPeriodicTask implements PeriodicTask
AtomicInteger instanceDown = new AtomicInteger(0);
List<Future<?>> futures = instancesMetadata.instances()
.stream()
- .map(instanceMetadata -> internalPool.executeBlocking(p -> {
- try
- {
- instanceMetadata.delegate().healthCheck();
- p.complete();
- }
- catch (Throwable cause)
- {
- instanceDown.incrementAndGet();
- p.fail(cause);
- LOGGER.error("Unable to complete health check on instance={}",
- instanceMetadata.id(), cause);
- }
- }, false))
+ .map(instanceMetadata -> healthCheck(instanceMetadata, instanceDown))
.collect(Collectors.toList());
// join always waits until all its futures are completed and will not fail as soon as one of the future fails
Future.join(futures)
- .onComplete(v -> {
- int instanceDownCount = instanceDown.get();
- int instanceUpCount = instancesMetadata.instances().size() - instanceDownCount;
- metrics.cassandraInstancesUp.metric.setValue(instanceUpCount);
- metrics.cassandraInstancesDown.metric.setValue(instanceDownCount);
- })
+ .onComplete(v -> updateMetrics(instanceDown))
.onSuccess(v -> promise.complete())
.onFailure(promise::fail);
}
@@ -125,4 +106,23 @@ public class HealthCheckPeriodicTask implements PeriodicTask
{
return "Health Check";
}
+
+ private void updateMetrics(AtomicInteger instanceDown)
+ {
+ int instanceDownCount = instanceDown.get();
+ int instanceUpCount = instancesMetadata.instances().size() - instanceDownCount;
+ metrics.cassandraInstancesUp.metric.setValue(instanceUpCount);
+ metrics.cassandraInstancesDown.metric.setValue(instanceDownCount);
+ }
+
+ private Future<Void> healthCheck(InstanceMetadata instanceMetadata, AtomicInteger instanceDown)
+ {
+ return internalPool
+ .runBlocking(() -> instanceMetadata.delegate().healthCheck(), false)
+ .onFailure(cause -> {
+ instanceDown.incrementAndGet();
+ LOGGER.error("Unable to complete health check on instance={}",
+ instanceMetadata.id(), cause);
+ });
+ }
}
diff --git a/server/src/main/java/org/apache/cassandra/sidecar/tasks/KeyStoreCheckPeriodicTask.java b/server/src/main/java/org/apache/cassandra/sidecar/tasks/KeyStoreCheckPeriodicTask.java
index d9744f8c..f05b73da 100644
--- a/server/src/main/java/org/apache/cassandra/sidecar/tasks/KeyStoreCheckPeriodicTask.java
+++ b/server/src/main/java/org/apache/cassandra/sidecar/tasks/KeyStoreCheckPeriodicTask.java
@@ -52,14 +52,14 @@ public class KeyStoreCheckPeriodicTask implements PeriodicTask
/**
* Skip check if the key store is not configured or if the key store should not be reloaded
*
- * @return {@code true} if the key store is not configured or if the keystore should not be reloaded,
- * {@code false} otherwise
+ * @return whether to {@link ScheduleDecision#SKIP} or {@link ScheduleDecision#EXECUTE} this task
*/
@Override
- public boolean shouldSkip()
+ public ScheduleDecision scheduleDecision()
{
- return !configuration.isKeystoreConfigured()
- || !configuration.keystore().reloadStore();
+ return shouldSkip()
+ ? ScheduleDecision.SKIP
+ : ScheduleDecision.EXECUTE;
}
@Override
@@ -124,4 +124,16 @@ public class KeyStoreCheckPeriodicTask implements PeriodicTask
lastModifiedTime = -1;
});
}
+
+ /**
+ * Skip check if the key store is not configured or if the key store should not be reloaded
+ *
+ * @return {@code true} if the key store is not configured or if the keystore should not be reloaded,
+ * {@code false} otherwise
+ */
+ private boolean shouldSkip()
+ {
+ return !configuration.isKeystoreConfigured()
+ || !configuration.keystore().reloadStore();
+ }
}
diff --git a/server/src/main/java/org/apache/cassandra/sidecar/tasks/PeriodicTask.java b/server/src/main/java/org/apache/cassandra/sidecar/tasks/PeriodicTask.java
index 6525b2a8..04731819 100644
--- a/server/src/main/java/org/apache/cassandra/sidecar/tasks/PeriodicTask.java
+++ b/server/src/main/java/org/apache/cassandra/sidecar/tasks/PeriodicTask.java
@@ -20,6 +20,8 @@ package org.apache.cassandra.sidecar.tasks;
import java.util.concurrent.TimeUnit;
+import io.vertx.core.Promise;
+
/**
* An interface that defines a periodic task that will be executed during the lifecycle of Cassandra Sidecar
*/
@@ -38,6 +40,11 @@ public interface PeriodicTask extends Task<Void>
return TimeUnit.MILLISECONDS;
}
+ default long delayMillis()
+ {
+ return delayUnit().toMillis(delay());
+ }
+
/**
* @return the initial delay for the task, defaults to the {@link #delay()}
*/
@@ -54,6 +61,11 @@ public interface PeriodicTask extends Task<Void>
return delayUnit();
}
+ default long initialDelayMillis()
+ {
+ return initialDelayUnit().toMillis(initialDelay());
+ }
+
/**
* Register the periodic task executor at the task. By default, it is no-op.
* If the reference to the executor is needed, the concrete {@link PeriodicTask} can implement this method
@@ -65,13 +77,14 @@ public interface PeriodicTask extends Task<Void>
}
/**
- * Specify whether the task should be skipped.
- * // TODO: consider returning the reason to skip, instead of just a boolean value
- * @return {@code true} to skip; otherwise, return {@code false}
+ * Specify the schedule decision of the upcoming run.
+ * The method is evaluated before calling {@link #execute(Promise)}
+ *
+ * @return schedule decision. The default is to {@link ScheduleDecision#EXECUTE}.
*/
- default boolean shouldSkip()
+ default ScheduleDecision scheduleDecision()
{
- return false;
+ return ScheduleDecision.EXECUTE;
}
@Override
diff --git a/server/src/main/java/org/apache/cassandra/sidecar/tasks/PeriodicTaskExecutor.java b/server/src/main/java/org/apache/cassandra/sidecar/tasks/PeriodicTaskExecutor.java
index 4be8c427..bbb5afb1 100644
--- a/server/src/main/java/org/apache/cassandra/sidecar/tasks/PeriodicTaskExecutor.java
+++ b/server/src/main/java/org/apache/cassandra/sidecar/tasks/PeriodicTaskExecutor.java
@@ -21,6 +21,8 @@ package org.apache.cassandra.sidecar.tasks;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -28,8 +30,8 @@ import org.slf4j.LoggerFactory;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import io.vertx.core.Closeable;
+import io.vertx.core.Future;
import io.vertx.core.Promise;
-import io.vertx.core.impl.ConcurrentHashSet;
import org.apache.cassandra.sidecar.concurrent.ExecutorPools;
import org.apache.cassandra.sidecar.concurrent.TaskExecutorPool;
import org.apache.cassandra.sidecar.coordination.ClusterLease;
@@ -38,23 +40,28 @@ import org.jetbrains.annotations.VisibleForTesting;
/**
* This class manages the scheduling and execution of {@link PeriodicTask}s.
+ * For the {@link PeriodicTask} that also is {@link ExecuteOnClusterLeaseholderOnly}, the executor ensures
+ * the task execution is only performed when {@link ClusterLease} is claimed by the local Sidecar instance.
+ *
+ * <p>The execution of each {@link PeriodicTask} is <i>ordered</i> and <i>serial</i>, meanwhile there could
+ * be concurrent execution of different {@link PeriodicTask}s.
+ * <p>Memory consistency effects: Actions in the prior {@link PeriodicTask} run <i>happen-before</i> its
+ * next run, perhaps in another thread. In other words, writes in the prior run can be read by its next run,
+ * as if it is running in a single thread.
*/
@Singleton
public class PeriodicTaskExecutor implements Closeable
{
private static final Logger LOGGER = LoggerFactory.getLogger(PeriodicTaskExecutor.class);
+ private static final long PLACEHOLDER_TIMER_ID = -1L; // used when the run start immediately, not scheduled via a timer
+ // keep track of the timerIds in order to cancel them when closing/unscheduling
private final Map<PeriodicTaskKey, Long> timerIds = new ConcurrentHashMap<>();
- private final Set<PeriodicTaskKey> activeTasks = new ConcurrentHashSet<>();
+ private final Map<PeriodicTaskKey, Future<Void>> activeRuns = new ConcurrentHashMap<>();
+ private final Set<PeriodicTaskKey> poisonPilledTasks = ConcurrentHashMap.newKeySet();
private final TaskExecutorPool internalPool;
private final ClusterLease clusterLease;
- @VisibleForTesting
- public PeriodicTaskExecutor(ExecutorPools executorPools)
- {
- this(executorPools, null);
- }
-
@Inject
public PeriodicTaskExecutor(ExecutorPools executorPools, ClusterLease clusterLease)
{
@@ -66,53 +73,142 @@ public class PeriodicTaskExecutor implements Closeable
* Schedules the {@code task} iff it has not been scheduled yet.
*
* @param task the task to execute
- * @return the identifier of the timer associated with this task
*/
- public long schedule(PeriodicTask task)
+ public void schedule(PeriodicTask task)
{
PeriodicTaskKey key = new PeriodicTaskKey(task);
- return timerIds.computeIfAbsent(key, k -> {
- task.registerPeriodicTaskExecutor(this);
- long initialDelayMillis = task.initialDelayUnit().toMillis(task.initialDelay());
- long delayMillis = task.delayUnit().toMillis(task.delay());
- return internalPool.setPeriodic(initialDelayMillis, delayMillis, id -> executeInternal(k));
+ schedule(key, 0, task.initialDelayMillis(), 0);
+ }
+
+ private void schedule(PeriodicTaskKey key, long priorExecDurationMillis, long delayMillis, long execCount)
+ {
+ long actualDelayMillis = delayMillis - priorExecDurationMillis;
+ AtomicBoolean runImmediately = new AtomicBoolean(actualDelayMillis <= 0);
+ timerIds.compute(key, (k, v) -> {
+ // The periodic task has been scheduled already. Exit early and avoid scheduling the duplication
+ if (v != null && execCount == 0)
+ {
+ LOGGER.debug("Task is already scheduled. task='{}'", key);
+ runImmediately.set(false);
+ return v;
+ }
+
+ LOGGER.debug("Scheduling task {}. task='{}' execCount={}",
+ runImmediately.get() ? "immediately" : "in " + actualDelayMillis + " milliseconds",
+ key, execCount);
+
+ try
+ {
+ key.task.registerPeriodicTaskExecutor(this);
+ }
+ catch (Exception e)
+ {
+ LOGGER.warn("Failed to invoke registerPeriodicTaskExecutor. task='{}'", key, e);
+ }
+
+ // If run immediately, do not execute within the compute block.
+ // Return the placeholder timer ID, and execute after exiting the compute block.
+ if (runImmediately.get())
+ {
+ return PLACEHOLDER_TIMER_ID; // use the placeholder ID, since this run is not scheduled as a timer
+ }
+ // Schedule and update the timer id
+ return internalPool.setTimer(delayMillis, timerId -> executeAndScheduleNext(key, execCount));
});
+
+ if (runImmediately.get())
+ {
+ executeAndScheduleNext(key, execCount);
+ }
+ }
+
+ /**
+ * This method ensures the happens-before memory consistency effect between runs of the same {@link PeriodicTask}.
+ * <p>Each run, essentially, is executed at {@link java.util.concurrent.Executor#execute(Runnable)}. At the end of the
+ * execution, the executor schedules a new run. When the scheduled time comes, the next run is executed.
+ * There are the following happens-before relationships, <i>hb(prior_run, scheduler></i> and <i>hb(scheduler, next_run)</i>.
+ * Therefore, <i>hb(prior_run, next_run)</i>, i.e. the effects from prior_run are visible to next_run.
+ * <p>More on <a href="https://docs.oracle.com/javase/specs/jls/se11/html/jls-17.html#jls-17.4">Java Memory Model</a>
+ */
+ private void executeAndScheduleNext(PeriodicTaskKey key, long execCount)
+ {
+ long startTime = System.nanoTime();
+ internalPool.<Void>executeBlocking(promise -> executeInternal(promise, key, execCount), false)
+ .onComplete(ignored -> {
+ LOGGER.debug("Task run finishes. task='{}' execCount={}", key, execCount);
+ // schedule the next run iff the task is not killed
+ if (poisonPilledTasks.remove(key))
+ {
+ // timerId might get populated after unschedule due to race.
+ // Have another attempt to clean up here.
+ timerIds.remove(key);
+ LOGGER.debug("Avoid scheduling the next run, and remove it from poisonPilledTasks. task='{}' execCount={}",
+ key, execCount);
+ return;
+ }
+ long priorExecutionDurationMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime);
+ schedule(key, priorExecutionDurationMillis, key.task.delayMillis(), execCount + 1);
+ });
}
/**
- * Unschedules the {@code task} iff the task has been registered previously and returns the identifier
- * of the timer associated with this task, or {@code -1} if the task is not registered.
+ * Unschedule and close the {@link PeriodicTask} iff it has been scheduled.
*
* @param task the task to unschedule
- * @return the identifier of the timer associated with this task, or {@code -1} if the task is not registered
*/
- public long unschedule(PeriodicTask task)
+ public void unschedule(PeriodicTask task)
{
- Long timerId = timerIds.remove(new PeriodicTaskKey(task));
- if (timerId != null)
+ unschedule(task, true);
+ }
+
+ /**
+ * Unschedule the {@link PeriodicTask} iff it has been scheduled.
+ *
+ * @param task the {@link PeriodicTask} to unschedule
+ * @param shouldCloseTask indicate whether {@link PeriodicTask#close()} should be called
+ * @return a future of unschedule
+ */
+ private Future<Void> unschedule(PeriodicTask task, boolean shouldCloseTask)
+ {
+ PeriodicTaskKey key = new PeriodicTaskKey(task);
+ Long timerId = timerIds.remove(key);
+ if (timerId == null)
+ {
+ return Future.failedFuture("No such PeriodicTask: " + key);
+ }
+
+ LOGGER.debug("Unscheduling task. task='{}' timerId={}", key, timerId);
+ // always insert a poison pill when unscheduling an existing task,
+ // and conditionally remove when the timer can be cancelled
+ poisonPilledTasks.add(key);
+ // if timer is not started, it can be cancelled
+ if (timerId != PLACEHOLDER_TIMER_ID && internalPool.cancelTimer(timerId))
{
- internalPool.cancelTimer(timerId);
- task.close();
- return timerId;
+ poisonPilledTasks.remove(key);
}
- return -1L; // valid timer ids are non-negative
+
+ // The current run might have started already.
+ // If so, a non-null activeRun should be retrieved and a poison pill
+ // is placed to avoid further scheduling.
+ // Reschedule should only happen after the activeRun finishes.
+ Future<Void> unscheduleFuture = activeRuns.getOrDefault(key, Future.succeededFuture());
+ return shouldCloseTask
+ ? unscheduleFuture.onComplete(ignored -> task.close())
+ : unscheduleFuture;
}
+
/**
- * Reschedules the provided {@code task} and returns the identifier of the timer associated with the rescheduled
- * task.
+ * Reschedules the provided {@code task}.
+ * <p>The difference from {@link #unschedule(PeriodicTask)} then {@link #schedule(PeriodicTask)}
+ * is that the {@link PeriodicTask} in question is not closed when unscheduling
*
* @param task the task to reschedule
- * @return the identifier of the timer associated with the rescheduled task
*/
- public long reschedule(PeriodicTask task)
+ public void reschedule(PeriodicTask task)
{
- Long timerId = timerIds.remove(new PeriodicTaskKey(task));
- if (timerId != null)
- {
- internalPool.cancelTimer(timerId);
- }
- return schedule(task);
+ unschedule(task, false)
+ .onComplete(ignored -> schedule(task));
}
@Override
@@ -123,7 +219,6 @@ public class PeriodicTaskExecutor implements Closeable
timerIds.values().forEach(internalPool::cancelTimer);
timerIds.keySet().forEach(key -> key.task.close());
timerIds.clear();
- activeTasks.clear();
completion.complete();
}
catch (Throwable throwable)
@@ -132,76 +227,73 @@ public class PeriodicTaskExecutor implements Closeable
}
}
- private void executeInternal(PeriodicTaskKey key)
+ private void executeInternal(Promise<Void> promise, PeriodicTaskKey key, long execCount)
{
PeriodicTask periodicTask = key.task;
-
- switch (determineExecution(periodicTask))
+ switch (consolidateScheduleDecision(periodicTask))
{
- case SKIP_EXECUTION:
- LOGGER.trace("Skip executing task. task={}", periodicTask.name());
+ case SKIP:
+ LOGGER.trace("Skip executing task. task='{}' execCount={}", key, execCount);
+ promise.tryComplete();
return;
case EXECUTE:
break;
- case INDETERMINATE:
+ case RESCHEDULE:
default:
- LOGGER.debug("Unable to determine execution for this task, rescheduling. task={}", periodicTask.name());
- // When the process is unable to determine whether a task should be executed, we want
- // the task to be rescheduled for a shorter period of time. Assume you have a PeriodicTask that
- // runs every day. If it happens to run during a period of time when there's no
- // determination whether task should run or not, we do not want to wait another day for the
- // task to run, so instead we reschedule it and only wait the initial delay of the task for
- // the task to try again.
+ LOGGER.debug("Reschedule the task. task='{}' execCount={}", key, execCount);
reschedule(periodicTask);
+ promise.tryComplete();
return;
}
- if (!activeTasks.add(key))
- {
- LOGGER.debug("Task is already running. task={}", periodicTask.name());
- return;
- }
-
- Promise<Void> promise = Promise.promise();
-
+ // Leverage a separate promise, taskRunPromise, to ensure the activeRun removal
+ // happens before the completion of the promise of executeBlocking and its subsequent calls
+ Promise<Void> taskRunPromise = Promise.promise();
+ Future<Void> futureResult = taskRunPromise.future().onComplete(res -> {
+ LOGGER.debug("Removing task from active runs. task='{}' execCount={}", key, execCount);
+ activeRuns.remove(key);
+ // finally complete the promise of executeBlocking
+ promise.handle(res);
+ });
try
{
- periodicTask.execute(promise);
+ LOGGER.debug("Executing task. task='{}' execCount={}", key, execCount);
+ activeRuns.put(key, futureResult);
+ periodicTask.execute(taskRunPromise);
}
catch (Throwable throwable)
{
- LOGGER.warn("Periodic task failed to execute. task={}", periodicTask.name(), throwable);
- promise.tryFail(throwable);
+ LOGGER.warn("Periodic task failed to execute. task='{}' execCount={}", periodicTask.name(), execCount, throwable);
+ taskRunPromise.tryFail(throwable);
}
-
- promise.future().onComplete(res -> activeTasks.remove(key));
}
/**
- * Determines whether the task should run.
+ * Consolidate the {@link ScheduleDecision} from the {@link PeriodicTask} and {@link ClusterLease}
*
* @param periodicTask the task
- * @return the result of the determination
+ * @return schedule decision
*/
- protected ExecutionDetermination determineExecution(PeriodicTask periodicTask)
+ protected ScheduleDecision consolidateScheduleDecision(PeriodicTask periodicTask)
{
- if (periodicTask.shouldSkip())
+ ScheduleDecision decisionFromTask = periodicTask.scheduleDecision();
+ // Take the cluster lease ownership into consideration, if the periodic task
+ // decides to execute or reschedule.
+ // For example, if the local sidecar is not the cluster leaseholder, the lease
+ // ownership should override the decision.
+ if (decisionFromTask != ScheduleDecision.SKIP
+ && periodicTask instanceof ExecuteOnClusterLeaseholderOnly)
{
- return ExecutionDetermination.SKIP_EXECUTION;
+ return clusterLease.toScheduleDecision();
}
-
- if (periodicTask instanceof ExecuteOnClusterLeaseholderOnly)
- {
- return clusterLease.executionDetermination();
- }
- return ExecutionDetermination.EXECUTE;
+ return decisionFromTask;
}
// A simple wrapper that implements equals and hashcode,
// which is not necessary for the actual ExecutionLoops to implement
- private static class PeriodicTaskKey
+ static class PeriodicTaskKey
{
private final String fqcnAndName;
private final PeriodicTask task;
@@ -231,5 +323,23 @@ public class PeriodicTaskExecutor implements Closeable
return false;
}
+
+ @Override
+ public String toString()
+ {
+ return fqcnAndName;
+ }
+ }
+
+ @VisibleForTesting
+ Map<PeriodicTaskKey, Long> timerIds()
+ {
+ return timerIds;
+ }
+
+ @VisibleForTesting
+ Set<PeriodicTaskKey> poisonPilledTasks()
+ {
+ return poisonPilledTasks;
}
}
diff --git a/server/src/main/java/org/apache/cassandra/sidecar/tasks/ExecutionDetermination.java b/server/src/main/java/org/apache/cassandra/sidecar/tasks/ScheduleDecision.java
similarity index 68%
rename from server/src/main/java/org/apache/cassandra/sidecar/tasks/ExecutionDetermination.java
rename to server/src/main/java/org/apache/cassandra/sidecar/tasks/ScheduleDecision.java
index 72b83c63..55e23c8e 100644
--- a/server/src/main/java/org/apache/cassandra/sidecar/tasks/ExecutionDetermination.java
+++ b/server/src/main/java/org/apache/cassandra/sidecar/tasks/ScheduleDecision.java
@@ -19,28 +19,22 @@
package org.apache.cassandra.sidecar.tasks;
/**
- * Determines whether executions can proceed, be skipped, or the state is indeterminate
- * and an action can be taken based on the indeterminate state (i.e. rescheduling a {@link PeriodicTask}).
+ * Determines whether executions can proceed, be skipped, or the {@link PeriodicTask} should be rescheduled
*/
-public enum ExecutionDetermination
+public enum ScheduleDecision
{
/**
- * The execution can proceed.
+ * Execute the upcoming run
*/
EXECUTE,
/**
- * The execution will be skipped.
+ * Skip the upcoming run, but do not change the schedule
*/
- SKIP_EXECUTION,
+ SKIP,
/**
- * It is not possible to determine whether the execution should proceed or be skipped.
+ * Skip the upcoming run and reschedule the {@link PeriodicTask}
*/
- INDETERMINATE;
-
- public boolean shouldExecuteOnLocalSidecar()
- {
- return this == EXECUTE;
- }
+ RESCHEDULE;
}
diff --git a/server/src/main/java/org/apache/cassandra/sidecar/tasks/Task.java b/server/src/main/java/org/apache/cassandra/sidecar/tasks/Task.java
index d63d3bbd..fad1929a 100644
--- a/server/src/main/java/org/apache/cassandra/sidecar/tasks/Task.java
+++ b/server/src/main/java/org/apache/cassandra/sidecar/tasks/Task.java
@@ -22,8 +22,8 @@ import io.vertx.core.Promise;
import org.jetbrains.annotations.Nullable;
/**
- * * An interface that defines a task that will be executed during the lifecycle of Cassandra Sidecar
- * @param <T>
+ * An interface that defines a task that will be executed during the lifecycle of Cassandra Sidecar
+ * @param <T> result value type
*/
public interface Task<T>
{
diff --git a/server/src/test/integration/org/apache/cassandra/sidecar/coordination/ClusterLeaseClaimTaskIntegrationTest.java b/server/src/test/integration/org/apache/cassandra/sidecar/coordination/ClusterLeaseClaimTaskIntegrationTest.java
index 6f67424a..900d5b24 100644
--- a/server/src/test/integration/org/apache/cassandra/sidecar/coordination/ClusterLeaseClaimTaskIntegrationTest.java
+++ b/server/src/test/integration/org/apache/cassandra/sidecar/coordination/ClusterLeaseClaimTaskIntegrationTest.java
@@ -63,7 +63,7 @@ import org.apache.cassandra.sidecar.metrics.CoordinationMetrics;
import org.apache.cassandra.sidecar.metrics.MetricRegistryFactory;
import org.apache.cassandra.sidecar.metrics.SidecarMetrics;
import org.apache.cassandra.sidecar.metrics.SidecarMetricsImpl;
-import org.apache.cassandra.sidecar.tasks.ExecutionDetermination;
+import org.apache.cassandra.sidecar.tasks.ScheduleDecision;
import org.apache.cassandra.sidecar.testing.SharedExecutorNettyOptions;
import org.apache.cassandra.testing.TestVersion;
import org.jetbrains.annotations.NotNull;
@@ -138,7 +138,7 @@ class ClusterLeaseClaimTaskIntegrationTest
assertThat(simulatedInstances).as("There are no leaseholders when the process has not run yet")
.allMatch(e -> !e.clusterLease.isClaimedByLocalSidecar());
assertThat(simulatedInstances).as("And the state for all of them is indeterminate")
- .allMatch(e -> e.clusterLease.executionDetermination() == ExecutionDetermination.INDETERMINATE);
+ .allMatch(e -> e.clusterLease.toScheduleDecision() == ScheduleDecision.RESCHEDULE);
AtomicReference<Object[][]> currentLeaseholderQueryResult = new AtomicReference<>();
AtomicReference<TestInstanceWrapper> currentLeaseholder = new AtomicReference<>();
@@ -159,9 +159,9 @@ class ClusterLeaseClaimTaskIntegrationTest
currentLeaseholder.get().clusterLeaseClaimTask.resetLeaseholder();
assertThat(simulatedInstances).as("No instances are expected as we've just reset the existing leaseholder information")
.allMatch(e -> !e.clusterLease.isClaimedByLocalSidecar());
- assertThat(currentLeaseholder.get().clusterLease.executionDetermination())
+ assertThat(currentLeaseholder.get().clusterLease.toScheduleDecision())
.as("And the state for the current leaseholder is indeterminate")
- .isEqualTo(ExecutionDetermination.INDETERMINATE);
+ .isEqualTo(ScheduleDecision.RESCHEDULE);
loopAssert(3, () -> {
runLeaseAcquireProcess(pool, simulatedInstances);
@@ -243,13 +243,13 @@ class ClusterLeaseClaimTaskIntegrationTest
// then disable binary
simulateDisableBinaryOfLeaseholder(simulatedInstances);
- ExecutionDetermination executionDetermination = null;
+ ScheduleDecision scheduleDecision = null;
for (int i = 0; i < 20; i++)
{
leaseholder.clusterLeaseClaimTask.runClaimProcess();
- executionDetermination = leaseholder.clusterLease.executionDetermination();
+ scheduleDecision = leaseholder.clusterLease.toScheduleDecision();
- if (executionDetermination != ExecutionDetermination.INDETERMINATE)
+ if (scheduleDecision != ScheduleDecision.RESCHEDULE)
{
int ttlSeconds = Math.max(1, maybeDetermineTTL(cluster));
LOGGER.info("TTL is {} seconds", ttlSeconds);
@@ -260,8 +260,8 @@ class ClusterLeaseClaimTaskIntegrationTest
}
else break;
}
- assertThat(executionDetermination).as("The leaseholder should give up the lease")
- .isEqualTo(ExecutionDetermination.INDETERMINATE);
+ assertThat(scheduleDecision).as("The leaseholder should give up the lease")
+ .isEqualTo(ScheduleDecision.RESCHEDULE);
// ensure the data is TTL'd in the database
for (int i = 0; i < 20; i++)
{
diff --git a/server/src/test/integration/org/apache/cassandra/sidecar/testing/IntegrationTestModule.java b/server/src/test/integration/org/apache/cassandra/sidecar/testing/IntegrationTestModule.java
index b9746163..a2fc3846 100644
--- a/server/src/test/integration/org/apache/cassandra/sidecar/testing/IntegrationTestModule.java
+++ b/server/src/test/integration/org/apache/cassandra/sidecar/testing/IntegrationTestModule.java
@@ -49,7 +49,6 @@ import org.apache.cassandra.sidecar.config.yaml.SslConfigurationImpl;
import org.apache.cassandra.sidecar.config.yaml.TestServiceConfiguration;
import org.apache.cassandra.sidecar.coordination.ClusterLease;
import org.apache.cassandra.sidecar.exceptions.NoSuchSidecarInstanceException;
-import org.apache.cassandra.sidecar.tasks.ExecutionDetermination;
import org.jetbrains.annotations.NotNull;
import static org.apache.cassandra.sidecar.server.SidecarServerEvents.ON_SERVER_STOP;
@@ -152,7 +151,7 @@ public class IntegrationTestModule extends AbstractModule
@Singleton
public ClusterLease clusterLease()
{
- return new ClusterLease(ExecutionDetermination.EXECUTE);
+ return new ClusterLease(ClusterLease.Ownership.CLAIMED);
}
private AccessControlConfiguration accessControlConfiguration()
diff --git a/server/src/test/java/org/apache/cassandra/sidecar/CassandraSidecarDaemonTest.java b/server/src/test/java/org/apache/cassandra/sidecar/CassandraSidecarDaemonTest.java
index bad370ae..6a16f7ec 100644
--- a/server/src/test/java/org/apache/cassandra/sidecar/CassandraSidecarDaemonTest.java
+++ b/server/src/test/java/org/apache/cassandra/sidecar/CassandraSidecarDaemonTest.java
@@ -29,6 +29,9 @@ import java.util.concurrent.TimeUnit;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import io.vertx.core.Vertx;
import io.vertx.ext.web.client.HttpResponse;
import io.vertx.ext.web.client.WebClient;
@@ -44,6 +47,8 @@ import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException
*/
class CassandraSidecarDaemonTest
{
+ private static final Logger LOGGER = LoggerFactory.getLogger(CassandraSidecarDaemonTest.class);
+
static final String[] NO_ARGS = {};
@BeforeEach
@@ -130,6 +135,7 @@ class CassandraSidecarDaemonTest
}
finally
{
+ LOGGER.debug("Tearing down");
maybeStopCassandraSidecar();
Files.deleteIfExists(targetFile);
diff --git a/server/src/test/java/org/apache/cassandra/sidecar/coordination/ClusterLeaseClaimTaskTest.java b/server/src/test/java/org/apache/cassandra/sidecar/coordination/ClusterLeaseClaimTaskTest.java
index dfa46aae..929a8607 100644
--- a/server/src/test/java/org/apache/cassandra/sidecar/coordination/ClusterLeaseClaimTaskTest.java
+++ b/server/src/test/java/org/apache/cassandra/sidecar/coordination/ClusterLeaseClaimTaskTest.java
@@ -30,7 +30,7 @@ import io.vertx.core.Vertx;
import org.apache.cassandra.sidecar.config.ServiceConfiguration;
import org.apache.cassandra.sidecar.db.SidecarLeaseDatabaseAccessor;
import org.apache.cassandra.sidecar.metrics.SidecarMetrics;
-import org.apache.cassandra.sidecar.tasks.ExecutionDetermination;
+import org.apache.cassandra.sidecar.tasks.ScheduleDecision;
import static org.apache.cassandra.sidecar.coordination.ClusterLeaseClaimTask.MINIMUM_DELAY_MILLIS;
import static org.assertj.core.api.Assertions.assertThat;
@@ -55,7 +55,7 @@ class ClusterLeaseClaimTaskTest
ClusterLeaseClaimTask task = new ClusterLeaseClaimTask(mock(Vertx.class), serviceConfiguration, mockElectorateMembership,
mock(SidecarLeaseDatabaseAccessor.class), new ClusterLease(),
mock(SidecarMetrics.class, RETURNS_DEEP_STUBS));
- assertThat(task.shouldSkip()).isTrue();
+ assertThat(task.scheduleDecision()).isEqualTo(ScheduleDecision.SKIP);
// avoid expensive calls
verifyNoInteractions(mockElectorateMembership);
}
@@ -71,9 +71,9 @@ class ClusterLeaseClaimTaskTest
mock(SidecarLeaseDatabaseAccessor.class), clusterLease,
mock(SidecarMetrics.class, RETURNS_DEEP_STUBS));
- assertThat(task.shouldSkip()).isTrue();
- assertThat(clusterLease.executionDetermination()).as("Skip when not a member of the electorate")
- .isEqualTo(ExecutionDetermination.SKIP_EXECUTION);
+ assertThat(task.scheduleDecision()).isEqualTo(ScheduleDecision.SKIP);
+ assertThat(clusterLease.toScheduleDecision()).as("Skip when not a member of the electorate")
+ .isEqualTo(ScheduleDecision.SKIP);
verify(mockElectorateMembership, times(1)).isMember();
}
@@ -87,7 +87,7 @@ class ClusterLeaseClaimTaskTest
mock(SidecarLeaseDatabaseAccessor.class), new ClusterLease(),
mock(SidecarMetrics.class, RETURNS_DEEP_STUBS));
- assertThat(task.shouldSkip()).isFalse();
+ assertThat(task.scheduleDecision()).isEqualTo(ScheduleDecision.EXECUTE);
verify(mockElectorateMembership, times(1)).isMember();
}
diff --git a/server/src/test/java/org/apache/cassandra/sidecar/db/SidecarSchemaTest.java b/server/src/test/java/org/apache/cassandra/sidecar/db/SidecarSchemaTest.java
index 34873154..f44d4f17 100644
--- a/server/src/test/java/org/apache/cassandra/sidecar/db/SidecarSchemaTest.java
+++ b/server/src/test/java/org/apache/cassandra/sidecar/db/SidecarSchemaTest.java
@@ -56,7 +56,6 @@ import org.apache.cassandra.sidecar.coordination.ClusterLease;
import org.apache.cassandra.sidecar.db.schema.SidecarSchema;
import org.apache.cassandra.sidecar.server.MainModule;
import org.apache.cassandra.sidecar.server.Server;
-import org.apache.cassandra.sidecar.tasks.ExecutionDetermination;
import org.mockito.stubbing.Answer;
import static org.assertj.core.api.Assertions.assertThat;
@@ -269,7 +268,7 @@ public class SidecarSchemaTest
@Singleton
public ClusterLease clusterLease()
{
- return new ClusterLease(ExecutionDetermination.EXECUTE);
+ return new ClusterLease(ClusterLease.Ownership.CLAIMED);
}
}
}
diff --git a/server/src/test/java/org/apache/cassandra/sidecar/metrics/SchemaMetricsTest.java b/server/src/test/java/org/apache/cassandra/sidecar/metrics/SchemaMetricsTest.java
index 3fa67d0b..de258d34 100644
--- a/server/src/test/java/org/apache/cassandra/sidecar/metrics/SchemaMetricsTest.java
+++ b/server/src/test/java/org/apache/cassandra/sidecar/metrics/SchemaMetricsTest.java
@@ -47,7 +47,6 @@ import org.apache.cassandra.sidecar.db.schema.SidecarSchema;
import org.apache.cassandra.sidecar.exceptions.SidecarSchemaModificationException;
import org.apache.cassandra.sidecar.server.MainModule;
import org.apache.cassandra.sidecar.server.Server;
-import org.apache.cassandra.sidecar.tasks.ExecutionDetermination;
import static org.apache.cassandra.sidecar.AssertionUtils.loopAssert;
import static org.apache.cassandra.sidecar.utils.TestMetricUtils.registry;
@@ -143,7 +142,7 @@ class SchemaMetricsTest
@Singleton
public ClusterLease clusterLease()
{
- return new ClusterLease(ExecutionDetermination.EXECUTE);
+ return new ClusterLease(ClusterLease.Ownership.CLAIMED);
}
}
}
diff --git a/server/src/test/java/org/apache/cassandra/sidecar/restore/RestoreJobDiscovererTest.java b/server/src/test/java/org/apache/cassandra/sidecar/restore/RestoreJobDiscovererTest.java
index 4cd6e529..fb6dba17 100644
--- a/server/src/test/java/org/apache/cassandra/sidecar/restore/RestoreJobDiscovererTest.java
+++ b/server/src/test/java/org/apache/cassandra/sidecar/restore/RestoreJobDiscovererTest.java
@@ -48,6 +48,7 @@ import org.apache.cassandra.sidecar.metrics.MetricRegistryFactory;
import org.apache.cassandra.sidecar.metrics.SidecarMetrics;
import org.apache.cassandra.sidecar.metrics.SidecarMetricsImpl;
import org.apache.cassandra.sidecar.tasks.PeriodicTaskExecutor;
+import org.apache.cassandra.sidecar.tasks.ScheduleDecision;
import org.apache.cassandra.sidecar.utils.InstanceMetadataFetcher;
import org.mockito.ArgumentCaptor;
@@ -197,7 +198,7 @@ class RestoreJobDiscovererTest
// Execution 3
// shouldSkip always returns false
- assertThat(loop.shouldSkip()).isFalse();
+ assertThat(loop.scheduleDecision()).isEqualTo(ScheduleDecision.EXECUTE);
// Execution 4
UUID newJobId2 = UUIDs.timeBased();
@@ -230,7 +231,7 @@ class RestoreJobDiscovererTest
void testSkipExecuteWhenSidecarSchemaIsNotInitialized()
{
when(sidecarSchema.isInitialized()).thenReturn(false);
- assertThat(loop.shouldSkip()).isTrue();
+ assertThat(loop.scheduleDecision()).isEqualTo(ScheduleDecision.SKIP);
}
@Test
diff --git a/server/src/test/java/org/apache/cassandra/sidecar/restore/RestoreProcessorTest.java b/server/src/test/java/org/apache/cassandra/sidecar/restore/RestoreProcessorTest.java
index a3aa1a6a..fa6f0ae6 100644
--- a/server/src/test/java/org/apache/cassandra/sidecar/restore/RestoreProcessorTest.java
+++ b/server/src/test/java/org/apache/cassandra/sidecar/restore/RestoreProcessorTest.java
@@ -44,6 +44,7 @@ import org.apache.cassandra.sidecar.metrics.instance.InstanceMetricsImpl;
import org.apache.cassandra.sidecar.metrics.instance.InstanceRestoreMetrics;
import org.apache.cassandra.sidecar.server.MainModule;
import org.apache.cassandra.sidecar.tasks.PeriodicTaskExecutor;
+import org.apache.cassandra.sidecar.tasks.ScheduleDecision;
import org.mockito.Mockito;
import static org.apache.cassandra.sidecar.AssertionUtils.loopAssert;
@@ -147,7 +148,7 @@ class RestoreProcessorTest
{
when(sidecarSchema.isInitialized()).thenReturn(false);
- assertThat(processor.shouldSkip()).isTrue();
+ assertThat(processor.scheduleDecision()).isEqualTo(ScheduleDecision.SKIP);
assertThat(processor.activeSlices()).isZero();
CountDownLatch latch = new CountDownLatch(1);
diff --git a/server/src/test/java/org/apache/cassandra/sidecar/tasks/PeriodicTaskExecutorTest.java b/server/src/test/java/org/apache/cassandra/sidecar/tasks/PeriodicTaskExecutorTest.java
index 0eb18e56..fa633897 100644
--- a/server/src/test/java/org/apache/cassandra/sidecar/tasks/PeriodicTaskExecutorTest.java
+++ b/server/src/test/java/org/apache/cassandra/sidecar/tasks/PeriodicTaskExecutorTest.java
@@ -18,6 +18,10 @@
package org.apache.cassandra.sidecar.tasks;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -25,6 +29,8 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import com.google.common.util.concurrent.Uninterruptibles;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import io.vertx.core.Promise;
@@ -34,6 +40,7 @@ import org.apache.cassandra.sidecar.config.yaml.ServiceConfigurationImpl;
import org.apache.cassandra.sidecar.coordination.ClusterLease;
import org.apache.cassandra.sidecar.coordination.ExecuteOnClusterLeaseholderOnly;
+import static org.apache.cassandra.sidecar.AssertionUtils.loopAssert;
import static org.assertj.core.api.Assertions.assertThat;
/**
@@ -41,14 +48,30 @@ import static org.assertj.core.api.Assertions.assertThat;
*/
class PeriodicTaskExecutorTest
{
- Vertx vertx = Vertx.vertx();
- ExecutorPools executorPools = new ExecutorPools(vertx, new ServiceConfigurationImpl());
+ private final Vertx vertx = Vertx.vertx();
+ private final ExecutorPools executorPools = new ExecutorPools(vertx, new ServiceConfigurationImpl());
+ private final ClusterLease clusterLease = new ClusterLease();
+ private PeriodicTaskExecutor taskExecutor;
+
+ @BeforeEach
+ void beforeEach()
+ {
+ taskExecutor = new PeriodicTaskExecutor(executorPools, clusterLease);
+ }
+
+ @AfterEach
+ void afterEach()
+ {
+ clusterLease.setOwnershipTesting(ClusterLease.Ownership.INDETERMINATE);
+ if (taskExecutor != null)
+ {
+ taskExecutor.close(Promise.promise());
+ }
+ }
@Test
void testLoopFailure()
{
- PeriodicTaskExecutor taskExecutor = new PeriodicTaskExecutor(executorPools);
-
int totalFailures = 5;
AtomicInteger failuresCount = new AtomicInteger(0);
CountDownLatch closeLatch = new CountDownLatch(1);
@@ -58,7 +81,7 @@ class PeriodicTaskExecutorTest
@Override
public long delay()
{
- return 20;
+ return 1;
}
@Override
@@ -84,33 +107,163 @@ class PeriodicTaskExecutorTest
}
@Test
- void testPeriodicTaskOnNonExecutor()
+ void testPeriodicTaskExecutionShouldEnsureMemoryConsistency()
{
- CountDownLatch latch = new CountDownLatch(1);
- SimulatedTask taskThatRunsOnNonExecutor = new SimulatedTask(latch);
- PeriodicTaskExecutor taskExecutorNeverExecute = new PeriodicTaskExecutor(executorPools, new ClusterLease(ExecutionDetermination.SKIP_EXECUTION));
+ // starts 10 concurrent incremental tasks, and
+ // assert that for each task, the non-thread-safe value update has
+ // the same effect of the thread-safe value update
+ List<IncrementPeriodicTask> tasks = new ArrayList<>(10);
+ for (int i = 0; i < 10; i++)
+ {
+ IncrementPeriodicTask incTask = new IncrementPeriodicTask("task" + i);
+ tasks.add(incTask);
+ taskExecutor.schedule(incTask);
+ }
+ Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
+ tasks.forEach(taskExecutor::unschedule);
+ // wait until unschedule is complete
+ loopAssert(1, () -> assertThat(taskExecutor.timerIds()).isEmpty());
+ loopAssert(1, () -> assertThat(taskExecutor.poisonPilledTasks()).isEmpty());
+ tasks.forEach(incTask -> assertThat(incTask.atomicValue.get())
+ .describedAs(incTask.name() + " should have same value")
+ .isEqualTo(incTask.value)
+ .isPositive());
+ }
- taskExecutorNeverExecute.schedule(taskThatRunsOnNonExecutor);
- assertThat(Uninterruptibles.awaitUninterruptibly(latch, 30, TimeUnit.SECONDS)).isTrue();
- assertThat(taskThatRunsOnNonExecutor.executionCount.get()).isEqualTo(0);
- assertThat(taskThatRunsOnNonExecutor.initialDelayCount.get()).isEqualTo(1);
+ @Test
+ void testScheduleSameTaskMultipleTimes()
+ {
+ AtomicInteger counter = new AtomicInteger(0);
+ // each time, the task is executed, the counter is incremented by 1.
+ // The task has the initial delay of 0, but the periodic interval is 1 day to ensure it is executed only once during the test runtime
+ PeriodicTask task = createSimplePeriodicTask("task", 0, TimeUnit.DAYS.toMillis(1), counter::incrementAndGet);
+ // schedule the same task for 10 times, only the first schedule goes through
+ for (int i = 0; i < 10; i++)
+ {
+ taskExecutor.schedule(task);
+ }
+ // in case that the test environment gets slow and the counter update is delayed
+ loopAssert(1, () -> assertThat(counter.get()).isEqualTo(1));
+ }
+
+ @Test
+ void testUnscheduleShouldStopExecution()
+ {
+ // the first run starts immediately, and the subsequent delay is 1 millis
+ testUnscheduleShouldStopExecution("task1", 0, 1);
+ // the first run starts immediately, and the subsequent runs also starts immediately
+ testUnscheduleShouldStopExecution("task2", 0, 0);
+ // tasks are scheduled with delay 1 millis
+ testUnscheduleShouldStopExecution("task3", 1, 1);
+ }
+
+ @Test
+ void testUnscheduleNonExistTaskHasNoEffect()
+ {
+ PeriodicTask notScheduled = createSimplePeriodicTask("simple task", 1, () -> {});
+ taskExecutor.unschedule(notScheduled);
+ assertThat(taskExecutor.poisonPilledTasks()).isEmpty();
+ assertThat(taskExecutor.timerIds()).isEmpty();
+ }
+
+ @Test
+ void testRescheduleNonExistTaskShouldNotClose()
+ {
+ AtomicBoolean isCloseCalled = new AtomicBoolean(false);
+ CountDownLatch taskScheduled = new CountDownLatch(1);
+ PeriodicTask task = new PeriodicTask()
+ {
+ @Override
+ public long delay()
+ {
+ return 1;
+ }
+
+ @Override
+ public void execute(Promise<Void> promise)
+ {
+ taskScheduled.countDown();
+ promise.complete();
+ }
+
+ @Override
+ public void close()
+ {
+ isCloseCalled.set(true);
+ }
+ };
+
+ // for such unscheduled task, reschedule has the same effect as schedule.
+ taskExecutor.reschedule(task);
+ Uninterruptibles.awaitUninterruptibly(taskScheduled);
+ assertThat(isCloseCalled.get())
+ .describedAs("When rescheduling an unscheduled task, the close method of the task should not be called")
+ .isFalse();
}
@Test
- void testPeriodicTaskOnExecutor()
+ void testReschedule()
+ {
+ AtomicInteger counter1 = new AtomicInteger(0);
+ AtomicInteger counter2 = new AtomicInteger(0);
+ int totalSamples = 10;
+ Set<Integer> counterValues = ConcurrentHashMap.newKeySet(totalSamples);
+
+ CountDownLatch latch1 = new CountDownLatch(1);
+ // The periodic task increment counter1 initially, then switches to increment counter 2 once it is rescheduled
+ PeriodicTask task = createSimplePeriodicTask("simple periodic task", 1, () -> {
+ if (latch1.getCount() > 0)
+ {
+ counter1.incrementAndGet();
+ }
+ else
+ {
+ counterValues.add(counter2.incrementAndGet());
+ }
+ Uninterruptibles.awaitUninterruptibly(latch1);
+ });
+ taskExecutor.schedule(task);
+ loopAssert(1, () -> assertThat(counter1.get()).isEqualTo(1));
+ assertThat(counterValues).isEmpty();
+ taskExecutor.reschedule(task);
+ assertThat(counter1.get()).isEqualTo(1);
+ assertThat(counterValues).isEmpty();
+ latch1.countDown();
+ // After getting rescheduled, it should produce unique values from counter2 and store in counterValues
+ assertThat(counter1.get()).isEqualTo(1);
+ loopAssert(2, () -> assertThat(counterValues.size()).isGreaterThanOrEqualTo(totalSamples));
+ taskExecutor.unschedule(task);
+ }
+
+ @Test
+ void testLeaseholderOnlyTaskOnNonClusterLeaseholder()
+ {
+ CountDownLatch skipLatch = new CountDownLatch(1);
+ SimulatedTask task = new SimulatedTask(skipLatch, null);
+ clusterLease.setOwnershipTesting(ClusterLease.Ownership.LOST);
+
+ taskExecutor.schedule(task);
+ assertThat(Uninterruptibles.awaitUninterruptibly(skipLatch, 30, TimeUnit.SECONDS)).isTrue();
+ assertThat(task.shouldSkipCount.get()).isEqualTo(5);
+ assertThat(task.executionCount.get()).isEqualTo(0);
+ assertThat(task.initialDelayCount.get()).isEqualTo(1);
+ }
+
+ @Test
+ void testLeaseholderOnlyTaskOnClusterLeaseholder()
{
CountDownLatch latch = new CountDownLatch(1);
- SimulatedTask taskThatRunsOnExecutor = new SimulatedTask(latch);
- PeriodicTaskExecutor taskExecutorAlwaysExecute = new PeriodicTaskExecutor(executorPools, new ClusterLease(ExecutionDetermination.EXECUTE));
+ SimulatedTask task = new SimulatedTask(latch);
+ clusterLease.setOwnershipTesting(ClusterLease.Ownership.CLAIMED);
- taskExecutorAlwaysExecute.schedule(taskThatRunsOnExecutor);
+ taskExecutor.schedule(task);
assertThat(Uninterruptibles.awaitUninterruptibly(latch, 30, TimeUnit.SECONDS)).isTrue();
- assertThat(taskThatRunsOnExecutor.executionCount.get()).isEqualTo(5);
- assertThat(taskThatRunsOnExecutor.initialDelayCount.get()).isEqualTo(1);
+ assertThat(task.executionCount.get()).isEqualTo(5);
+ assertThat(task.initialDelayCount.get()).isEqualTo(1);
}
@Test
- void testPeriodicTaskIsRescheduledWhenIndeterminate()
+ void testLeaseholderOnlyTaskIsRescheduledWhenIndeterminate()
{
CountDownLatch latch = new CountDownLatch(1);
SimulatedTask taskThatRunsOnExecutor = new SimulatedTask(latch);
@@ -124,6 +277,26 @@ class PeriodicTaskExecutorTest
assertThat(taskThatRunsOnExecutor.initialDelayCount.get()).isEqualTo(2);
// and we only actually execute 4 times, since the first time was rescheduled
assertThat(taskThatRunsOnExecutor.executionCount.get()).isEqualTo(4);
+ taskExecutor.close(Promise.promise());
+ }
+
+ private void testUnscheduleShouldStopExecution(String taskName, long taskInitialDelay, long taskDelayMillis)
+ {
+ AtomicInteger counter = new AtomicInteger(0);
+ CountDownLatch testFinish = new CountDownLatch(1);
+ PeriodicTask task = createSimplePeriodicTask(taskName, taskInitialDelay, taskDelayMillis, () -> {
+ counter.incrementAndGet();
+ testFinish.countDown();
+ });
+ taskExecutor.schedule(task);
+ Uninterruptibles.awaitUninterruptibly(testFinish);
+ taskExecutor.unschedule(task);
+ loopAssert(1,
+ () -> assertThat(taskExecutor.poisonPilledTasks()).isEmpty());
+ loopAssert(1,
+ () -> assertThat(taskExecutor.timerIds())
+ .describedAs("Execution should stop after unschedule is called")
+ .isEmpty());
}
static class TestClusterLease extends ClusterLease
@@ -136,14 +309,14 @@ class PeriodicTaskExecutorTest
}
@Override
- public ExecutionDetermination executionDetermination()
+ public ScheduleDecision toScheduleDecision()
{
- ExecutionDetermination executionDetermination = delegate.get().executionDetermination();
- if (executionDetermination == ExecutionDetermination.INDETERMINATE)
+ ScheduleDecision scheduleDecision = delegate.get().toScheduleDecision();
+ if (scheduleDecision == ScheduleDecision.RESCHEDULE)
{
- delegate.set(new ClusterLease(ExecutionDetermination.EXECUTE));
+ delegate.set(new ClusterLease(Ownership.CLAIMED));
}
- return executionDetermination;
+ return scheduleDecision;
}
@Override
@@ -158,11 +331,25 @@ class PeriodicTaskExecutorTest
final AtomicInteger executionCount = new AtomicInteger(0);
final AtomicInteger shouldSkipCount = new AtomicInteger(0);
final AtomicInteger initialDelayCount = new AtomicInteger(0);
- private final CountDownLatch latch;
+ private final CountDownLatch shouldSkipLatch;
+ private final CountDownLatch executeLatch;
+ private PeriodicTaskExecutor executor;
+
+ SimulatedTask(CountDownLatch executeLatch)
+ {
+ this(new CountDownLatch(1), executeLatch);
+ }
+
+ SimulatedTask(CountDownLatch shouldSkipLatch, CountDownLatch executeLatch)
+ {
+ this.shouldSkipLatch = shouldSkipLatch;
+ this.executeLatch = executeLatch;
+ }
- SimulatedTask(CountDownLatch latch)
+ @Override
+ public void registerPeriodicTaskExecutor(PeriodicTaskExecutor executor)
{
- this.latch = latch;
+ this.executor = executor;
}
@Override
@@ -179,20 +366,100 @@ class PeriodicTaskExecutorTest
}
@Override
- public boolean shouldSkip()
+ public ScheduleDecision scheduleDecision()
{
if (shouldSkipCount.incrementAndGet() == 5)
{
- latch.countDown();
+ shouldSkipLatch.countDown();
+ if (executeLatch == null)
+ {
+ // unschedule to avoid flakiness
+ executor.unschedule(this);
+ }
}
- return false;
+ return ScheduleDecision.EXECUTE;
}
@Override
public void execute(Promise<Void> promise)
{
executionCount.incrementAndGet();
+ if (shouldSkipCount.get() == 5)
+ {
+ executeLatch.countDown();
+ // unschedule to avoid flakiness
+ executor.unschedule(this);
+ }
promise.complete();
}
}
+
+ // a stateful periodic task that increments its value every run
+ private static class IncrementPeriodicTask implements PeriodicTask
+ {
+ private final String name;
+ int value = 0;
+ AtomicInteger atomicValue = new AtomicInteger(0);
+
+ IncrementPeriodicTask(String name)
+ {
+ this.name = name;
+ }
+
+ @Override
+ public String name()
+ {
+ return name;
+ }
+
+ @Override
+ public long delay()
+ {
+ return 1;
+ }
+
+ @Override
+ public void execute(Promise<Void> promise)
+ {
+ value += 1;
+ atomicValue.incrementAndGet();
+ promise.complete();
+ }
+ }
+
+ static PeriodicTask createSimplePeriodicTask(String name, long delayMillis, Runnable taskBody)
+ {
+ return createSimplePeriodicTask(name, delayMillis, delayMillis, taskBody);
+ }
+
+ static PeriodicTask createSimplePeriodicTask(String name, long initialDelayMillis, long delayMillis, Runnable taskBody)
+ {
+ return new PeriodicTask()
+ {
+ @Override
+ public String name()
+ {
+ return name;
+ }
+
+ @Override
+ public long initialDelay()
+ {
+ return initialDelayMillis;
+ }
+
+ @Override
+ public long delay()
+ {
+ return delayMillis;
+ }
+
+ @Override
+ public void execute(Promise<Void> promise)
+ {
+ taskBody.run();
+ promise.complete();
+ }
+ };
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org