You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by yc...@apache.org on 2025/01/03 20:43:22 UTC

(cassandra-sidecar) branch trunk updated: CASSSIDECAR-181: Ensure memory consistency from PeriodicTask executio… (#167)

This is an automated email from the ASF dual-hosted git repository.

ycai pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra-sidecar.git


The following commit(s) were added to refs/heads/trunk by this push:
     new 58a6bfe1 CASSSIDECAR-181: Ensure memory consistency from PeriodicTask executio… (#167)
58a6bfe1 is described below

commit 58a6bfe140021d21283e45d95ab50c8c132968ed
Author: Yifan Cai <yc...@apache.org>
AuthorDate: Fri Jan 3 12:43:17 2025 -0800

    CASSSIDECAR-181: Ensure memory consistency from PeriodicTask executio… (#167)
    
    Patch by Yifan Cai; Reviewed by Francisco Guerrero for CASSSIDECAR-181
---
 CHANGES.txt                                        |   1 +
 checkstyle.xml                                     |   1 +
 server/build.gradle                                |   2 +
 .../sidecar/concurrent/TaskExecutorPool.java       |   5 +-
 .../sidecar/coordination/ClusterLease.java         |  69 ++++-
 .../coordination/ClusterLeaseClaimTask.java        |  22 +-
 .../cassandra/sidecar/job/OperationalJob.java      |   3 +-
 .../sidecar/restore/RestoreJobDiscoverer.java      |  36 ++-
 .../sidecar/restore/RestoreProcessor.java          |   6 +-
 .../sidecar/restore/RingTopologyRefresher.java     |   5 +-
 .../sidecar/tasks/HealthCheckPeriodicTask.java     |  44 +--
 .../sidecar/tasks/KeyStoreCheckPeriodicTask.java   |  22 +-
 .../cassandra/sidecar/tasks/PeriodicTask.java      |  23 +-
 .../sidecar/tasks/PeriodicTaskExecutor.java        | 258 +++++++++++-----
 ...ionDetermination.java => ScheduleDecision.java} |  20 +-
 .../org/apache/cassandra/sidecar/tasks/Task.java   |   4 +-
 .../ClusterLeaseClaimTaskIntegrationTest.java      |  18 +-
 .../sidecar/testing/IntegrationTestModule.java     |   3 +-
 .../sidecar/CassandraSidecarDaemonTest.java        |   6 +
 .../coordination/ClusterLeaseClaimTaskTest.java    |  12 +-
 .../cassandra/sidecar/db/SidecarSchemaTest.java    |   3 +-
 .../sidecar/metrics/SchemaMetricsTest.java         |   3 +-
 .../sidecar/restore/RestoreJobDiscovererTest.java  |   5 +-
 .../sidecar/restore/RestoreProcessorTest.java      |   3 +-
 .../sidecar/tasks/PeriodicTaskExecutorTest.java    | 329 +++++++++++++++++++--
 25 files changed, 681 insertions(+), 222 deletions(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 057500b8..ffbd3cc6 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,6 @@
 1.0.0
 -----
+ * Ensure memory consistency from PeriodicTask executions and expose richer ScheduleDecision (CASSSIDECAR-181)
  * Refactor access to delegate methods to simplify (CASSSIDECAR-182)
  * Renaming InstancesConfig to InstancesMetadata (CASSSIDECAR-175)
  * Mechanism to have a reduced number of Sidecar instances run operations (CASSSIDECAR-174)
diff --git a/checkstyle.xml b/checkstyle.xml
index 2b4b36e6..0b19a07e 100644
--- a/checkstyle.xml
+++ b/checkstyle.xml
@@ -329,6 +329,7 @@ page at http://checkstyle.sourceforge.net/config.html -->
         SL, SLIST, SL_ASSIGN, SR_ASSIGN, STAR, STAR_ASSIGN" />
             <property name="allowEmptyConstructors" value="true" />
             <property name="allowEmptyMethods" value="true" />
+            <property name="allowEmptyLambdas" value="true" />
             <property name="severity" value="error" />
         </module>
 
diff --git a/server/build.gradle b/server/build.gradle
index 565b30a5..d74f55df 100644
--- a/server/build.gradle
+++ b/server/build.gradle
@@ -191,6 +191,8 @@ publishing {
 test {
     systemProperty "vertxweb.environment", "dev"
     systemProperty "vertx.logger-delegate-factory-class-name", "io.vertx.core.logging.SLF4JLogDelegateFactory"
+    // There is no native lib (JNR) for getting time for testing
+    systemProperty "com.datastax.driver.USE_NATIVE_CLOCK", "false"
     // ordinarily we don't need integration tests
     // see the integrationTest task
     useJUnitPlatform()
diff --git a/server/src/main/java/org/apache/cassandra/sidecar/concurrent/TaskExecutorPool.java b/server/src/main/java/org/apache/cassandra/sidecar/concurrent/TaskExecutorPool.java
index 53292ef4..3e58dc7d 100644
--- a/server/src/main/java/org/apache/cassandra/sidecar/concurrent/TaskExecutorPool.java
+++ b/server/src/main/java/org/apache/cassandra/sidecar/concurrent/TaskExecutorPool.java
@@ -167,11 +167,10 @@ public abstract class TaskExecutorPool implements WorkerExecutor
     }
 
     @Override
-    public <T> Future<T> executeBlocking(Handler<Promise<T>> blockingCodeHandler,
-                                         boolean ordered)
+    public <T> Future<T> executeBlocking(Handler<Promise<T>> blockingCodeHandler, boolean ordered)
     {
         // TODO: migrate to org.apache.cassandra.sidecar.concurrent.TaskExecutorPool.executeBlocking(java.util.concurrent.Callable<T>, boolean)
-        return StopWatch.measureTimeTaken(workerExecutor.executeBlocking(blockingCodeHandler), this::recordTimeTaken);
+        return StopWatch.measureTimeTaken(workerExecutor.executeBlocking(blockingCodeHandler, ordered), this::recordTimeTaken);
     }
 
     @Override
diff --git a/server/src/main/java/org/apache/cassandra/sidecar/coordination/ClusterLease.java b/server/src/main/java/org/apache/cassandra/sidecar/coordination/ClusterLease.java
index 2f1c02d2..6b1c1909 100644
--- a/server/src/main/java/org/apache/cassandra/sidecar/coordination/ClusterLease.java
+++ b/server/src/main/java/org/apache/cassandra/sidecar/coordination/ClusterLease.java
@@ -21,7 +21,8 @@ package org.apache.cassandra.sidecar.coordination;
 import java.util.Objects;
 
 import com.google.inject.Singleton;
-import org.apache.cassandra.sidecar.tasks.ExecutionDetermination;
+import org.apache.cassandra.sidecar.tasks.ScheduleDecision;
+import org.jetbrains.annotations.VisibleForTesting;
 
 /**
  * Holds information about whether this Sidecar instance has claimed the lease, it has not, or it's unable
@@ -30,23 +31,24 @@ import org.apache.cassandra.sidecar.tasks.ExecutionDetermination;
 @Singleton
 public class ClusterLease
 {
-    private volatile ExecutionDetermination executionDetermination = ExecutionDetermination.INDETERMINATE;
+    private volatile Ownership leaseOwnership;
 
     public ClusterLease()
     {
+        this(Ownership.INDETERMINATE);
     }
 
-    public ClusterLease(ExecutionDetermination executionDetermination)
+    public ClusterLease(Ownership leaseOwnership)
     {
-        this.executionDetermination = executionDetermination;
+        this.leaseOwnership = leaseOwnership;
     }
 
     /**
      * @return the current execution determination
      */
-    public ExecutionDetermination executionDetermination()
+    public ScheduleDecision toScheduleDecision()
     {
-        return executionDetermination;
+        return leaseOwnership.toScheduleDecision();
     }
 
     /**
@@ -54,16 +56,61 @@ public class ClusterLease
      */
     public boolean isClaimedByLocalSidecar()
     {
-        return executionDetermination.shouldExecuteOnLocalSidecar();
+        return leaseOwnership == Ownership.CLAIMED;
     }
 
     /**
-     * Updates the determination
+     * Updates the ownership of the lease
      *
-     * @param executionDetermination the new value
+     * @param ownership ownership of the lease
      */
-    void setExecutionDetermination(ExecutionDetermination executionDetermination)
+    void setOwnership(Ownership ownership)
     {
-        this.executionDetermination = Objects.requireNonNull(executionDetermination, "executionDetermination must be provided");
+        this.leaseOwnership = Objects.requireNonNull(ownership, "ownership must be provided");
+    }
+
+    @VisibleForTesting // only use it for testing
+    public void setOwnershipTesting(Ownership ownership)
+    {
+        setOwnership(ownership);
+    }
+
+    /**
+     * Ownership of the cluster-wide lease
+     */
+    public enum Ownership
+    {
+        /**
+         * The cluster lease is claimed by the local Sidecar
+         */
+        CLAIMED,
+        /**
+         * The cluster lease is lost by the local Sidecar. It is claimed by another remote Sidecar.
+         */
+        LOST,
+        /**
+         * The cluster lease is neither claimed by the local Sidecar nor any other remote Sidecar
+         */
+        INDETERMINATE;
+
+        private ScheduleDecision toScheduleDecision()
+        {
+            switch (this)
+            {
+                case CLAIMED:
+                    return ScheduleDecision.EXECUTE;
+                case LOST:
+                    return ScheduleDecision.SKIP;
+                case INDETERMINATE:
+                default:
+                    // When the process is unable to determine whether it is the leaseholder, we want
+                    // the task to be rescheduled for a shorter period of time. Assume you have a PeriodicTask that
+                    // runs every day. If it happens to run during a period of time when there's no
+                    // determination whether task should run or not, we do not want to wait another day for the
+                    // task to run, so instead we reschedule it and only wait the initial delay of the task for
+                    // the task to try again.
+                    return ScheduleDecision.RESCHEDULE;
+            }
+        }
     }
 }
diff --git a/server/src/main/java/org/apache/cassandra/sidecar/coordination/ClusterLeaseClaimTask.java b/server/src/main/java/org/apache/cassandra/sidecar/coordination/ClusterLeaseClaimTask.java
index 05e09531..77216a05 100644
--- a/server/src/main/java/org/apache/cassandra/sidecar/coordination/ClusterLeaseClaimTask.java
+++ b/server/src/main/java/org/apache/cassandra/sidecar/coordination/ClusterLeaseClaimTask.java
@@ -36,8 +36,8 @@ import org.apache.cassandra.sidecar.config.ServiceConfiguration;
 import org.apache.cassandra.sidecar.db.SidecarLeaseDatabaseAccessor;
 import org.apache.cassandra.sidecar.metrics.CoordinationMetrics;
 import org.apache.cassandra.sidecar.metrics.SidecarMetrics;
-import org.apache.cassandra.sidecar.tasks.ExecutionDetermination;
 import org.apache.cassandra.sidecar.tasks.PeriodicTask;
+import org.apache.cassandra.sidecar.tasks.ScheduleDecision;
 import org.jetbrains.annotations.VisibleForTesting;
 
 import static org.apache.cassandra.sidecar.server.SidecarServerEvents.ON_SIDECAR_GLOBAL_LEASE_CLAIMED;
@@ -73,8 +73,8 @@ public class ClusterLeaseClaimTask implements PeriodicTask
     private final PeriodicTaskConfiguration periodicTaskConfiguration;
     private final ServiceConfiguration config;
     private final Vertx vertx;
-    private volatile String currentLeaseholder;
-    private volatile Instant leaseTime;
+    private String currentLeaseholder;
+    private Instant leaseTime;
 
     public ClusterLeaseClaimTask(Vertx vertx,
                                  ServiceConfiguration serviceConfiguration,
@@ -96,7 +96,7 @@ public class ClusterLeaseClaimTask implements PeriodicTask
      * {@inheritDoc}
      */
     @Override
-    public boolean shouldSkip()
+    public ScheduleDecision scheduleDecision()
     {
         // The Sidecar schema feature is required for this implementation
         // so skip when the feature is not enabled
@@ -111,10 +111,10 @@ public class ClusterLeaseClaimTask implements PeriodicTask
         }
         if (!isEnabled || !isMember)
         {
-            clusterLease.setExecutionDetermination(ExecutionDetermination.SKIP_EXECUTION);
-            return true;
+            clusterLease.setOwnership(ClusterLease.Ownership.LOST);
+            return ScheduleDecision.SKIP;
         }
-        return false;
+        return ScheduleDecision.EXECUTE;
     }
 
     /**
@@ -225,7 +225,7 @@ public class ClusterLeaseClaimTask implements PeriodicTask
                 }
 
                 leaseTime = null;
-                clusterLease.setExecutionDetermination(ExecutionDetermination.INDETERMINATE);
+                clusterLease.setOwnership(ClusterLease.Ownership.INDETERMINATE);
             }
             return;
         }
@@ -233,12 +233,12 @@ public class ClusterLeaseClaimTask implements PeriodicTask
         if (isCurrentLeaseholder(sidecarHostId))
         {
             leaseTime = Instant.now();
-            clusterLease.setExecutionDetermination(ExecutionDetermination.EXECUTE);
+            clusterLease.setOwnership(ClusterLease.Ownership.CLAIMED);
         }
         else
         {
             leaseTime = null;
-            clusterLease.setExecutionDetermination(ExecutionDetermination.SKIP_EXECUTION);
+            clusterLease.setOwnership(ClusterLease.Ownership.LOST);
         }
     }
 
@@ -314,6 +314,6 @@ public class ClusterLeaseClaimTask implements PeriodicTask
     {
         currentLeaseholder = null;
         leaseTime = null;
-        clusterLease.setExecutionDetermination(ExecutionDetermination.INDETERMINATE);
+        clusterLease.setOwnership(ClusterLease.Ownership.INDETERMINATE);
     }
 }
diff --git a/server/src/main/java/org/apache/cassandra/sidecar/job/OperationalJob.java b/server/src/main/java/org/apache/cassandra/sidecar/job/OperationalJob.java
index 1076b44f..80ec374c 100644
--- a/server/src/main/java/org/apache/cassandra/sidecar/job/OperationalJob.java
+++ b/server/src/main/java/org/apache/cassandra/sidecar/job/OperationalJob.java
@@ -193,11 +193,11 @@ public abstract class OperationalJob implements Task<Void>
     {
         isExecuting = true;
         LOGGER.info("Executing job. jobId={}", jobId);
+        promise.future().onComplete(executionPromise);
         try
         {
             // Blocking call to perform concrete job-specific execution, returning the status
             executeInternal();
-            executionPromise.tryComplete();
             promise.tryComplete();
             if (LOGGER.isDebugEnabled())
             {
@@ -208,7 +208,6 @@ public abstract class OperationalJob implements Task<Void>
         {
             OperationalJobException oje = OperationalJobException.wraps(e);
             LOGGER.error("Job execution failed. jobId={} reason='{}'", jobId, oje.getMessage(), oje);
-            executionPromise.tryFail(oje);
             promise.tryFail(oje);
         }
     }
diff --git a/server/src/main/java/org/apache/cassandra/sidecar/restore/RestoreJobDiscoverer.java b/server/src/main/java/org/apache/cassandra/sidecar/restore/RestoreJobDiscoverer.java
index 1984a1eb..308ff2b0 100644
--- a/server/src/main/java/org/apache/cassandra/sidecar/restore/RestoreJobDiscoverer.java
+++ b/server/src/main/java/org/apache/cassandra/sidecar/restore/RestoreJobDiscoverer.java
@@ -56,6 +56,7 @@ import org.apache.cassandra.sidecar.metrics.RestoreMetrics;
 import org.apache.cassandra.sidecar.metrics.SidecarMetrics;
 import org.apache.cassandra.sidecar.tasks.PeriodicTask;
 import org.apache.cassandra.sidecar.tasks.PeriodicTaskExecutor;
+import org.apache.cassandra.sidecar.tasks.ScheduleDecision;
 import org.apache.cassandra.sidecar.utils.InstanceMetadataFetcher;
 
 /**
@@ -133,22 +134,9 @@ public class RestoreJobDiscoverer implements PeriodicTask
     }
 
     @Override
-    public boolean shouldSkip()
+    public ScheduleDecision scheduleDecision()
     {
-        boolean shouldSkip = !sidecarSchema.isInitialized();
-        if (shouldSkip)
-        {
-            LOGGER.trace("Skipping restore job discovering due to sidecarSchema not initialized");
-        }
-        boolean executing = isExecuting.get();
-        shouldSkip = shouldSkip || executing;
-        if (executing)
-        {
-            LOGGER.trace("Skipping restore job discovering due to overlapping execution of this task");
-        }
-
-        // skip the task
-        return shouldSkip;
+        return shouldSkip() ? ScheduleDecision.SKIP : ScheduleDecision.EXECUTE;
     }
 
     @Override
@@ -189,6 +177,24 @@ public class RestoreJobDiscoverer implements PeriodicTask
         }
     }
 
+    private boolean shouldSkip()
+    {
+        boolean shouldSkip = !sidecarSchema.isInitialized();
+        if (shouldSkip)
+        {
+            LOGGER.trace("Skipping restore job discovering due to sidecarSchema not initialized");
+        }
+        boolean executing = isExecuting.get();
+        shouldSkip = shouldSkip || executing;
+        if (executing)
+        {
+            LOGGER.trace("Skipping restore job discovering due to overlapping execution of this task");
+        }
+
+        // skip the task
+        return shouldSkip;
+    }
+
     private void executeInternal()
     {
         Preconditions.checkState(periodicTaskExecutor != null, "Loop executor is not registered");
diff --git a/server/src/main/java/org/apache/cassandra/sidecar/restore/RestoreProcessor.java b/server/src/main/java/org/apache/cassandra/sidecar/restore/RestoreProcessor.java
index 76a0b321..58b4bc77 100644
--- a/server/src/main/java/org/apache/cassandra/sidecar/restore/RestoreProcessor.java
+++ b/server/src/main/java/org/apache/cassandra/sidecar/restore/RestoreProcessor.java
@@ -49,6 +49,7 @@ import org.apache.cassandra.sidecar.exceptions.RestoreJobExceptions;
 import org.apache.cassandra.sidecar.metrics.SidecarMetrics;
 import org.apache.cassandra.sidecar.metrics.instance.InstanceRestoreMetrics;
 import org.apache.cassandra.sidecar.tasks.PeriodicTask;
+import org.apache.cassandra.sidecar.tasks.ScheduleDecision;
 import org.apache.cassandra.sidecar.utils.SSTableImporter;
 
 /**
@@ -130,14 +131,15 @@ public class RestoreProcessor implements PeriodicTask
     }
 
     @Override
-    public boolean shouldSkip()
+    public ScheduleDecision scheduleDecision()
     {
         boolean shouldSkip = !sidecarSchema().isInitialized();
         if (shouldSkip)
         {
             LOGGER.trace("Skipping restore job processing");
+            return ScheduleDecision.SKIP;
         }
-        return shouldSkip;
+        return ScheduleDecision.EXECUTE;
     }
 
     @Override
diff --git a/server/src/main/java/org/apache/cassandra/sidecar/restore/RingTopologyRefresher.java b/server/src/main/java/org/apache/cassandra/sidecar/restore/RingTopologyRefresher.java
index 53ba5059..27db0081 100644
--- a/server/src/main/java/org/apache/cassandra/sidecar/restore/RingTopologyRefresher.java
+++ b/server/src/main/java/org/apache/cassandra/sidecar/restore/RingTopologyRefresher.java
@@ -42,6 +42,7 @@ import org.apache.cassandra.sidecar.config.SidecarConfiguration;
 import org.apache.cassandra.sidecar.db.RestoreJob;
 import org.apache.cassandra.sidecar.exceptions.CassandraUnavailableException;
 import org.apache.cassandra.sidecar.tasks.PeriodicTask;
+import org.apache.cassandra.sidecar.tasks.ScheduleDecision;
 import org.apache.cassandra.sidecar.utils.InstanceMetadataFetcher;
 import org.jetbrains.annotations.Nullable;
 import org.jetbrains.annotations.VisibleForTesting;
@@ -74,9 +75,9 @@ public class RingTopologyRefresher implements PeriodicTask
     }
 
     @Override
-    public boolean shouldSkip()
+    public ScheduleDecision scheduleDecision()
     {
-        return replicaByTokenRangePerKeyspace.isEmpty();
+        return replicaByTokenRangePerKeyspace.isEmpty() ? ScheduleDecision.SKIP : ScheduleDecision.EXECUTE;
     }
 
     @Override
diff --git a/server/src/main/java/org/apache/cassandra/sidecar/tasks/HealthCheckPeriodicTask.java b/server/src/main/java/org/apache/cassandra/sidecar/tasks/HealthCheckPeriodicTask.java
index daaeb44f..20e7bac2 100644
--- a/server/src/main/java/org/apache/cassandra/sidecar/tasks/HealthCheckPeriodicTask.java
+++ b/server/src/main/java/org/apache/cassandra/sidecar/tasks/HealthCheckPeriodicTask.java
@@ -30,13 +30,13 @@ import io.vertx.core.Promise;
 import io.vertx.core.Vertx;
 import io.vertx.core.eventbus.EventBus;
 import org.apache.cassandra.sidecar.cluster.InstancesMetadata;
+import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata;
 import org.apache.cassandra.sidecar.concurrent.ExecutorPools;
 import org.apache.cassandra.sidecar.concurrent.TaskExecutorPool;
 import org.apache.cassandra.sidecar.config.SidecarConfiguration;
 import org.apache.cassandra.sidecar.metrics.HealthMetrics;
 import org.apache.cassandra.sidecar.metrics.SidecarMetrics;
 
-import static org.apache.cassandra.sidecar.server.SidecarServerEvents.ON_SERVER_START;
 import static org.apache.cassandra.sidecar.server.SidecarServerEvents.ON_SERVER_STOP;
 
 /**
@@ -67,7 +67,6 @@ public class HealthCheckPeriodicTask implements PeriodicTask
     @Override
     public void registerPeriodicTaskExecutor(PeriodicTaskExecutor executor)
     {
-        eventBus.localConsumer(ON_SERVER_START.address(), message -> executor.schedule(this));
         eventBus.localConsumer(ON_SERVER_STOP.address(), message -> executor.unschedule(this));
     }
 
@@ -92,30 +91,12 @@ public class HealthCheckPeriodicTask implements PeriodicTask
         AtomicInteger instanceDown = new AtomicInteger(0);
         List<Future<?>> futures = instancesMetadata.instances()
                                                    .stream()
-                                                   .map(instanceMetadata -> internalPool.executeBlocking(p -> {
-                                                     try
-                                                     {
-                                                         instanceMetadata.delegate().healthCheck();
-                                                         p.complete();
-                                                     }
-                                                     catch (Throwable cause)
-                                                     {
-                                                         instanceDown.incrementAndGet();
-                                                         p.fail(cause);
-                                                         LOGGER.error("Unable to complete health check on instance={}",
-                                                                      instanceMetadata.id(), cause);
-                                                     }
-                                                 }, false))
+                                                   .map(instanceMetadata -> healthCheck(instanceMetadata, instanceDown))
                                                    .collect(Collectors.toList());
 
         // join always waits until all its futures are completed and will not fail as soon as one of the future fails
         Future.join(futures)
-              .onComplete(v -> {
-                  int instanceDownCount = instanceDown.get();
-                  int instanceUpCount = instancesMetadata.instances().size() - instanceDownCount;
-                  metrics.cassandraInstancesUp.metric.setValue(instanceUpCount);
-                  metrics.cassandraInstancesDown.metric.setValue(instanceDownCount);
-              })
+              .onComplete(v -> updateMetrics(instanceDown))
               .onSuccess(v -> promise.complete())
               .onFailure(promise::fail);
     }
@@ -125,4 +106,23 @@ public class HealthCheckPeriodicTask implements PeriodicTask
     {
         return "Health Check";
     }
+
+    private void updateMetrics(AtomicInteger instanceDown)
+    {
+        int instanceDownCount = instanceDown.get();
+        int instanceUpCount = instancesMetadata.instances().size() - instanceDownCount;
+        metrics.cassandraInstancesUp.metric.setValue(instanceUpCount);
+        metrics.cassandraInstancesDown.metric.setValue(instanceDownCount);
+    }
+
+    private Future<Void> healthCheck(InstanceMetadata instanceMetadata, AtomicInteger instanceDown)
+    {
+        return internalPool
+               .runBlocking(() -> instanceMetadata.delegate().healthCheck(), false)
+               .onFailure(cause -> {
+                   instanceDown.incrementAndGet();
+                   LOGGER.error("Unable to complete health check on instance={}",
+                                instanceMetadata.id(), cause);
+               });
+    }
 }
diff --git a/server/src/main/java/org/apache/cassandra/sidecar/tasks/KeyStoreCheckPeriodicTask.java b/server/src/main/java/org/apache/cassandra/sidecar/tasks/KeyStoreCheckPeriodicTask.java
index d9744f8c..f05b73da 100644
--- a/server/src/main/java/org/apache/cassandra/sidecar/tasks/KeyStoreCheckPeriodicTask.java
+++ b/server/src/main/java/org/apache/cassandra/sidecar/tasks/KeyStoreCheckPeriodicTask.java
@@ -52,14 +52,14 @@ public class KeyStoreCheckPeriodicTask implements PeriodicTask
     /**
      * Skip check if the key store is not configured or if the key store should not be reloaded
      *
-     * @return {@code true} if the key store is not configured or if the keystore should not be reloaded,
-     * {@code false} otherwise
+     * @return whether to {@link ScheduleDecision#SKIP} or {@link ScheduleDecision#EXECUTE} this task
      */
     @Override
-    public boolean shouldSkip()
+    public ScheduleDecision scheduleDecision()
     {
-        return !configuration.isKeystoreConfigured()
-               || !configuration.keystore().reloadStore();
+        return shouldSkip()
+               ? ScheduleDecision.SKIP
+               : ScheduleDecision.EXECUTE;
     }
 
     @Override
@@ -124,4 +124,16 @@ public class KeyStoreCheckPeriodicTask implements PeriodicTask
                  lastModifiedTime = -1;
              });
     }
+
+    /**
+     * Skip check if the key store is not configured or if the key store should not be reloaded
+     *
+     * @return {@code true} if the key store is not configured or if the keystore should not be reloaded,
+     * {@code false} otherwise
+     */
+    private boolean shouldSkip()
+    {
+        return !configuration.isKeystoreConfigured()
+               || !configuration.keystore().reloadStore();
+    }
 }
diff --git a/server/src/main/java/org/apache/cassandra/sidecar/tasks/PeriodicTask.java b/server/src/main/java/org/apache/cassandra/sidecar/tasks/PeriodicTask.java
index 6525b2a8..04731819 100644
--- a/server/src/main/java/org/apache/cassandra/sidecar/tasks/PeriodicTask.java
+++ b/server/src/main/java/org/apache/cassandra/sidecar/tasks/PeriodicTask.java
@@ -20,6 +20,8 @@ package org.apache.cassandra.sidecar.tasks;
 
 import java.util.concurrent.TimeUnit;
 
+import io.vertx.core.Promise;
+
 /**
  * An interface that defines a periodic task that will be executed during the lifecycle of Cassandra Sidecar
  */
@@ -38,6 +40,11 @@ public interface PeriodicTask extends Task<Void>
         return TimeUnit.MILLISECONDS;
     }
 
+    default long delayMillis()
+    {
+        return delayUnit().toMillis(delay());
+    }
+
     /**
      * @return the initial delay for the task, defaults to the {@link #delay()}
      */
@@ -54,6 +61,11 @@ public interface PeriodicTask extends Task<Void>
         return delayUnit();
     }
 
+    default long initialDelayMillis()
+    {
+        return initialDelayUnit().toMillis(initialDelay());
+    }
+
     /**
      * Register the periodic task executor at the task. By default, it is no-op.
      * If the reference to the executor is needed, the concrete {@link PeriodicTask} can implement this method
@@ -65,13 +77,14 @@ public interface PeriodicTask extends Task<Void>
     }
 
     /**
-     * Specify whether the task should be skipped.
-     * // TODO: consider returning the reason to skip, instead of just a boolean value
-     * @return {@code true} to skip; otherwise, return {@code false}
+     * Specify the schedule decision of the upcoming run.
+     * The method is evaluated before calling {@link #execute(Promise)}
+     *
+     * @return schedule decision. The default is to {@link ScheduleDecision#EXECUTE}.
      */
-    default boolean shouldSkip()
+    default ScheduleDecision scheduleDecision()
     {
-        return false;
+        return ScheduleDecision.EXECUTE;
     }
 
     @Override
diff --git a/server/src/main/java/org/apache/cassandra/sidecar/tasks/PeriodicTaskExecutor.java b/server/src/main/java/org/apache/cassandra/sidecar/tasks/PeriodicTaskExecutor.java
index 4be8c427..bbb5afb1 100644
--- a/server/src/main/java/org/apache/cassandra/sidecar/tasks/PeriodicTaskExecutor.java
+++ b/server/src/main/java/org/apache/cassandra/sidecar/tasks/PeriodicTaskExecutor.java
@@ -21,6 +21,8 @@ package org.apache.cassandra.sidecar.tasks;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -28,8 +30,8 @@ import org.slf4j.LoggerFactory;
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import io.vertx.core.Closeable;
+import io.vertx.core.Future;
 import io.vertx.core.Promise;
-import io.vertx.core.impl.ConcurrentHashSet;
 import org.apache.cassandra.sidecar.concurrent.ExecutorPools;
 import org.apache.cassandra.sidecar.concurrent.TaskExecutorPool;
 import org.apache.cassandra.sidecar.coordination.ClusterLease;
@@ -38,23 +40,28 @@ import org.jetbrains.annotations.VisibleForTesting;
 
 /**
  * This class manages the scheduling and execution of {@link PeriodicTask}s.
+ * For the {@link PeriodicTask} that also is {@link ExecuteOnClusterLeaseholderOnly}, the executor ensures
+ * the task execution is only performed when {@link ClusterLease} is claimed by the local Sidecar instance.
+ *
+ * <p>The execution of each {@link PeriodicTask} is <i>ordered</i> and <i>serial</i>, meanwhile there could
+ * be concurrent execution of different {@link PeriodicTask}s.
+ * <p>Memory consistency effects: Actions in the prior {@link PeriodicTask} run <i>happen-before</i> its
+ * next run, perhaps in another thread. In other words, writes in the prior run can be read by its next run,
+ * as if it is running in a single thread.
  */
 @Singleton
 public class PeriodicTaskExecutor implements Closeable
 {
     private static final Logger LOGGER = LoggerFactory.getLogger(PeriodicTaskExecutor.class);
+    private static final long PLACEHOLDER_TIMER_ID = -1L; // used when the run start immediately, not scheduled via a timer
 
+    // keep track of the timerIds in order to cancel them when closing/unscheduling
     private final Map<PeriodicTaskKey, Long> timerIds = new ConcurrentHashMap<>();
-    private final Set<PeriodicTaskKey> activeTasks = new ConcurrentHashSet<>();
+    private final Map<PeriodicTaskKey, Future<Void>> activeRuns = new ConcurrentHashMap<>();
+    private final Set<PeriodicTaskKey> poisonPilledTasks = ConcurrentHashMap.newKeySet();
     private final TaskExecutorPool internalPool;
     private final ClusterLease clusterLease;
 
-    @VisibleForTesting
-    public PeriodicTaskExecutor(ExecutorPools executorPools)
-    {
-        this(executorPools, null);
-    }
-
     @Inject
     public PeriodicTaskExecutor(ExecutorPools executorPools, ClusterLease clusterLease)
     {
@@ -66,53 +73,142 @@ public class PeriodicTaskExecutor implements Closeable
      * Schedules the {@code task} iff it has not been scheduled yet.
      *
      * @param task the task to execute
-     * @return the identifier of the timer associated with this task
      */
-    public long schedule(PeriodicTask task)
+    public void schedule(PeriodicTask task)
     {
         PeriodicTaskKey key = new PeriodicTaskKey(task);
-        return timerIds.computeIfAbsent(key, k -> {
-            task.registerPeriodicTaskExecutor(this);
-            long initialDelayMillis = task.initialDelayUnit().toMillis(task.initialDelay());
-            long delayMillis = task.delayUnit().toMillis(task.delay());
-            return internalPool.setPeriodic(initialDelayMillis, delayMillis, id -> executeInternal(k));
+        schedule(key, 0, task.initialDelayMillis(), 0);
+    }
+
+    private void schedule(PeriodicTaskKey key, long priorExecDurationMillis, long delayMillis, long execCount)
+    {
+        long actualDelayMillis = delayMillis - priorExecDurationMillis;
+        AtomicBoolean runImmediately = new AtomicBoolean(actualDelayMillis <= 0);
+        timerIds.compute(key, (k, v) -> {
+            // The periodic task has been scheduled already. Exit early and avoid scheduling the duplication
+            if (v != null && execCount == 0)
+            {
+                LOGGER.debug("Task is already scheduled. task='{}'", key);
+                runImmediately.set(false);
+                return v;
+            }
+
+            LOGGER.debug("Scheduling task {}. task='{}' execCount={}",
+                         runImmediately.get() ? "immediately" : "in " + actualDelayMillis + " milliseconds",
+                         key, execCount);
+
+            try
+            {
+                key.task.registerPeriodicTaskExecutor(this);
+            }
+            catch (Exception e)
+            {
+                LOGGER.warn("Failed to invoke registerPeriodicTaskExecutor. task='{}'", key, e);
+            }
+
+            // If run immediately, do not execute within the compute block.
+            // Return the placeholder timer ID, and execute after exiting the compute block.
+            if (runImmediately.get())
+            {
+                return PLACEHOLDER_TIMER_ID; // use the placeholder ID, since this run is not scheduled as a timer
+            }
+            // Schedule and update the timer id
+            return internalPool.setTimer(delayMillis, timerId -> executeAndScheduleNext(key, execCount));
         });
+
+        if (runImmediately.get())
+        {
+            executeAndScheduleNext(key, execCount);
+        }
+    }
+
+    /**
+     * This method ensures the happens-before memory consistency effect between runs of the same {@link PeriodicTask}.
+     * <p>Each run, essentially, is executed at {@link java.util.concurrent.Executor#execute(Runnable)}. At the end of the
+     * execution, the executor schedules a new run. When the scheduled time comes, the next run is executed.
+     * There are the following happens-before relationships, <i>hb(prior_run, scheduler></i> and <i>hb(scheduler, next_run)</i>.
+     * Therefore, <i>hb(prior_run, next_run)</i>, i.e. the effects from prior_run are visible to next_run.
+     * <p>More on <a href="https://docs.oracle.com/javase/specs/jls/se11/html/jls-17.html#jls-17.4">Java Memory Model</a>
+     */
+    private void executeAndScheduleNext(PeriodicTaskKey key, long execCount)
+    {
+        long startTime = System.nanoTime();
+        internalPool.<Void>executeBlocking(promise -> executeInternal(promise, key, execCount), false)
+                    .onComplete(ignored -> {
+                        LOGGER.debug("Task run finishes. task='{}' execCount={}", key, execCount);
+                        // schedule the next run iff the task is not killed
+                        if (poisonPilledTasks.remove(key))
+                        {
+                            // timerId might get populated after unschedule due to race.
+                            // Have another attempt to clean up here.
+                            timerIds.remove(key);
+                            LOGGER.debug("Avoid scheduling the next run, and remove it from poisonPilledTasks. task='{}' execCount={}",
+                                         key, execCount);
+                            return;
+                        }
+                        long priorExecutionDurationMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime);
+                        schedule(key, priorExecutionDurationMillis, key.task.delayMillis(), execCount + 1);
+                    });
     }
 
     /**
-     * Unschedules the {@code task} iff the task has been registered previously and returns the identifier
-     * of the timer associated with this task, or {@code -1} if the task is not registered.
+     * Unschedule and close the {@link PeriodicTask} iff it has been scheduled.
      *
      * @param task the task to unschedule
-     * @return the identifier of the timer associated with this task, or {@code -1} if the task is not registered
      */
-    public long unschedule(PeriodicTask task)
+    public void unschedule(PeriodicTask task)
     {
-        Long timerId = timerIds.remove(new PeriodicTaskKey(task));
-        if (timerId != null)
+        unschedule(task, true);
+    }
+
+    /**
+     * Unschedule the {@link PeriodicTask} iff it has been scheduled.
+     *
+     * @param task the {@link PeriodicTask} to unschedule
+     * @param shouldCloseTask indicate whether {@link PeriodicTask#close()} should be called
+     * @return a future of unschedule
+     */
+    private Future<Void> unschedule(PeriodicTask task, boolean shouldCloseTask)
+    {
+        PeriodicTaskKey key = new PeriodicTaskKey(task);
+        Long timerId = timerIds.remove(key);
+        if (timerId == null)
+        {
+            return Future.failedFuture("No such PeriodicTask: " + key);
+        }
+
+        LOGGER.debug("Unscheduling task. task='{}' timerId={}", key, timerId);
+        // always insert a poison pill when unscheduling an existing task,
+        // and conditionally remove when the timer can be cancelled
+        poisonPilledTasks.add(key);
+        // if timer is not started, it can be cancelled
+        if (timerId != PLACEHOLDER_TIMER_ID && internalPool.cancelTimer(timerId))
         {
-            internalPool.cancelTimer(timerId);
-            task.close();
-            return timerId;
+            poisonPilledTasks.remove(key);
         }
-        return -1L; // valid timer ids are non-negative
+
+        // The current run might have started already.
+        // If so, a non-null activeRun should be retrieved and a poison pill
+        // is placed to avoid further scheduling.
+        // Reschedule should only happen after the activeRun finishes.
+        Future<Void> unscheduleFuture = activeRuns.getOrDefault(key, Future.succeededFuture());
+        return shouldCloseTask
+               ? unscheduleFuture.onComplete(ignored -> task.close())
+               : unscheduleFuture;
     }
 
+
     /**
-     * Reschedules the provided {@code task} and returns the identifier of the timer associated with the rescheduled
-     * task.
+     * Reschedules the provided {@code task}.
+     * <p>The difference from {@link #unschedule(PeriodicTask)} then {@link #schedule(PeriodicTask)}
+     * is that the {@link PeriodicTask} in question is not closed when unscheduling
      *
      * @param task the task to reschedule
-     * @return the identifier of the timer associated with the rescheduled task
      */
-    public long reschedule(PeriodicTask task)
+    public void reschedule(PeriodicTask task)
     {
-        Long timerId = timerIds.remove(new PeriodicTaskKey(task));
-        if (timerId != null)
-        {
-            internalPool.cancelTimer(timerId);
-        }
-        return schedule(task);
+        unschedule(task, false)
+        .onComplete(ignored -> schedule(task));
     }
 
     @Override
@@ -123,7 +219,6 @@ public class PeriodicTaskExecutor implements Closeable
             timerIds.values().forEach(internalPool::cancelTimer);
             timerIds.keySet().forEach(key -> key.task.close());
             timerIds.clear();
-            activeTasks.clear();
             completion.complete();
         }
         catch (Throwable throwable)
@@ -132,76 +227,73 @@ public class PeriodicTaskExecutor implements Closeable
         }
     }
 
-    private void executeInternal(PeriodicTaskKey key)
+    private void executeInternal(Promise<Void> promise, PeriodicTaskKey key, long execCount)
     {
         PeriodicTask periodicTask = key.task;
-
-        switch (determineExecution(periodicTask))
+        switch (consolidateScheduleDecision(periodicTask))
         {
-            case SKIP_EXECUTION:
-                LOGGER.trace("Skip executing task. task={}", periodicTask.name());
+            case SKIP:
+                LOGGER.trace("Skip executing task. task='{}' execCount={}", key, execCount);
+                promise.tryComplete();
                 return;
 
             case EXECUTE:
                 break;
 
-            case INDETERMINATE:
+            case RESCHEDULE:
             default:
-                LOGGER.debug("Unable to determine execution for this task, rescheduling. task={}", periodicTask.name());
-                // When the process is unable to determine whether a task should be executed, we want
-                // the task to be rescheduled for a shorter period of time. Assume you have a PeriodicTask that
-                // runs every day. If it happens to run during a period of time when there's no
-                // determination whether task should run or not, we do not want to wait another day for the
-                // task to run, so instead we reschedule it and only wait the initial delay of the task for
-                // the task to try again.
+                LOGGER.debug("Reschedule the task. task='{}' execCount={}", key, execCount);
                 reschedule(periodicTask);
+                promise.tryComplete();
                 return;
         }
 
-        if (!activeTasks.add(key))
-        {
-            LOGGER.debug("Task is already running. task={}", periodicTask.name());
-            return;
-        }
-
-        Promise<Void> promise = Promise.promise();
-
+        // Leverage a separate promise, taskRunPromise, to ensure the activeRun removal
+        // happens before the completion of the promise of executeBlocking and its subsequent calls
+        Promise<Void> taskRunPromise = Promise.promise();
+        Future<Void> futureResult = taskRunPromise.future().onComplete(res -> {
+            LOGGER.debug("Removing task from active runs. task='{}' execCount={}", key, execCount);
+            activeRuns.remove(key);
+            // finally complete the promise of executeBlocking
+            promise.handle(res);
+        });
         try
         {
-            periodicTask.execute(promise);
+            LOGGER.debug("Executing task. task='{}' execCount={}", key, execCount);
+            activeRuns.put(key, futureResult);
+            periodicTask.execute(taskRunPromise);
         }
         catch (Throwable throwable)
         {
-            LOGGER.warn("Periodic task failed to execute. task={}", periodicTask.name(), throwable);
-            promise.tryFail(throwable);
+            LOGGER.warn("Periodic task failed to execute. task='{}' execCount={}", periodicTask.name(), execCount, throwable);
+            taskRunPromise.tryFail(throwable);
         }
-
-        promise.future().onComplete(res -> activeTasks.remove(key));
     }
 
     /**
-     * Determines whether the task should run.
+     * Consolidate the {@link ScheduleDecision} from the {@link PeriodicTask} and {@link ClusterLease}
      *
      * @param periodicTask the task
-     * @return the result of the determination
+     * @return schedule decision
      */
-    protected ExecutionDetermination determineExecution(PeriodicTask periodicTask)
+    protected ScheduleDecision consolidateScheduleDecision(PeriodicTask periodicTask)
     {
-        if (periodicTask.shouldSkip())
+        ScheduleDecision decisionFromTask = periodicTask.scheduleDecision();
+        // Take the cluster lease ownership into consideration, if the periodic task
+        // decides to execute or reschedule.
+        // For example, if the local sidecar is not the cluster leaseholder, the lease
+        // ownership should override the decision.
+        if (decisionFromTask != ScheduleDecision.SKIP
+            && periodicTask instanceof ExecuteOnClusterLeaseholderOnly)
         {
-            return ExecutionDetermination.SKIP_EXECUTION;
+            return clusterLease.toScheduleDecision();
         }
-
-        if (periodicTask instanceof ExecuteOnClusterLeaseholderOnly)
-        {
-            return clusterLease.executionDetermination();
-        }
-        return ExecutionDetermination.EXECUTE;
+        return decisionFromTask;
     }
 
     // A simple wrapper that implements equals and hashcode,
     // which is not necessary for the actual ExecutionLoops to implement
-    private static class PeriodicTaskKey
+    static class PeriodicTaskKey
     {
         private final String fqcnAndName;
         private final PeriodicTask task;
@@ -231,5 +323,23 @@ public class PeriodicTaskExecutor implements Closeable
 
             return false;
         }
+
+        @Override
+        public String toString()
+        {
+            return fqcnAndName;
+        }
+    }
+
+    @VisibleForTesting
+    Map<PeriodicTaskKey, Long> timerIds()
+    {
+        return timerIds;
+    }
+
+    @VisibleForTesting
+    Set<PeriodicTaskKey> poisonPilledTasks()
+    {
+        return poisonPilledTasks;
     }
 }
diff --git a/server/src/main/java/org/apache/cassandra/sidecar/tasks/ExecutionDetermination.java b/server/src/main/java/org/apache/cassandra/sidecar/tasks/ScheduleDecision.java
similarity index 68%
rename from server/src/main/java/org/apache/cassandra/sidecar/tasks/ExecutionDetermination.java
rename to server/src/main/java/org/apache/cassandra/sidecar/tasks/ScheduleDecision.java
index 72b83c63..55e23c8e 100644
--- a/server/src/main/java/org/apache/cassandra/sidecar/tasks/ExecutionDetermination.java
+++ b/server/src/main/java/org/apache/cassandra/sidecar/tasks/ScheduleDecision.java
@@ -19,28 +19,22 @@
 package org.apache.cassandra.sidecar.tasks;
 
 /**
- * Determines whether executions can proceed, be skipped, or the state is indeterminate
- * and an action can be taken based on the indeterminate state (i.e. rescheduling a {@link PeriodicTask}).
+ * Determines whether executions can proceed, be skipped, or the {@link PeriodicTask} should be rescheduled
  */
-public enum ExecutionDetermination
+public enum ScheduleDecision
 {
     /**
-     * The execution can proceed.
+     * Execute the upcoming run
      */
     EXECUTE,
 
     /**
-     * The execution will be skipped.
+     * Skip the upcoming run, but do not change the schedule
      */
-    SKIP_EXECUTION,
+    SKIP,
 
     /**
-     * It is not possible to determine whether the execution should proceed or be skipped.
+     * Skip the upcoming run and reschedule the {@link PeriodicTask}
      */
-    INDETERMINATE;
-
-    public boolean shouldExecuteOnLocalSidecar()
-    {
-        return this == EXECUTE;
-    }
+    RESCHEDULE;
 }
diff --git a/server/src/main/java/org/apache/cassandra/sidecar/tasks/Task.java b/server/src/main/java/org/apache/cassandra/sidecar/tasks/Task.java
index d63d3bbd..fad1929a 100644
--- a/server/src/main/java/org/apache/cassandra/sidecar/tasks/Task.java
+++ b/server/src/main/java/org/apache/cassandra/sidecar/tasks/Task.java
@@ -22,8 +22,8 @@ import io.vertx.core.Promise;
 import org.jetbrains.annotations.Nullable;
 
 /**
- *  * An interface that defines a task that will be executed during the lifecycle of Cassandra Sidecar
- * @param <T>
+ * An interface that defines a task that will be executed during the lifecycle of Cassandra Sidecar
+ * @param <T> result value type
  */
 public interface Task<T>
 {
diff --git a/server/src/test/integration/org/apache/cassandra/sidecar/coordination/ClusterLeaseClaimTaskIntegrationTest.java b/server/src/test/integration/org/apache/cassandra/sidecar/coordination/ClusterLeaseClaimTaskIntegrationTest.java
index 6f67424a..900d5b24 100644
--- a/server/src/test/integration/org/apache/cassandra/sidecar/coordination/ClusterLeaseClaimTaskIntegrationTest.java
+++ b/server/src/test/integration/org/apache/cassandra/sidecar/coordination/ClusterLeaseClaimTaskIntegrationTest.java
@@ -63,7 +63,7 @@ import org.apache.cassandra.sidecar.metrics.CoordinationMetrics;
 import org.apache.cassandra.sidecar.metrics.MetricRegistryFactory;
 import org.apache.cassandra.sidecar.metrics.SidecarMetrics;
 import org.apache.cassandra.sidecar.metrics.SidecarMetricsImpl;
-import org.apache.cassandra.sidecar.tasks.ExecutionDetermination;
+import org.apache.cassandra.sidecar.tasks.ScheduleDecision;
 import org.apache.cassandra.sidecar.testing.SharedExecutorNettyOptions;
 import org.apache.cassandra.testing.TestVersion;
 import org.jetbrains.annotations.NotNull;
@@ -138,7 +138,7 @@ class ClusterLeaseClaimTaskIntegrationTest
         assertThat(simulatedInstances).as("There are no leaseholders when the process has not run yet")
                                       .allMatch(e -> !e.clusterLease.isClaimedByLocalSidecar());
         assertThat(simulatedInstances).as("And the state for all of them is indeterminate")
-                                      .allMatch(e -> e.clusterLease.executionDetermination() == ExecutionDetermination.INDETERMINATE);
+                                      .allMatch(e -> e.clusterLease.toScheduleDecision() == ScheduleDecision.RESCHEDULE);
 
         AtomicReference<Object[][]> currentLeaseholderQueryResult = new AtomicReference<>();
         AtomicReference<TestInstanceWrapper> currentLeaseholder = new AtomicReference<>();
@@ -159,9 +159,9 @@ class ClusterLeaseClaimTaskIntegrationTest
         currentLeaseholder.get().clusterLeaseClaimTask.resetLeaseholder();
         assertThat(simulatedInstances).as("No instances are expected as we've just reset the existing leaseholder information")
                                       .allMatch(e -> !e.clusterLease.isClaimedByLocalSidecar());
-        assertThat(currentLeaseholder.get().clusterLease.executionDetermination())
+        assertThat(currentLeaseholder.get().clusterLease.toScheduleDecision())
         .as("And the state for the current leaseholder is indeterminate")
-        .isEqualTo(ExecutionDetermination.INDETERMINATE);
+        .isEqualTo(ScheduleDecision.RESCHEDULE);
 
         loopAssert(3, () -> {
             runLeaseAcquireProcess(pool, simulatedInstances);
@@ -243,13 +243,13 @@ class ClusterLeaseClaimTaskIntegrationTest
         // then disable binary
         simulateDisableBinaryOfLeaseholder(simulatedInstances);
 
-        ExecutionDetermination executionDetermination = null;
+        ScheduleDecision scheduleDecision = null;
         for (int i = 0; i < 20; i++)
         {
             leaseholder.clusterLeaseClaimTask.runClaimProcess();
-            executionDetermination = leaseholder.clusterLease.executionDetermination();
+            scheduleDecision = leaseholder.clusterLease.toScheduleDecision();
 
-            if (executionDetermination != ExecutionDetermination.INDETERMINATE)
+            if (scheduleDecision != ScheduleDecision.RESCHEDULE)
             {
                 int ttlSeconds = Math.max(1, maybeDetermineTTL(cluster));
                 LOGGER.info("TTL is {} seconds", ttlSeconds);
@@ -260,8 +260,8 @@ class ClusterLeaseClaimTaskIntegrationTest
             }
             else break;
         }
-        assertThat(executionDetermination).as("The leaseholder should give up the lease")
-                                          .isEqualTo(ExecutionDetermination.INDETERMINATE);
+        assertThat(scheduleDecision).as("The leaseholder should give up the lease")
+                                    .isEqualTo(ScheduleDecision.RESCHEDULE);
         // ensure the data is TTL'd in the database
         for (int i = 0; i < 20; i++)
         {
diff --git a/server/src/test/integration/org/apache/cassandra/sidecar/testing/IntegrationTestModule.java b/server/src/test/integration/org/apache/cassandra/sidecar/testing/IntegrationTestModule.java
index b9746163..a2fc3846 100644
--- a/server/src/test/integration/org/apache/cassandra/sidecar/testing/IntegrationTestModule.java
+++ b/server/src/test/integration/org/apache/cassandra/sidecar/testing/IntegrationTestModule.java
@@ -49,7 +49,6 @@ import org.apache.cassandra.sidecar.config.yaml.SslConfigurationImpl;
 import org.apache.cassandra.sidecar.config.yaml.TestServiceConfiguration;
 import org.apache.cassandra.sidecar.coordination.ClusterLease;
 import org.apache.cassandra.sidecar.exceptions.NoSuchSidecarInstanceException;
-import org.apache.cassandra.sidecar.tasks.ExecutionDetermination;
 import org.jetbrains.annotations.NotNull;
 
 import static org.apache.cassandra.sidecar.server.SidecarServerEvents.ON_SERVER_STOP;
@@ -152,7 +151,7 @@ public class IntegrationTestModule extends AbstractModule
     @Singleton
     public ClusterLease clusterLease()
     {
-        return new ClusterLease(ExecutionDetermination.EXECUTE);
+        return new ClusterLease(ClusterLease.Ownership.CLAIMED);
     }
 
     private AccessControlConfiguration accessControlConfiguration()
diff --git a/server/src/test/java/org/apache/cassandra/sidecar/CassandraSidecarDaemonTest.java b/server/src/test/java/org/apache/cassandra/sidecar/CassandraSidecarDaemonTest.java
index bad370ae..6a16f7ec 100644
--- a/server/src/test/java/org/apache/cassandra/sidecar/CassandraSidecarDaemonTest.java
+++ b/server/src/test/java/org/apache/cassandra/sidecar/CassandraSidecarDaemonTest.java
@@ -29,6 +29,9 @@ import java.util.concurrent.TimeUnit;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import io.vertx.core.Vertx;
 import io.vertx.ext.web.client.HttpResponse;
 import io.vertx.ext.web.client.WebClient;
@@ -44,6 +47,8 @@ import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException
  */
 class CassandraSidecarDaemonTest
 {
+    private static final Logger LOGGER = LoggerFactory.getLogger(CassandraSidecarDaemonTest.class);
+
     static final String[] NO_ARGS = {};
 
     @BeforeEach
@@ -130,6 +135,7 @@ class CassandraSidecarDaemonTest
         }
         finally
         {
+            LOGGER.debug("Tearing down");
             maybeStopCassandraSidecar();
             Files.deleteIfExists(targetFile);
 
diff --git a/server/src/test/java/org/apache/cassandra/sidecar/coordination/ClusterLeaseClaimTaskTest.java b/server/src/test/java/org/apache/cassandra/sidecar/coordination/ClusterLeaseClaimTaskTest.java
index dfa46aae..929a8607 100644
--- a/server/src/test/java/org/apache/cassandra/sidecar/coordination/ClusterLeaseClaimTaskTest.java
+++ b/server/src/test/java/org/apache/cassandra/sidecar/coordination/ClusterLeaseClaimTaskTest.java
@@ -30,7 +30,7 @@ import io.vertx.core.Vertx;
 import org.apache.cassandra.sidecar.config.ServiceConfiguration;
 import org.apache.cassandra.sidecar.db.SidecarLeaseDatabaseAccessor;
 import org.apache.cassandra.sidecar.metrics.SidecarMetrics;
-import org.apache.cassandra.sidecar.tasks.ExecutionDetermination;
+import org.apache.cassandra.sidecar.tasks.ScheduleDecision;
 
 import static org.apache.cassandra.sidecar.coordination.ClusterLeaseClaimTask.MINIMUM_DELAY_MILLIS;
 import static org.assertj.core.api.Assertions.assertThat;
@@ -55,7 +55,7 @@ class ClusterLeaseClaimTaskTest
         ClusterLeaseClaimTask task = new ClusterLeaseClaimTask(mock(Vertx.class), serviceConfiguration, mockElectorateMembership,
                                                                mock(SidecarLeaseDatabaseAccessor.class), new ClusterLease(),
                                                                mock(SidecarMetrics.class, RETURNS_DEEP_STUBS));
-        assertThat(task.shouldSkip()).isTrue();
+        assertThat(task.scheduleDecision()).isEqualTo(ScheduleDecision.SKIP);
         // avoid expensive calls
         verifyNoInteractions(mockElectorateMembership);
     }
@@ -71,9 +71,9 @@ class ClusterLeaseClaimTaskTest
                                                                mock(SidecarLeaseDatabaseAccessor.class), clusterLease,
                                                                mock(SidecarMetrics.class, RETURNS_DEEP_STUBS));
 
-        assertThat(task.shouldSkip()).isTrue();
-        assertThat(clusterLease.executionDetermination()).as("Skip when not a member of the electorate")
-                                                         .isEqualTo(ExecutionDetermination.SKIP_EXECUTION);
+        assertThat(task.scheduleDecision()).isEqualTo(ScheduleDecision.SKIP);
+        assertThat(clusterLease.toScheduleDecision()).as("Skip when not a member of the electorate")
+                                                     .isEqualTo(ScheduleDecision.SKIP);
         verify(mockElectorateMembership, times(1)).isMember();
     }
 
@@ -87,7 +87,7 @@ class ClusterLeaseClaimTaskTest
                                                                mock(SidecarLeaseDatabaseAccessor.class), new ClusterLease(),
                                                                mock(SidecarMetrics.class, RETURNS_DEEP_STUBS));
 
-        assertThat(task.shouldSkip()).isFalse();
+        assertThat(task.scheduleDecision()).isEqualTo(ScheduleDecision.EXECUTE);
         verify(mockElectorateMembership, times(1)).isMember();
     }
 
diff --git a/server/src/test/java/org/apache/cassandra/sidecar/db/SidecarSchemaTest.java b/server/src/test/java/org/apache/cassandra/sidecar/db/SidecarSchemaTest.java
index 34873154..f44d4f17 100644
--- a/server/src/test/java/org/apache/cassandra/sidecar/db/SidecarSchemaTest.java
+++ b/server/src/test/java/org/apache/cassandra/sidecar/db/SidecarSchemaTest.java
@@ -56,7 +56,6 @@ import org.apache.cassandra.sidecar.coordination.ClusterLease;
 import org.apache.cassandra.sidecar.db.schema.SidecarSchema;
 import org.apache.cassandra.sidecar.server.MainModule;
 import org.apache.cassandra.sidecar.server.Server;
-import org.apache.cassandra.sidecar.tasks.ExecutionDetermination;
 import org.mockito.stubbing.Answer;
 
 import static org.assertj.core.api.Assertions.assertThat;
@@ -269,7 +268,7 @@ public class SidecarSchemaTest
         @Singleton
         public ClusterLease clusterLease()
         {
-            return new ClusterLease(ExecutionDetermination.EXECUTE);
+            return new ClusterLease(ClusterLease.Ownership.CLAIMED);
         }
     }
 }
diff --git a/server/src/test/java/org/apache/cassandra/sidecar/metrics/SchemaMetricsTest.java b/server/src/test/java/org/apache/cassandra/sidecar/metrics/SchemaMetricsTest.java
index 3fa67d0b..de258d34 100644
--- a/server/src/test/java/org/apache/cassandra/sidecar/metrics/SchemaMetricsTest.java
+++ b/server/src/test/java/org/apache/cassandra/sidecar/metrics/SchemaMetricsTest.java
@@ -47,7 +47,6 @@ import org.apache.cassandra.sidecar.db.schema.SidecarSchema;
 import org.apache.cassandra.sidecar.exceptions.SidecarSchemaModificationException;
 import org.apache.cassandra.sidecar.server.MainModule;
 import org.apache.cassandra.sidecar.server.Server;
-import org.apache.cassandra.sidecar.tasks.ExecutionDetermination;
 
 import static org.apache.cassandra.sidecar.AssertionUtils.loopAssert;
 import static org.apache.cassandra.sidecar.utils.TestMetricUtils.registry;
@@ -143,7 +142,7 @@ class SchemaMetricsTest
         @Singleton
         public ClusterLease clusterLease()
         {
-            return new ClusterLease(ExecutionDetermination.EXECUTE);
+            return new ClusterLease(ClusterLease.Ownership.CLAIMED);
         }
     }
 }
diff --git a/server/src/test/java/org/apache/cassandra/sidecar/restore/RestoreJobDiscovererTest.java b/server/src/test/java/org/apache/cassandra/sidecar/restore/RestoreJobDiscovererTest.java
index 4cd6e529..fb6dba17 100644
--- a/server/src/test/java/org/apache/cassandra/sidecar/restore/RestoreJobDiscovererTest.java
+++ b/server/src/test/java/org/apache/cassandra/sidecar/restore/RestoreJobDiscovererTest.java
@@ -48,6 +48,7 @@ import org.apache.cassandra.sidecar.metrics.MetricRegistryFactory;
 import org.apache.cassandra.sidecar.metrics.SidecarMetrics;
 import org.apache.cassandra.sidecar.metrics.SidecarMetricsImpl;
 import org.apache.cassandra.sidecar.tasks.PeriodicTaskExecutor;
+import org.apache.cassandra.sidecar.tasks.ScheduleDecision;
 import org.apache.cassandra.sidecar.utils.InstanceMetadataFetcher;
 import org.mockito.ArgumentCaptor;
 
@@ -197,7 +198,7 @@ class RestoreJobDiscovererTest
 
         // Execution 3
         // shouldSkip always returns false
-        assertThat(loop.shouldSkip()).isFalse();
+        assertThat(loop.scheduleDecision()).isEqualTo(ScheduleDecision.EXECUTE);
 
         // Execution 4
         UUID newJobId2 = UUIDs.timeBased();
@@ -230,7 +231,7 @@ class RestoreJobDiscovererTest
     void testSkipExecuteWhenSidecarSchemaIsNotInitialized()
     {
         when(sidecarSchema.isInitialized()).thenReturn(false);
-        assertThat(loop.shouldSkip()).isTrue();
+        assertThat(loop.scheduleDecision()).isEqualTo(ScheduleDecision.SKIP);
     }
 
     @Test
diff --git a/server/src/test/java/org/apache/cassandra/sidecar/restore/RestoreProcessorTest.java b/server/src/test/java/org/apache/cassandra/sidecar/restore/RestoreProcessorTest.java
index a3aa1a6a..fa6f0ae6 100644
--- a/server/src/test/java/org/apache/cassandra/sidecar/restore/RestoreProcessorTest.java
+++ b/server/src/test/java/org/apache/cassandra/sidecar/restore/RestoreProcessorTest.java
@@ -44,6 +44,7 @@ import org.apache.cassandra.sidecar.metrics.instance.InstanceMetricsImpl;
 import org.apache.cassandra.sidecar.metrics.instance.InstanceRestoreMetrics;
 import org.apache.cassandra.sidecar.server.MainModule;
 import org.apache.cassandra.sidecar.tasks.PeriodicTaskExecutor;
+import org.apache.cassandra.sidecar.tasks.ScheduleDecision;
 import org.mockito.Mockito;
 
 import static org.apache.cassandra.sidecar.AssertionUtils.loopAssert;
@@ -147,7 +148,7 @@ class RestoreProcessorTest
     {
         when(sidecarSchema.isInitialized()).thenReturn(false);
 
-        assertThat(processor.shouldSkip()).isTrue();
+        assertThat(processor.scheduleDecision()).isEqualTo(ScheduleDecision.SKIP);
         assertThat(processor.activeSlices()).isZero();
 
         CountDownLatch latch = new CountDownLatch(1);
diff --git a/server/src/test/java/org/apache/cassandra/sidecar/tasks/PeriodicTaskExecutorTest.java b/server/src/test/java/org/apache/cassandra/sidecar/tasks/PeriodicTaskExecutorTest.java
index 0eb18e56..fa633897 100644
--- a/server/src/test/java/org/apache/cassandra/sidecar/tasks/PeriodicTaskExecutorTest.java
+++ b/server/src/test/java/org/apache/cassandra/sidecar/tasks/PeriodicTaskExecutorTest.java
@@ -18,6 +18,10 @@
 
 package org.apache.cassandra.sidecar.tasks;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -25,6 +29,8 @@ import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 
 import com.google.common.util.concurrent.Uninterruptibles;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
 import io.vertx.core.Promise;
@@ -34,6 +40,7 @@ import org.apache.cassandra.sidecar.config.yaml.ServiceConfigurationImpl;
 import org.apache.cassandra.sidecar.coordination.ClusterLease;
 import org.apache.cassandra.sidecar.coordination.ExecuteOnClusterLeaseholderOnly;
 
+import static org.apache.cassandra.sidecar.AssertionUtils.loopAssert;
 import static org.assertj.core.api.Assertions.assertThat;
 
 /**
@@ -41,14 +48,30 @@ import static org.assertj.core.api.Assertions.assertThat;
  */
 class PeriodicTaskExecutorTest
 {
-    Vertx vertx = Vertx.vertx();
-    ExecutorPools executorPools = new ExecutorPools(vertx, new ServiceConfigurationImpl());
+    private final Vertx vertx = Vertx.vertx();
+    private final ExecutorPools executorPools = new ExecutorPools(vertx, new ServiceConfigurationImpl());
+    private final ClusterLease clusterLease = new ClusterLease();
+    private PeriodicTaskExecutor taskExecutor;
+
+    @BeforeEach
+    void beforeEach()
+    {
+        taskExecutor = new PeriodicTaskExecutor(executorPools, clusterLease);
+    }
+
+    @AfterEach
+    void afterEach()
+    {
+        clusterLease.setOwnershipTesting(ClusterLease.Ownership.INDETERMINATE);
+        if (taskExecutor != null)
+        {
+            taskExecutor.close(Promise.promise());
+        }
+    }
 
     @Test
     void testLoopFailure()
     {
-        PeriodicTaskExecutor taskExecutor = new PeriodicTaskExecutor(executorPools);
-
         int totalFailures = 5;
         AtomicInteger failuresCount = new AtomicInteger(0);
         CountDownLatch closeLatch = new CountDownLatch(1);
@@ -58,7 +81,7 @@ class PeriodicTaskExecutorTest
             @Override
             public long delay()
             {
-                return 20;
+                return 1;
             }
 
             @Override
@@ -84,33 +107,163 @@ class PeriodicTaskExecutorTest
     }
 
     @Test
-    void testPeriodicTaskOnNonExecutor()
+    void testPeriodicTaskExecutionShouldEnsureMemoryConsistency()
     {
-        CountDownLatch latch = new CountDownLatch(1);
-        SimulatedTask taskThatRunsOnNonExecutor = new SimulatedTask(latch);
-        PeriodicTaskExecutor taskExecutorNeverExecute = new PeriodicTaskExecutor(executorPools, new ClusterLease(ExecutionDetermination.SKIP_EXECUTION));
+        // starts 10 concurrent incremental tasks, and
+        // assert that for each task, the non-thread-safe value update has
+        // the same effect of the thread-safe value update
+        List<IncrementPeriodicTask> tasks = new ArrayList<>(10);
+        for (int i = 0; i < 10; i++)
+        {
+            IncrementPeriodicTask incTask = new IncrementPeriodicTask("task" + i);
+            tasks.add(incTask);
+            taskExecutor.schedule(incTask);
+        }
+        Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
+        tasks.forEach(taskExecutor::unschedule);
+        // wait until unschedule is complete
+        loopAssert(1, () -> assertThat(taskExecutor.timerIds()).isEmpty());
+        loopAssert(1, () -> assertThat(taskExecutor.poisonPilledTasks()).isEmpty());
+        tasks.forEach(incTask -> assertThat(incTask.atomicValue.get())
+                                 .describedAs(incTask.name() + " should have same value")
+                                 .isEqualTo(incTask.value)
+                                 .isPositive());
+    }
 
-        taskExecutorNeverExecute.schedule(taskThatRunsOnNonExecutor);
-        assertThat(Uninterruptibles.awaitUninterruptibly(latch, 30, TimeUnit.SECONDS)).isTrue();
-        assertThat(taskThatRunsOnNonExecutor.executionCount.get()).isEqualTo(0);
-        assertThat(taskThatRunsOnNonExecutor.initialDelayCount.get()).isEqualTo(1);
+    @Test
+    void testScheduleSameTaskMultipleTimes()
+    {
+        AtomicInteger counter = new AtomicInteger(0);
+        // each time, the task is executed, the counter is incremented by 1.
+        // The task has the initial delay of 0, but the periodic interval is 1 day to ensure it is executed only once during the test runtime
+        PeriodicTask task = createSimplePeriodicTask("task", 0, TimeUnit.DAYS.toMillis(1), counter::incrementAndGet);
+        // schedule the same task for 10 times, only the first schedule goes through
+        for (int i = 0; i < 10; i++)
+        {
+            taskExecutor.schedule(task);
+        }
+        // in case that the test environment gets slow and the counter update is delayed
+        loopAssert(1, () -> assertThat(counter.get()).isEqualTo(1));
+    }
+
+    @Test
+    void testUnscheduleShouldStopExecution()
+    {
+        // the first run starts immediately, and the subsequent delay is 1 millis
+        testUnscheduleShouldStopExecution("task1", 0, 1);
+        // the first run starts immediately, and the subsequent runs also starts immediately
+        testUnscheduleShouldStopExecution("task2", 0, 0);
+        // tasks are scheduled with delay 1 millis
+        testUnscheduleShouldStopExecution("task3", 1, 1);
+    }
+
+    @Test
+    void testUnscheduleNonExistTaskHasNoEffect()
+    {
+        PeriodicTask notScheduled = createSimplePeriodicTask("simple task", 1, () -> {});
+        taskExecutor.unschedule(notScheduled);
+        assertThat(taskExecutor.poisonPilledTasks()).isEmpty();
+        assertThat(taskExecutor.timerIds()).isEmpty();
+    }
+
+    @Test
+    void testRescheduleNonExistTaskShouldNotClose()
+    {
+        AtomicBoolean isCloseCalled = new AtomicBoolean(false);
+        CountDownLatch taskScheduled = new CountDownLatch(1);
+        PeriodicTask task = new PeriodicTask()
+        {
+            @Override
+            public long delay()
+            {
+                return 1;
+            }
+
+            @Override
+            public void execute(Promise<Void> promise)
+            {
+                taskScheduled.countDown();
+                promise.complete();
+            }
+
+            @Override
+            public void close()
+            {
+                isCloseCalled.set(true);
+            }
+        };
+
+        // for such unscheduled task, reschedule has the same effect as schedule.
+        taskExecutor.reschedule(task);
+        Uninterruptibles.awaitUninterruptibly(taskScheduled);
+        assertThat(isCloseCalled.get())
+        .describedAs("When rescheduling an unscheduled task, the close method of the task should not be called")
+        .isFalse();
     }
 
     @Test
-    void testPeriodicTaskOnExecutor()
+    void testReschedule()
+    {
+        AtomicInteger counter1 = new AtomicInteger(0);
+        AtomicInteger counter2 = new AtomicInteger(0);
+        int totalSamples = 10;
+        Set<Integer> counterValues = ConcurrentHashMap.newKeySet(totalSamples);
+
+        CountDownLatch latch1 = new CountDownLatch(1);
+        // The periodic task increment counter1 initially, then switches to increment counter 2 once it is rescheduled
+        PeriodicTask task = createSimplePeriodicTask("simple periodic task", 1, () -> {
+            if (latch1.getCount() > 0)
+            {
+                counter1.incrementAndGet();
+            }
+            else
+            {
+                counterValues.add(counter2.incrementAndGet());
+            }
+            Uninterruptibles.awaitUninterruptibly(latch1);
+        });
+        taskExecutor.schedule(task);
+        loopAssert(1, () -> assertThat(counter1.get()).isEqualTo(1));
+        assertThat(counterValues).isEmpty();
+        taskExecutor.reschedule(task);
+        assertThat(counter1.get()).isEqualTo(1);
+        assertThat(counterValues).isEmpty();
+        latch1.countDown();
+        // After getting rescheduled, it should produce unique values from counter2 and store in counterValues
+        assertThat(counter1.get()).isEqualTo(1);
+        loopAssert(2, () -> assertThat(counterValues.size()).isGreaterThanOrEqualTo(totalSamples));
+        taskExecutor.unschedule(task);
+    }
+
+    @Test
+    void testLeaseholderOnlyTaskOnNonClusterLeaseholder()
+    {
+        CountDownLatch skipLatch = new CountDownLatch(1);
+        SimulatedTask task = new SimulatedTask(skipLatch, null);
+        clusterLease.setOwnershipTesting(ClusterLease.Ownership.LOST);
+
+        taskExecutor.schedule(task);
+        assertThat(Uninterruptibles.awaitUninterruptibly(skipLatch, 30, TimeUnit.SECONDS)).isTrue();
+        assertThat(task.shouldSkipCount.get()).isEqualTo(5);
+        assertThat(task.executionCount.get()).isEqualTo(0);
+        assertThat(task.initialDelayCount.get()).isEqualTo(1);
+    }
+
+    @Test
+    void testLeaseholderOnlyTaskOnClusterLeaseholder()
     {
         CountDownLatch latch = new CountDownLatch(1);
-        SimulatedTask taskThatRunsOnExecutor = new SimulatedTask(latch);
-        PeriodicTaskExecutor taskExecutorAlwaysExecute = new PeriodicTaskExecutor(executorPools, new ClusterLease(ExecutionDetermination.EXECUTE));
+        SimulatedTask task = new SimulatedTask(latch);
+        clusterLease.setOwnershipTesting(ClusterLease.Ownership.CLAIMED);
 
-        taskExecutorAlwaysExecute.schedule(taskThatRunsOnExecutor);
+        taskExecutor.schedule(task);
         assertThat(Uninterruptibles.awaitUninterruptibly(latch, 30, TimeUnit.SECONDS)).isTrue();
-        assertThat(taskThatRunsOnExecutor.executionCount.get()).isEqualTo(5);
-        assertThat(taskThatRunsOnExecutor.initialDelayCount.get()).isEqualTo(1);
+        assertThat(task.executionCount.get()).isEqualTo(5);
+        assertThat(task.initialDelayCount.get()).isEqualTo(1);
     }
 
     @Test
-    void testPeriodicTaskIsRescheduledWhenIndeterminate()
+    void testLeaseholderOnlyTaskIsRescheduledWhenIndeterminate()
     {
         CountDownLatch latch = new CountDownLatch(1);
         SimulatedTask taskThatRunsOnExecutor = new SimulatedTask(latch);
@@ -124,6 +277,26 @@ class PeriodicTaskExecutorTest
         assertThat(taskThatRunsOnExecutor.initialDelayCount.get()).isEqualTo(2);
         // and we only actually execute 4 times, since the first time was rescheduled
         assertThat(taskThatRunsOnExecutor.executionCount.get()).isEqualTo(4);
+        taskExecutor.close(Promise.promise());
+    }
+
+    private void testUnscheduleShouldStopExecution(String taskName, long taskInitialDelay, long taskDelayMillis)
+    {
+        AtomicInteger counter = new AtomicInteger(0);
+        CountDownLatch testFinish = new CountDownLatch(1);
+        PeriodicTask task = createSimplePeriodicTask(taskName, taskInitialDelay, taskDelayMillis, () -> {
+            counter.incrementAndGet();
+            testFinish.countDown();
+        });
+        taskExecutor.schedule(task);
+        Uninterruptibles.awaitUninterruptibly(testFinish);
+        taskExecutor.unschedule(task);
+        loopAssert(1,
+                   () -> assertThat(taskExecutor.poisonPilledTasks()).isEmpty());
+        loopAssert(1,
+                   () -> assertThat(taskExecutor.timerIds())
+                         .describedAs("Execution should stop after unschedule is called")
+                         .isEmpty());
     }
 
     static class TestClusterLease extends ClusterLease
@@ -136,14 +309,14 @@ class PeriodicTaskExecutorTest
         }
 
         @Override
-        public ExecutionDetermination executionDetermination()
+        public ScheduleDecision toScheduleDecision()
         {
-            ExecutionDetermination executionDetermination = delegate.get().executionDetermination();
-            if (executionDetermination == ExecutionDetermination.INDETERMINATE)
+            ScheduleDecision scheduleDecision = delegate.get().toScheduleDecision();
+            if (scheduleDecision == ScheduleDecision.RESCHEDULE)
             {
-                delegate.set(new ClusterLease(ExecutionDetermination.EXECUTE));
+                delegate.set(new ClusterLease(Ownership.CLAIMED));
             }
-            return executionDetermination;
+            return scheduleDecision;
         }
 
         @Override
@@ -158,11 +331,25 @@ class PeriodicTaskExecutorTest
         final AtomicInteger executionCount = new AtomicInteger(0);
         final AtomicInteger shouldSkipCount = new AtomicInteger(0);
         final AtomicInteger initialDelayCount = new AtomicInteger(0);
-        private final CountDownLatch latch;
+        private final CountDownLatch shouldSkipLatch;
+        private final CountDownLatch executeLatch;
+        private PeriodicTaskExecutor executor;
+
+        SimulatedTask(CountDownLatch executeLatch)
+        {
+            this(new CountDownLatch(1), executeLatch);
+        }
+
+        SimulatedTask(CountDownLatch shouldSkipLatch, CountDownLatch executeLatch)
+        {
+            this.shouldSkipLatch = shouldSkipLatch;
+            this.executeLatch = executeLatch;
+        }
 
-        SimulatedTask(CountDownLatch latch)
+        @Override
+        public void registerPeriodicTaskExecutor(PeriodicTaskExecutor executor)
         {
-            this.latch = latch;
+            this.executor = executor;
         }
 
         @Override
@@ -179,20 +366,100 @@ class PeriodicTaskExecutorTest
         }
 
         @Override
-        public boolean shouldSkip()
+        public ScheduleDecision scheduleDecision()
         {
             if (shouldSkipCount.incrementAndGet() == 5)
             {
-                latch.countDown();
+                shouldSkipLatch.countDown();
+                if (executeLatch == null)
+                {
+                    // unschedule to avoid flakiness
+                    executor.unschedule(this);
+                }
             }
-            return false;
+            return ScheduleDecision.EXECUTE;
         }
 
         @Override
         public void execute(Promise<Void> promise)
         {
             executionCount.incrementAndGet();
+            if (shouldSkipCount.get() == 5)
+            {
+                executeLatch.countDown();
+                // unschedule to avoid flakiness
+                executor.unschedule(this);
+            }
             promise.complete();
         }
     }
+
+    // a stateful periodic task that increments its value every run
+    private static class IncrementPeriodicTask implements PeriodicTask
+    {
+        private final String name;
+        int value = 0;
+        AtomicInteger atomicValue = new AtomicInteger(0);
+
+        IncrementPeriodicTask(String name)
+        {
+            this.name = name;
+        }
+
+        @Override
+        public String name()
+        {
+            return name;
+        }
+
+        @Override
+        public long delay()
+        {
+            return 1;
+        }
+
+        @Override
+        public void execute(Promise<Void> promise)
+        {
+            value += 1;
+            atomicValue.incrementAndGet();
+            promise.complete();
+        }
+    }
+
+    static PeriodicTask createSimplePeriodicTask(String name, long delayMillis, Runnable taskBody)
+    {
+        return createSimplePeriodicTask(name, delayMillis, delayMillis, taskBody);
+    }
+
+    static PeriodicTask createSimplePeriodicTask(String name, long initialDelayMillis, long delayMillis, Runnable taskBody)
+    {
+        return new PeriodicTask()
+        {
+            @Override
+            public String name()
+            {
+                return name;
+            }
+
+            @Override
+            public long initialDelay()
+            {
+                return initialDelayMillis;
+            }
+
+            @Override
+            public long delay()
+            {
+                return delayMillis;
+            }
+
+            @Override
+            public void execute(Promise<Void> promise)
+            {
+                taskBody.run();
+                promise.complete();
+            }
+        };
+    }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org