You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ad...@apache.org on 2024/02/08 11:42:34 UTC

(cassandra) branch cassandra-4.0 updated: Ensure that repair doesn't exceed repair_session_space by running limiting validation parallelism

This is an automated email from the ASF dual-hosted git repository.

adelapena pushed a commit to branch cassandra-4.0
in repository https://gitbox.apache.org/repos/asf/cassandra.git


The following commit(s) were added to refs/heads/cassandra-4.0 by this push:
     new 505f5af645 Ensure that repair doesn't exceed repair_session_space by running limiting validation parallelism
505f5af645 is described below

commit 505f5af645c1712c3da42d98d005276396ff2667
Author: Andrés de la Peña <a....@gmail.com>
AuthorDate: Tue Feb 6 22:17:14 2024 +0000

    Ensure that repair doesn't exceed repair_session_space by running limiting validation parallelism
    
    patch by Andrés de la Peña; reviewed by David Capwell for CASSANDRA-19336
    
    Co-authored-by: Andrés de la Peña <a....@gmail.com>
    Co-authored-by: David Capwell <dc...@apache.org>
---
 CHANGES.txt                                        |   1 +
 conf/cassandra.yaml                                |  36 +++++-
 src/java/org/apache/cassandra/config/Config.java   |   2 +
 .../cassandra/config/DatabaseDescriptor.java       |  10 ++
 .../org/apache/cassandra/repair/RepairJob.java     |  95 +++++++-------
 .../apache/cassandra/repair/RepairRunnable.java    |  10 +-
 .../org/apache/cassandra/repair/RepairSession.java |   3 +
 .../org/apache/cassandra/repair/Scheduler.java     | 142 +++++++++++++++++++++
 .../cassandra/service/ActiveRepairService.java     |   5 +-
 .../test/OptimiseStreamsRepairTest.java            |  18 +--
 .../repair/ConcurrentValidationRequestsTest.java   | 136 ++++++++++++++++++++
 .../org/apache/cassandra/repair/RepairJobTest.java |   3 +-
 .../apache/cassandra/repair/RepairSessionTest.java |   2 +-
 13 files changed, 392 insertions(+), 71 deletions(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index c02e1ffc7d..baeb9b7f61 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 4.0.13
+ * Add new concurrent_merkle_tree_requests config property to prevent OOM during multi-range and/or multi-table repairs (CASSANDRA-19336)
 Merged from 3.11:
 Merged from 3.0:
  * Backport CASSANDRA-16418 to 3.x (CASSANDRA-18824)
diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml
index 3e4da52f5f..7f162749d2 100644
--- a/conf/cassandra.yaml
+++ b/conf/cassandra.yaml
@@ -553,17 +553,41 @@ concurrent_materialized_view_writes: 32
 #    off heap objects
 memtable_allocation_type: heap_buffers
 
-# Limit memory usage for Merkle tree calculations during repairs. The default
-# is 1/16th of the available heap. The main tradeoff is that smaller trees
-# have less resolution, which can lead to over-streaming data. If you see heap
-# pressure during repairs, consider lowering this, but you cannot go below
-# one megabyte. If you see lots of over-streaming, consider raising
-# this or using subrange repair.
+# Limit memory usage for Merkle tree calculations during repairs of a certain
+# table and common token range. Repair commands targetting multiple tables or
+# virtual nodes can exceed this limit depending on concurrent_merkle_tree_requests.
+#
+# The default is 1/16th of the available heap. The main tradeoff is that
+# smaller trees have less resolution, which can lead to over-streaming data.
+# If you see heap pressure during repairs, consider lowering this, but you
+# cannot go below one mebibyte. If you see lots of over-streaming, consider
+# raising this or using subrange repair.
 #
 # For more details see https://issues.apache.org/jira/browse/CASSANDRA-14096.
 #
 # repair_session_space_in_mb:
 
+# The number of simultaneous Merkle tree requests during repairs that can
+# be performed by a repair command. The size of each validation request is
+# limited by the repair_session_space property, so setting this to 1 will make
+# sure that a repair command doesn't exceed that limit, even if the repair
+# command is repairing multiple tables or multiple virtual nodes.
+#
+# There isn't a limit by default for backwards compatibility, but this can
+# produce OOM for  commands repairing multiple tables or multiple virtual nodes.
+# A limit of just 1 simultaneous Merkle tree request is generally recommended
+# with no virtual nodes so repair_session_space, and thereof the Merkle tree
+# resolution, can be high. For virtual nodes a value of 1 with the default
+# repair_session_space value will produce higher resolution Merkle trees
+# at the expense of speed. Alternatively, when working with virtual nodes it
+# can make sense to reduce the repair_session_space and increase the value of
+# concurrent_merkle_tree_requests because each range will contain fewer data.
+#
+# For more details see https://issues.apache.org/jira/browse/CASSANDRA-19336.
+#
+# A zero value means no limit.
+# concurrent_merkle_tree_requests: 0
+
 # Total space to use for commit logs on disk.
 #
 # If space gets above this value, Cassandra will flush every dirty CF
diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java
index f8d8d46db8..d7517124df 100644
--- a/src/java/org/apache/cassandra/config/Config.java
+++ b/src/java/org/apache/cassandra/config/Config.java
@@ -136,6 +136,8 @@ public class Config
 
     public volatile long repair_request_timeout_in_ms = TimeUnit.MILLISECONDS.convert(1, TimeUnit.MINUTES);
 
+    public volatile int concurrent_merkle_tree_requests = 0;
+
     public volatile boolean use_offheap_merkle_trees = true;
 
     public int storage_port = 7000;
diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index 8893f58e56..377f67117d 100644
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@ -2880,6 +2880,16 @@ public class DatabaseDescriptor
         conf.repair_session_space_in_mb = sizeInMegabytes;
     }
 
+    public static int getConcurrentMerkleTreeRequests()
+    {
+        return conf.concurrent_merkle_tree_requests;
+    }
+
+    public static void setConcurrentMerkleTreeRequests(int value)
+    {
+        conf.concurrent_merkle_tree_requests = value;
+    }
+
     public static Float getMemtableCleanupThreshold()
     {
         return conf.memtable_cleanup_threshold;
diff --git a/src/java/org/apache/cassandra/repair/RepairJob.java b/src/java/org/apache/cassandra/repair/RepairJob.java
index a9ac6af98e..95fe84f550 100644
--- a/src/java/org/apache/cassandra/repair/RepairJob.java
+++ b/src/java/org/apache/cassandra/repair/RepairJob.java
@@ -90,7 +90,7 @@ public class RepairJob extends AbstractFuture<RepairResult> implements Runnable
 
     /**
      * Runs repair job.
-     *
+     * <p/>
      * This sets up necessary task and runs them on given {@code taskExecutor}.
      * After submitting all tasks, waits until validation with replica completes.
      */
@@ -103,11 +103,10 @@ public class RepairJob extends AbstractFuture<RepairResult> implements Runnable
         List<InetAddressAndPort> allEndpoints = new ArrayList<>(session.commonRange.endpoints);
         allEndpoints.add(FBUtilities.getBroadcastAddressAndPort());
 
-        ListenableFuture<List<TreeResponse>> validations;
         // Create a snapshot at all nodes unless we're using pure parallel repairs
+        ListenableFuture<List<InetAddressAndPort>> allSnapshotTasks;
         if (parallelismDegree != RepairParallelism.PARALLEL)
         {
-            ListenableFuture<List<InetAddressAndPort>> allSnapshotTasks;
             if (session.isIncremental)
             {
                 // consistent repair does it's own "snapshotting"
@@ -125,29 +124,17 @@ public class RepairJob extends AbstractFuture<RepairResult> implements Runnable
                 }
                 allSnapshotTasks = Futures.allAsList(snapshotTasks);
             }
-
-            // When all snapshot complete, send validation requests
-            validations = Futures.transformAsync(allSnapshotTasks, new AsyncFunction<List<InetAddressAndPort>, List<TreeResponse>>()
-            {
-                public ListenableFuture<List<TreeResponse>> apply(List<InetAddressAndPort> endpoints)
-                {
-                    if (parallelismDegree == RepairParallelism.SEQUENTIAL)
-                        return sendSequentialValidationRequest(endpoints);
-                    else
-                        return sendDCAwareValidationRequest(endpoints);
-                }
-            }, taskExecutor);
         }
         else
         {
-            // If not sequential, just send validation request to all replica
-            validations = sendValidationRequest(allEndpoints);
+            allSnapshotTasks = null;
         }
 
-        // When all validations complete, submit sync tasks
-        ListenableFuture<List<SyncStat>> syncResults = Futures.transformAsync(validations,
-                                                                              session.optimiseStreams && !session.pullRepair ? this::optimisedSyncing : this::standardSyncing,
-                                                                              taskExecutor);
+        // Run validations and the creation of sync tasks in the scheduler, so it can limit the number of Merkle trees
+        // that there are in memory at once. When all validations complete, submit sync tasks out of the scheduler.
+        ListenableFuture<List<SyncStat>> syncResults = Futures.transformAsync(
+                session.validationScheduler.schedule(() -> createSyncTasks(allSnapshotTasks, allEndpoints), taskExecutor),
+                this::executeTasks, taskExecutor);
 
         // When all sync complete, set the final result
         Futures.addCallback(syncResults, new FutureCallback<List<SyncStat>>()
@@ -183,21 +170,44 @@ public class RepairJob extends AbstractFuture<RepairResult> implements Runnable
         }, taskExecutor);
     }
 
+    private ListenableFuture<List<SyncTask>> createSyncTasks(ListenableFuture<List<InetAddressAndPort>> allSnapshotTasks, List<InetAddressAndPort> allEndpoints)
+    {
+        ListenableFuture<List<TreeResponse>> validations;
+        if (allSnapshotTasks != null)
+        {
+            // When all snapshot complete, send validation requests
+            validations = Futures.transformAsync(allSnapshotTasks, endpoints -> {
+                if (parallelismDegree == RepairParallelism.SEQUENTIAL)
+                    return sendSequentialValidationRequest(endpoints);
+                else
+                    return sendDCAwareValidationRequest(endpoints);
+            }, taskExecutor);
+        }
+        else
+        {
+            // If not sequential, just send validation request to all replica
+            validations = sendValidationRequest(allEndpoints);
+        }
+
+        return Futures.transform(validations,
+                                 session.optimiseStreams && !session.pullRepair ? this::optimisedSyncing : this::standardSyncing,
+                                 taskExecutor);
+    }
+
     private boolean isTransient(InetAddressAndPort ep)
     {
         return session.commonRange.transEndpoints.contains(ep);
     }
 
-    private ListenableFuture<List<SyncStat>> standardSyncing(List<TreeResponse> trees)
+    private List<SyncTask> standardSyncing(List<TreeResponse> trees)
     {
-        List<SyncTask> syncTasks = createStandardSyncTasks(desc,
-                                                           trees,
-                                                           FBUtilities.getLocalAddressAndPort(),
-                                                           this::isTransient,
-                                                           session.isIncremental,
-                                                           session.pullRepair,
-                                                           session.previewKind);
-        return executeTasks(syncTasks);
+        return createStandardSyncTasks(desc,
+                                       trees,
+                                       FBUtilities.getLocalAddressAndPort(),
+                                       this::isTransient,
+                                       session.isIncremental,
+                                       session.pullRepair,
+                                       session.previewKind);
     }
 
     static List<SyncTask> createStandardSyncTasks(RepairJobDesc desc,
@@ -267,17 +277,15 @@ public class RepairJob extends AbstractFuture<RepairResult> implements Runnable
         return syncTasks;
     }
 
-    private ListenableFuture<List<SyncStat>> optimisedSyncing(List<TreeResponse> trees)
+    private List<SyncTask> optimisedSyncing(List<TreeResponse> trees)
     {
-        List<SyncTask> syncTasks = createOptimisedSyncingSyncTasks(desc,
-                                                                   trees,
-                                                                   FBUtilities.getLocalAddressAndPort(),
-                                                                   this::isTransient,
-                                                                   this::getDC,
-                                                                   session.isIncremental,
-                                                                   session.previewKind);
-
-        return executeTasks(syncTasks);
+        return createOptimisedSyncingSyncTasks(desc,
+                                               trees,
+                                               FBUtilities.getLocalAddressAndPort(),
+                                               this::isTransient,
+                                               this::getDC,
+                                               session.isIncremental,
+                                               session.previewKind);
     }
 
     @SuppressWarnings("UnstableApiUsage")
@@ -448,12 +456,7 @@ public class RepairJob extends AbstractFuture<RepairResult> implements Runnable
         for (InetAddressAndPort endpoint : endpoints)
         {
             String dc = DatabaseDescriptor.getEndpointSnitch().getDatacenter(endpoint);
-            Queue<InetAddressAndPort> queue = requestsByDatacenter.get(dc);
-            if (queue == null)
-            {
-                queue = new LinkedList<>();
-                requestsByDatacenter.put(dc, queue);
-            }
+            Queue<InetAddressAndPort> queue = requestsByDatacenter.computeIfAbsent(dc, k -> new LinkedList<>());
             queue.add(endpoint);
         }
 
diff --git a/src/java/org/apache/cassandra/repair/RepairRunnable.java b/src/java/org/apache/cassandra/repair/RepairRunnable.java
index 48a7e36adc..14fab59741 100644
--- a/src/java/org/apache/cassandra/repair/RepairRunnable.java
+++ b/src/java/org/apache/cassandra/repair/RepairRunnable.java
@@ -114,6 +114,7 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier
 
     private static final AtomicInteger threadCounter = new AtomicInteger(1);
     private final AtomicReference<Throwable> firstError = new AtomicReference<>(null);
+    private final Scheduler validationScheduler;
 
     private TraceState traceState;
 
@@ -123,6 +124,7 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier
         this.cmd = cmd;
         this.options = options;
         this.keyspace = keyspace;
+        this.validationScheduler = Scheduler.build(DatabaseDescriptor.getConcurrentMerkleTreeRequests());
 
         this.tag = "repair:" + cmd;
         // get valid column families, calculate neighbors, validation, prepare for repair + number of ranges to repair
@@ -437,7 +439,7 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier
         ListeningExecutorService executor = createExecutor();
 
         // Setting the repairedAt time to UNREPAIRED_SSTABLE causes the repairedAt times to be preserved across streamed sstables
-        final ListenableFuture<List<RepairSessionResult>> allSessions = submitRepairSessions(parentSession, false, executor, commonRanges, cfnames);
+        final ListenableFuture<List<RepairSessionResult>> allSessions = submitRepairSessions(parentSession, false, executor, validationScheduler, commonRanges, cfnames);
 
         // After all repair sessions completes(successful or not),
         // run anticompaction if necessary and send finish notice back to client
@@ -497,7 +499,7 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier
         CoordinatorSession coordinatorSession = ActiveRepairService.instance.consistent.coordinated.registerSession(parentSession, allParticipants, neighborsAndRanges.shouldExcludeDeadParticipants);
         ListeningExecutorService executor = createExecutor();
         AtomicBoolean hasFailure = new AtomicBoolean(false);
-        ListenableFuture repairResult = coordinatorSession.execute(() -> submitRepairSessions(parentSession, true, executor, allRanges, cfnames),
+        ListenableFuture repairResult = coordinatorSession.execute(() -> submitRepairSessions(parentSession, true, executor, validationScheduler, allRanges, cfnames),
                                                                    hasFailure);
         Collection<Range<Token>> ranges = new HashSet<>();
         for (Collection<Range<Token>> range : Iterables.transform(allRanges, cr -> cr.ranges))
@@ -520,7 +522,7 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier
         // Set up RepairJob executor for this repair command.
         ListeningExecutorService executor = createExecutor();
 
-        final ListenableFuture<List<RepairSessionResult>> allSessions = submitRepairSessions(parentSession, false, executor, commonRanges, cfnames);
+        final ListenableFuture<List<RepairSessionResult>> allSessions = submitRepairSessions(parentSession, false, executor, validationScheduler, commonRanges, cfnames);
 
         Futures.addCallback(allSessions, new FutureCallback<List<RepairSessionResult>>()
         {
@@ -635,6 +637,7 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier
     private ListenableFuture<List<RepairSessionResult>> submitRepairSessions(UUID parentSession,
                                                                              boolean isIncremental,
                                                                              ListeningExecutorService executor,
+                                                                             Scheduler validationScheduler,
                                                                              List<CommonRange> commonRanges,
                                                                              String... cfnames)
     {
@@ -652,6 +655,7 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier
                                                                                      options.getPreviewKind(),
                                                                                      options.optimiseStreams(),
                                                                                      executor,
+                                                                                     validationScheduler,
                                                                                      cfnames);
             if (session == null)
                 continue;
diff --git a/src/java/org/apache/cassandra/repair/RepairSession.java b/src/java/org/apache/cassandra/repair/RepairSession.java
index 692cdcf90c..1036f00f74 100644
--- a/src/java/org/apache/cassandra/repair/RepairSession.java
+++ b/src/java/org/apache/cassandra/repair/RepairSession.java
@@ -113,6 +113,7 @@ public class RepairSession extends AbstractFuture<RepairSessionResult> implement
     // Tasks(snapshot, validate request, differencing, ...) are run on taskExecutor
     public final ListeningExecutorService taskExecutor;
     public final boolean optimiseStreams;
+    public final Scheduler validationScheduler;
 
     private volatile boolean terminated = false;
 
@@ -128,6 +129,7 @@ public class RepairSession extends AbstractFuture<RepairSessionResult> implement
      */
     public RepairSession(UUID parentRepairSession,
                          UUID id,
+                         Scheduler validationScheduler,
                          CommonRange commonRange,
                          String keyspace,
                          RepairParallelism parallelismDegree,
@@ -139,6 +141,7 @@ public class RepairSession extends AbstractFuture<RepairSessionResult> implement
     {
         assert cfnames.length > 0 : "Repairing no column families seems pointless, doesn't it";
 
+        this.validationScheduler = validationScheduler;
         this.parentRepairSession = parentRepairSession;
         this.id = id;
         this.parallelismDegree = parallelismDegree;
diff --git a/src/java/org/apache/cassandra/repair/Scheduler.java b/src/java/org/apache/cassandra/repair/Scheduler.java
new file mode 100644
index 0000000000..c15113ffe1
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/Scheduler.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair;
+
+import java.util.LinkedList;
+import java.util.Queue;
+import java.util.concurrent.Executor;
+import java.util.function.Supplier;
+import javax.annotation.concurrent.GuardedBy;
+
+import com.google.common.util.concurrent.AbstractFuture;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+
+import org.apache.cassandra.utils.Pair;
+
+/**
+ * Task scheduler that limits the number of concurrent tasks across multiple executors.
+ */
+public interface Scheduler
+{
+    default <T> ListenableFuture<T> schedule(Supplier<ListenableFuture<T>> task, Executor executor)
+    {
+        return schedule(new Task<>(task, executor), executor);
+    }
+
+    <T> Task<T> schedule(Task<T> task, Executor executor);
+
+    static Scheduler build(int concurrentValidations)
+    {
+        return concurrentValidations <= 0
+               ? new NoopScheduler()
+               : new LimitedConcurrentScheduler(concurrentValidations);
+    }
+
+    final class NoopScheduler implements Scheduler
+    {
+        @Override
+        public <T> Task<T> schedule(Task<T> task, Executor executor)
+        {
+            executor.execute(task);
+            return task;
+        }
+    }
+
+    final class LimitedConcurrentScheduler implements Scheduler
+    {
+        private final int concurrentValidations;
+        @GuardedBy("this")
+        private int inflight = 0;
+        @GuardedBy("this")
+        private final Queue<Pair<Task<?>, Executor>> tasks = new LinkedList<>();
+
+        LimitedConcurrentScheduler(int concurrentValidations)
+        {
+            this.concurrentValidations = concurrentValidations;
+        }
+
+        @Override
+        public synchronized <T> Task<T> schedule(Task<T> task, Executor executor)
+        {
+            tasks.offer(Pair.create(task, executor));
+            maybeSchedule();
+            return task;
+        }
+
+        private synchronized void onDone()
+        {
+            inflight--;
+            maybeSchedule();
+        }
+
+        private void maybeSchedule()
+        {
+            if (inflight == concurrentValidations || tasks.isEmpty())
+                return;
+            inflight++;
+            Pair<Task<?>, Executor> pair = tasks.poll();
+            Futures.addCallback(pair.left, new FutureCallback<Object>() {
+                @Override
+                public void onSuccess(Object result)
+                {
+                    onDone();
+                }
+
+                @Override
+                public void onFailure(Throwable t)
+                {
+                    onDone();
+                }
+            }, pair.right);
+            pair.right.execute(pair.left);
+        }
+    }
+
+    class Task<T> extends AbstractFuture<T> implements Runnable
+    {
+        private final Supplier<ListenableFuture<T>> supplier;
+        private final Executor executor;
+
+        public Task(Supplier<ListenableFuture<T>> supplier, Executor executor)
+        {
+            this.supplier = supplier;
+            this.executor = executor;
+        }
+
+        @Override
+        public void run()
+        {
+            Futures.addCallback(supplier.get(), new FutureCallback<T>() {
+                @Override
+                public void onSuccess(T result)
+                {
+                    set(result);
+                }
+
+                @Override
+                public void onFailure(Throwable t)
+                {
+                    setException(t);
+                }
+            }, executor);
+        }
+    }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/service/ActiveRepairService.java b/src/java/org/apache/cassandra/service/ActiveRepairService.java
index e3ec218f6d..99c93a1d7b 100644
--- a/src/java/org/apache/cassandra/service/ActiveRepairService.java
+++ b/src/java/org/apache/cassandra/service/ActiveRepairService.java
@@ -73,6 +73,7 @@ import org.apache.cassandra.repair.CommonRange;
 import org.apache.cassandra.repair.RepairJobDesc;
 import org.apache.cassandra.repair.RepairParallelism;
 import org.apache.cassandra.repair.RepairSession;
+import org.apache.cassandra.repair.Scheduler;
 import org.apache.cassandra.repair.consistent.CoordinatorSessions;
 import org.apache.cassandra.repair.consistent.LocalSessions;
 import org.apache.cassandra.repair.consistent.admin.CleanupSummary;
@@ -338,6 +339,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
                                              PreviewKind previewKind,
                                              boolean optimiseStreams,
                                              ListeningExecutorService executor,
+                                             Scheduler validationScheduler,
                                              String... cfnames)
     {
         if (range.endpoints.isEmpty())
@@ -346,7 +348,8 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
         if (cfnames.length == 0)
             return null;
 
-        final RepairSession session = new RepairSession(parentRepairSession, UUIDGen.getTimeUUID(), range, keyspace,
+        final RepairSession session = new RepairSession(parentRepairSession, UUIDGen.getTimeUUID(),
+                                                        validationScheduler, range, keyspace,
                                                         parallelismDegree, isIncremental, pullRepair,
                                                         previewKind, optimiseStreams, cfnames);
 
diff --git a/test/distributed/org/apache/cassandra/distributed/test/OptimiseStreamsRepairTest.java b/test/distributed/org/apache/cassandra/distributed/test/OptimiseStreamsRepairTest.java
index 14d49dc0ca..e2529b4915 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/OptimiseStreamsRepairTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/OptimiseStreamsRepairTest.java
@@ -29,8 +29,6 @@ import java.util.Map;
 import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.Callable;
-import java.util.function.Function;
-import java.util.function.Predicate;
 
 import java.util.concurrent.TimeoutException;
 import org.junit.Test;
@@ -48,12 +46,11 @@ import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.repair.AsymmetricRemoteSyncTask;
 import org.apache.cassandra.repair.LocalSyncTask;
 import org.apache.cassandra.repair.RepairJob;
-import org.apache.cassandra.repair.RepairJobDesc;
 import org.apache.cassandra.repair.SyncTask;
 import org.apache.cassandra.repair.TreeResponse;
-import org.apache.cassandra.streaming.PreviewKind;
 
 import static net.bytebuddy.matcher.ElementMatchers.named;
+import static net.bytebuddy.matcher.ElementMatchers.takesArguments;
 import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
 import static org.apache.cassandra.distributed.api.Feature.NETWORK;
 import static org.junit.Assert.assertEquals;
@@ -108,22 +105,17 @@ public class OptimiseStreamsRepairTest extends TestBaseImpl
         public static void install(ClassLoader cl, int id)
         {
             new ByteBuddy().rebase(RepairJob.class)
-                           .method(named("createOptimisedSyncingSyncTasks"))
+                           .method(named("createOptimisedSyncingSyncTasks").and(takesArguments(1)))
                            .intercept(MethodDelegation.to(BBHelper.class))
                            .make()
                            .load(cl, ClassLoadingStrategy.Default.INJECTION);
         }
 
-        public static List<SyncTask> createOptimisedSyncingSyncTasks(RepairJobDesc desc,
-                                                                     List<TreeResponse> trees,
-                                                                     InetAddressAndPort local,
-                                                                     Predicate<InetAddressAndPort> isTransient,
-                                                                     Function<InetAddressAndPort, String> getDC,
-                                                                     boolean isIncremental,
-                                                                     PreviewKind previewKind,
+        @SuppressWarnings("unused")
+        public static List<SyncTask> createOptimisedSyncingSyncTasks(List<TreeResponse> trees,
                                                                      @SuperCall Callable<List<SyncTask>> zuperCall)
         {
-            List<SyncTask> tasks = null;
+            List<SyncTask> tasks;
             try
             {
                 tasks = zuperCall.call();
diff --git a/test/distributed/org/apache/cassandra/distributed/test/repair/ConcurrentValidationRequestsTest.java b/test/distributed/org/apache/cassandra/distributed/test/repair/ConcurrentValidationRequestsTest.java
new file mode 100644
index 0000000000..82a0b73371
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/repair/ConcurrentValidationRequestsTest.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.repair;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.Test;
+
+import net.bytebuddy.ByteBuddy;
+import net.bytebuddy.dynamic.loading.ClassLoadingStrategy;
+import net.bytebuddy.implementation.MethodDelegation;
+import net.bytebuddy.implementation.bind.annotation.SuperCall;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.api.NodeToolResult;
+import org.apache.cassandra.distributed.test.TestBaseImpl;
+import org.apache.cassandra.repair.ValidationTask;
+import org.apache.cassandra.utils.MerkleTrees;
+
+import static net.bytebuddy.matcher.ElementMatchers.named;
+import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
+import static org.apache.cassandra.distributed.api.Feature.NETWORK;
+
+/**
+ * Verifies that the config property {@code concurrent_merkle_tree_requests} limits the number of concurrent validation
+ * requests.
+ */
+public class ConcurrentValidationRequestsTest extends TestBaseImpl
+{
+    private static final int NODES = 4;
+    private static final int RF = 3;
+    private static final int TABLES = 5;
+    private static final int ROWS = 100;
+    private static final int MAX_REQUESTS = 1;
+
+    @Test
+    public void testConcurrentValidations() throws Throwable
+    {
+        try (Cluster cluster = init(builder().withNodes(NODES)
+                                             .withInstanceInitializer(ConcurrentValidationRequestsTest.BBHelper::install)
+                                             .withConfig(c -> c.set("concurrent_merkle_tree_requests", MAX_REQUESTS)
+                                                               .with(NETWORK, GOSSIP))
+                                             .start(), RF))
+        {
+            for (int t = 1; t <= TABLES; t++)
+                cluster.schemaChange(withKeyspace("CREATE TABLE %s.t" + t + " (k int PRIMARY KEY, v int)"));
+
+            int v = 0;
+            for (int k = 0; k < ROWS; k++)
+            {
+                v++;
+                for (int t = 1; t <= TABLES; t++)
+                {
+                    String insert = withKeyspace("INSERT INTO %s.t" + t + " (k, v) VALUES (?, ?)");
+                    cluster.coordinator(1).execute(insert, ConsistencyLevel.ALL, k, v);
+                }
+            }
+            cluster.forEach(x -> x.flush(KEYSPACE));
+
+            NodeToolResult res = cluster.get(1).nodetoolResult("repair", "-j=4", KEYSPACE);
+            res.asserts().success();
+        }
+    }
+
+    public static class BBHelper
+    {
+        /**
+         * Keeps track of the number of concurrent validation requests, which should never be greater than RF times
+         * the value of the {@code concurrent_merkle_tree_requests} config property.
+         */
+        public static volatile AtomicInteger requests = new AtomicInteger(0);
+
+        public static void install(ClassLoader cl, int node)
+        {
+            if (node != 1)
+                return;
+
+            new ByteBuddy().rebase(ValidationTask.class)
+                           .method(named("run"))
+                           .intercept(MethodDelegation.to(ConcurrentValidationRequestsTest.BBHelper.class))
+                           .method(named("treesReceived"))
+                           .intercept(MethodDelegation.to(ConcurrentValidationRequestsTest.BBHelper.class))
+                           .make()
+                           .load(cl, ClassLoadingStrategy.Default.INJECTION);
+        }
+
+        @SuppressWarnings("unused")
+        public static void run(@SuperCall Callable<Void> zuper)
+        {
+            try
+            {
+                zuper.call();
+            }
+            catch (Exception e)
+            {
+                throw new RuntimeException(e);
+            }
+
+            int requests = BBHelper.requests.incrementAndGet();
+            if (requests > MAX_REQUESTS * RF)
+                throw new AssertionError("Too many concurrent validation requests: " + requests);
+        }
+
+        @SuppressWarnings("unused")
+        public static void treesReceived(MerkleTrees trees, @SuperCall Callable<Void> zuperCall)
+        {
+            requests.decrementAndGet();
+
+            try
+            {
+                zuperCall.call();
+            }
+            catch (Exception e)
+            {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/repair/RepairJobTest.java b/test/unit/org/apache/cassandra/repair/RepairJobTest.java
index 87863b5b67..8d3b6a7099 100644
--- a/test/unit/org/apache/cassandra/repair/RepairJobTest.java
+++ b/test/unit/org/apache/cassandra/repair/RepairJobTest.java
@@ -110,7 +110,8 @@ public class RepairJobTest
                                         RepairParallelism parallelismDegree, boolean isIncremental, boolean pullRepair,
                                         PreviewKind previewKind, boolean optimiseStreams, String... cfnames)
         {
-            super(parentRepairSession, id, commonRange, keyspace, parallelismDegree, isIncremental, pullRepair, previewKind, optimiseStreams, cfnames);
+            super(parentRepairSession, id, Scheduler.build(0), commonRange, keyspace, parallelismDegree, isIncremental,
+                  pullRepair, previewKind, optimiseStreams, cfnames);
         }
 
         protected DebuggableThreadPoolExecutor createExecutor()
diff --git a/test/unit/org/apache/cassandra/repair/RepairSessionTest.java b/test/unit/org/apache/cassandra/repair/RepairSessionTest.java
index 2ad5831fc9..77b7102117 100644
--- a/test/unit/org/apache/cassandra/repair/RepairSessionTest.java
+++ b/test/unit/org/apache/cassandra/repair/RepairSessionTest.java
@@ -63,7 +63,7 @@ public class RepairSessionTest
         IPartitioner p = Murmur3Partitioner.instance;
         Range<Token> repairRange = new Range<>(p.getToken(ByteBufferUtil.bytes(0)), p.getToken(ByteBufferUtil.bytes(100)));
         Set<InetAddressAndPort> endpoints = Sets.newHashSet(remote);
-        RepairSession session = new RepairSession(parentSessionId, sessionId,
+        RepairSession session = new RepairSession(parentSessionId, sessionId, Scheduler.build(0),
                                                   new CommonRange(endpoints, Collections.emptySet(), Arrays.asList(repairRange)),
                                                   "Keyspace1", RepairParallelism.SEQUENTIAL,
                                                   false, false,


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org