You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by dc...@apache.org on 2023/09/27 23:23:18 UTC
[cassandra] branch cassandra-5.0 updated: Add support for repair coordinator to retry messages that timeout
This is an automated email from the ASF dual-hosted git repository.
dcapwell pushed a commit to branch cassandra-5.0
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/cassandra-5.0 by this push:
new 164fa7949d Add support for repair coordinator to retry messages that timeout
164fa7949d is described below
commit 164fa7949d1437528363465d14ed20898f1daa9e
Author: David Capwell <dc...@apache.org>
AuthorDate: Wed Sep 27 08:46:40 2023 -0700
Add support for repair coordinator to retry messages that timeout
patch by David Capwell; reviewed by Caleb Rackliffe, Marcus Eriksson for CASSANDRA-18816
---
CHANGES.txt | 1 +
conf/cassandra.yaml | 11 +
.../org/apache/cassandra/concurrent/Stage.java | 6 +
src/java/org/apache/cassandra/config/Config.java | 2 +
.../cassandra/config/DatabaseDescriptor.java | 11 +
.../org/apache/cassandra/config/RepairConfig.java | 24 +
.../apache/cassandra/config/RepairRetrySpec.java | 43 +
.../org/apache/cassandra/config/RetrySpec.java | 165 +++
.../org/apache/cassandra/cql3/QueryProcessor.java | 2 +-
.../org/apache/cassandra/db/ColumnFamilyStore.java | 2 +-
src/java/org/apache/cassandra/db/Keyspace.java | 2 +-
src/java/org/apache/cassandra/db/ReadCommand.java | 4 +-
.../cassandra/db/compaction/CompactionManager.java | 5 +-
.../db/compaction/ICompactionManager.java | 24 +
.../db/compaction/PendingRepairManager.java | 4 +-
.../db/repair/CassandraTableRepairManager.java | 12 +-
.../db/repair/CassandraValidationIterator.java | 13 +-
.../cassandra/db/repair/PendingAntiCompaction.java | 2 +-
.../cassandra/db/virtual/LocalRepairTables.java | 19 +-
.../cassandra/exceptions/RepairException.java | 10 +-
src/java/org/apache/cassandra/gms/Gossiper.java | 12 +-
src/java/org/apache/cassandra/gms/IGossiper.java | 39 +
src/java/org/apache/cassandra/net/Message.java | 12 +-
.../org/apache/cassandra/net/MessageDelivery.java | 31 +
.../org/apache/cassandra/net/MessagingService.java | 72 +-
src/java/org/apache/cassandra/net/Verb.java | 43 +-
.../cassandra/repair/AbstractRepairTask.java | 44 +-
.../cassandra/repair/AsymmetricRemoteSyncTask.java | 7 +-
.../cassandra/repair/IValidationManager.java | 28 +
.../cassandra/repair/IncrementalRepairTask.java | 17 +-
.../org/apache/cassandra/repair/LocalSyncTask.java | 12 +-
.../apache/cassandra/repair/NormalRepairTask.java | 7 +-
.../apache/cassandra/repair/PreviewRepairTask.java | 7 +-
...{RepairRunnable.java => RepairCoordinator.java} | 162 +--
.../org/apache/cassandra/repair/RepairJob.java | 76 +-
.../org/apache/cassandra/repair/RepairJobDesc.java | 11 +-
.../cassandra/repair/RepairMessageVerbHandler.java | 250 +++-
.../org/apache/cassandra/repair/RepairSession.java | 151 ++-
.../org/apache/cassandra/repair/SharedContext.java | 165 +++
.../org/apache/cassandra/repair/SnapshotTask.java | 11 +-
.../apache/cassandra/repair/StreamExecutor.java | 27 +
.../cassandra/repair/StreamingRepairTask.java | 24 +-
.../cassandra/repair/SymmetricRemoteSyncTask.java | 7 +-
src/java/org/apache/cassandra/repair/SyncTask.java | 18 +-
.../apache/cassandra/repair/ValidationManager.java | 16 +-
.../apache/cassandra/repair/ValidationTask.java | 61 +-
.../org/apache/cassandra/repair/Validator.java | 25 +-
.../repair/consistent/ConsistentSession.java | 9 +
.../repair/consistent/CoordinatorSession.java | 38 +-
.../repair/consistent/CoordinatorSessions.java | 13 +-
.../cassandra/repair/consistent/LocalSession.java | 13 +-
.../cassandra/repair/consistent/LocalSessions.java | 36 +-
.../cassandra/repair/messages/CleanupMessage.java | 6 +
.../cassandra/repair/messages/PrepareMessage.java | 6 +
.../cassandra/repair/messages/RepairMessage.java | 179 ++-
.../repair/state/AbstractCompletable.java | 28 +-
.../cassandra/repair/state/AbstractState.java | 69 +-
.../apache/cassandra/repair/state/Completable.java | 33 +-
.../cassandra/repair/state/CoordinatorState.java | 41 +-
.../apache/cassandra/repair/state/JobState.java | 5 +-
.../cassandra/repair/state/ParticipateState.java | 160 ++-
.../cassandra/repair/state/SessionState.java | 21 +-
.../apache/cassandra/repair/state/SyncState.java | 85 ++
.../cassandra/repair/state/ValidationState.java | 13 +-
src/java/org/apache/cassandra/schema/Tables.java | 2 +-
.../cassandra/service/ActiveRepairService.java | 270 ++--
.../apache/cassandra/service/CassandraDaemon.java | 2 +-
.../apache/cassandra/service/StorageService.java | 24 +-
.../cassandra/service/StorageServiceMBean.java | 2 +-
.../apache/cassandra/streaming/PreviewKind.java | 2 +-
.../org/apache/cassandra/streaming/StreamPlan.java | 16 +
.../apache/cassandra/streaming/StreamSession.java | 6 +-
.../apache/cassandra/tools/nodetool/Repair.java | 85 +-
src/java/org/apache/cassandra/utils/Backoff.java | 96 ++
src/java/org/apache/cassandra/utils/Clock.java | 10 +
.../apache/cassandra/utils/FailingBiConsumer.java | 44 +
.../org/apache/cassandra/utils/MerkleTree.java | 2 +-
.../cassandra/distributed/impl/Instance.java | 4 +-
.../mock/nodetool/InternalNodeProbe.java | 2 +-
.../distributed/test/ClearSnapshotTest.java | 2 +-
.../distributed/test/IncRepairAdminTest.java | 16 +-
.../test/IncRepairCoordinatorErrorTest.java | 2 +-
.../test/OptimiseStreamsRepairTest.java | 4 +-
.../distributed/test/PaxosRepair2Test.java | 2 +-
.../distributed/test/PaxosRepairTest.java | 2 +-
.../distributed/test/RepairErrorsTest.java | 2 +-
test/unit/accord/utils/DefaultRandom.java | 94 ++
test/unit/accord/utils/Gen.java | 179 +++
test/unit/accord/utils/Gens.java | 576 +++++++++
test/unit/accord/utils/Invariants.java | 327 +++++
test/unit/accord/utils/Property.java | 382 ++++++
test/unit/accord/utils/RandomSource.java | 361 ++++++
test/unit/accord/utils/WrappedRandomSource.java | 97 ++
.../concurrent/ForwardingExecutorPlus.java | 219 ++++
.../ForwardingLocalAwareExecutorPlus.java | 167 +++
.../ForwardingScheduledExecutorPlus.java | 92 ++
.../concurrent/SimulatedExecutorFactory.java | 506 ++++++++
.../config/DatabaseDescriptorRefTest.java | 5 +
.../cassandra/config/UnitConfigOverride.java | 83 ++
.../config/YamlConfigurationLoaderTest.java | 45 +
.../org/apache/cassandra/db/ReadCommandTest.java | 16 +-
.../db/compaction/AbstractPendingRepairTest.java | 2 +-
.../db/compaction/AntiCompactionTest.java | 10 +-
.../db/compaction/CancelCompactionsTest.java | 2 +-
...CompactionStrategyManagerPendingRepairTest.java | 2 +-
.../compaction/LeveledCompactionStrategyTest.java | 19 +-
.../repair/AbstractPendingAntiCompactionTest.java | 2 +-
...pactionManagerGetSSTablesForValidationTest.java | 30 +-
.../db/repair/PendingAntiCompactionTest.java | 30 +-
.../db/streaming/CassandraStreamManagerTest.java | 6 +-
.../db/virtual/LocalRepairTablesTest.java | 25 +-
.../index/sai/functional/CompactionTest.java | 16 +-
.../cassandra/repair/AbstractRepairTest.java | 16 +-
.../repair/ConcurrentIrWithPreviewFuzzTest.java | 92 ++
.../org/apache/cassandra/repair/FailedAckTest.java | 136 ++
.../cassandra/repair/FailingRepairFuzzTest.java | 162 +++
.../org/apache/cassandra/repair/FuzzTestBase.java | 1296 ++++++++++++++++++++
.../apache/cassandra/repair/HappyPathFuzzTest.java | 62 +
.../apache/cassandra/repair/LocalSyncTaskTest.java | 28 +-
.../cassandra/repair/NeighborsAndRangesTest.java | 2 +-
.../org/apache/cassandra/repair/RepairJobTest.java | 59 +-
.../apache/cassandra/repair/RepairSessionTest.java | 2 +-
.../cassandra/repair/SlowMessageFuzzTest.java | 64 +
.../cassandra/repair/StreamingRepairTaskTest.java | 10 +-
.../repair/SymmetricRemoteSyncTaskTest.java | 2 +-
.../cassandra/repair/ValidationTaskTest.java | 11 +-
.../org/apache/cassandra/repair/ValidatorTest.java | 29 +-
.../consistent/AbstractConsistentSessionTest.java | 16 +-
.../consistent/CoordinatorMessagingTest.java | 12 +-
.../repair/consistent/CoordinatorSessionTest.java | 3 +-
.../repair/consistent/CoordinatorSessionsTest.java | 10 +-
.../repair/consistent/LocalSessionAccessor.java | 2 +-
.../repair/consistent/LocalSessionTest.java | 18 +-
.../repair/consistent/PendingRepairStatTest.java | 3 +-
.../cassandra/service/ActiveRepairServiceTest.java | 47 +-
.../cassandra/service/SerializationsTest.java | 5 +-
.../cassandra/tools/JMXCompatabilityTest.java | 12 +-
.../org/apache/cassandra/tools/ToolRunner.java | 18 +
.../cassandra/utils/CassandraGenerators.java | 68 +-
test/unit/org/quicktheories/impl/JavaRandom.java | 6 +
140 files changed, 7609 insertions(+), 924 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 863b99740c..12ffc27c9b 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
5.0-alpha2
+ * Add support for repair coordinator to retry messages that timeout (CASSANDRA-18816)
* Upgrade slf4j-api to 1.7.36 (CASSANDRA-18882)
* Make the output of ON/OFF commands in cqlsh consistent (CASSANDRA-18547)
* Do not create sstable files before registering in txn (CASSANDRA-18737)
diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml
index c41c3cb973..fb6f454bb9 100644
--- a/conf/cassandra.yaml
+++ b/conf/cassandra.yaml
@@ -723,6 +723,17 @@ memtable_allocation_type: heap_buffers
# Min unit: MiB
# repair_session_space:
+# repair:
+# # Configure the retries for each of the repair messages that support it. As of this moment retries use an exponential algorithm where each attempt sleeps longer based off the base_sleep_time and attempt.
+# retries:
+# max_attempts: 10
+# base_sleep_time: 200ms
+# max_sleep_time: 1s
+# # Increase the timeout of validation responses due to them containing the merkle tree
+# merkle_tree_response:
+# base_sleep_time: 30s
+# max_sleep_time: 1m
+
# Total space to use for commit logs on disk.
#
# If space gets above this value, Cassandra will flush every dirty CF
diff --git a/src/java/org/apache/cassandra/concurrent/Stage.java b/src/java/org/apache/cassandra/concurrent/Stage.java
index ac609aa774..05b6c612fa 100644
--- a/src/java/org/apache/cassandra/concurrent/Stage.java
+++ b/src/java/org/apache/cassandra/concurrent/Stage.java
@@ -144,6 +144,12 @@ public enum Stage
return executor;
}
+ @VisibleForTesting
+ public void unsafeSetExecutor(ExecutorPlus executor)
+ {
+ this.executor = executor;
+ }
+
private static List<ExecutorPlus> executors()
{
return Stream.of(Stage.values())
diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java
index 2157b225eb..ebe7039969 100644
--- a/src/java/org/apache/cassandra/config/Config.java
+++ b/src/java/org/apache/cassandra/config/Config.java
@@ -1105,6 +1105,8 @@ public class Config
public volatile long min_tracked_partition_tombstone_count = 5000;
public volatile boolean top_partitions_enabled = true;
+ public final RepairConfig repair = new RepairConfig();
+
/**
* Default compaction configuration, used if a table does not specify any.
*/
diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index 298ed3f8f9..e6d62a9ff9 100644
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@ -3695,6 +3695,12 @@ public class DatabaseDescriptor
return localDC;
}
+ @VisibleForTesting
+ public static void setLocalDataCenter(String value)
+ {
+ localDC = value;
+ }
+
public static Comparator<Replica> getLocalComparator()
{
return localComparator;
@@ -4906,4 +4912,9 @@ public class DatabaseDescriptor
{
return conf.sai_options.segment_write_buffer_size;
}
+
+ public static RepairRetrySpec getRepairRetrySpec()
+ {
+ return conf == null ? new RepairRetrySpec() : conf.repair.retries;
+ }
}
diff --git a/src/java/org/apache/cassandra/config/RepairConfig.java b/src/java/org/apache/cassandra/config/RepairConfig.java
new file mode 100644
index 0000000000..4912f51dd3
--- /dev/null
+++ b/src/java/org/apache/cassandra/config/RepairConfig.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.config;
+
+public class RepairConfig
+{
+ public final RepairRetrySpec retries = new RepairRetrySpec();
+}
diff --git a/src/java/org/apache/cassandra/config/RepairRetrySpec.java b/src/java/org/apache/cassandra/config/RepairRetrySpec.java
new file mode 100644
index 0000000000..42900dee01
--- /dev/null
+++ b/src/java/org/apache/cassandra/config/RepairRetrySpec.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.config;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+
+public class RepairRetrySpec extends RetrySpec
+{
+ public RetrySpec.Partial merkle_tree_response = null;
+
+ public boolean isMerkleTreeRetriesEnabled()
+ {
+ RetrySpec.Partial partial = merkle_tree_response;
+ if (partial == null || partial.maxAttempts == null)
+ return isEnabled();
+ return partial.isEnabled();
+ }
+
+ @JsonIgnore
+ public RetrySpec getMerkleTreeResponseSpec()
+ {
+ RetrySpec.Partial partial = merkle_tree_response;
+ if (partial == null)
+ return this;
+ return partial.withDefaults(this);
+ }
+}
diff --git a/src/java/org/apache/cassandra/config/RetrySpec.java b/src/java/org/apache/cassandra/config/RetrySpec.java
new file mode 100644
index 0000000000..4f113af962
--- /dev/null
+++ b/src/java/org/apache/cassandra/config/RetrySpec.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.config;
+
+import java.util.Objects;
+
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.config.DurationSpec.LongMillisecondsBound;
+
+public class RetrySpec
+{
+ public static class MaxAttempt
+ {
+ public static final MaxAttempt DISABLED = new MaxAttempt();
+
+ public final int value;
+
+ public MaxAttempt(int value)
+ {
+ if (value < 1)
+ throw new IllegalArgumentException("max attempt must be positive; but given " + value);
+ this.value = value;
+ }
+
+ private MaxAttempt()
+ {
+ value = 0;
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null) return false;
+ if (o instanceof Integer) return this.value == ((Integer) o).intValue();
+ if (getClass() != o.getClass()) return false;
+ MaxAttempt that = (MaxAttempt) o;
+ return value == that.value;
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(value);
+ }
+
+ @Override
+ public String toString()
+ {
+ return Integer.toString(value);
+ }
+ }
+
+ public static class Partial extends RetrySpec
+ {
+ public Partial()
+ {
+ this.maxAttempts = null;
+ this.baseSleepTime = null;
+ this.maxSleepTime = null;
+ }
+
+ public RetrySpec withDefaults(RetrySpec defaultValues)
+ {
+ MaxAttempt maxAttempts = nonNull(this.maxAttempts, defaultValues.getMaxAttempts(), DEFAULT_MAX_ATTEMPTS);
+ LongMillisecondsBound baseSleepTime = nonNull(this.baseSleepTime, defaultValues.getBaseSleepTime(), DEFAULT_BASE_SLEEP);
+ LongMillisecondsBound maxSleepTime = nonNull(this.maxSleepTime, defaultValues.getMaxSleepTime(), DEFAULT_MAX_SLEEP);
+ return new RetrySpec(maxAttempts, baseSleepTime, maxSleepTime);
+ }
+
+ private static <T> T nonNull(@Nullable T left, @Nullable T right, T defaultValue)
+ {
+ if (left != null)
+ return left;
+ if (right != null)
+ return right;
+ return defaultValue;
+ }
+ }
+
+ public static final MaxAttempt DEFAULT_MAX_ATTEMPTS = MaxAttempt.DISABLED;
+ public static final LongMillisecondsBound DEFAULT_BASE_SLEEP = new LongMillisecondsBound("200ms");
+ public static final LongMillisecondsBound DEFAULT_MAX_SLEEP = new LongMillisecondsBound("1s");
+
+ /**
+ * Represents how many retry attempts are allowed. If the value is 2, this will cause 2 retries + 1 original request, for a total of 3 requests!
+ * <p/>
+ * To disable, set to 0.
+ */
+ public MaxAttempt maxAttempts = DEFAULT_MAX_ATTEMPTS; // 2 retries, 1 original request; so 3 total
+ public LongMillisecondsBound baseSleepTime = DEFAULT_BASE_SLEEP;
+ public LongMillisecondsBound maxSleepTime = DEFAULT_MAX_SLEEP;
+
+ public RetrySpec()
+ {
+ }
+
+ public RetrySpec(MaxAttempt maxAttempts, LongMillisecondsBound baseSleepTime, LongMillisecondsBound maxSleepTime)
+ {
+ this.maxAttempts = maxAttempts;
+ this.baseSleepTime = baseSleepTime;
+ this.maxSleepTime = maxSleepTime;
+ }
+
+ public boolean isEnabled()
+ {
+ return maxAttempts != MaxAttempt.DISABLED;
+ }
+
+ public void setEnabled(boolean enabled)
+ {
+ if (!enabled)
+ {
+ maxAttempts = MaxAttempt.DISABLED;
+ }
+ else if (maxAttempts == MaxAttempt.DISABLED)
+ {
+ maxAttempts = new MaxAttempt(2);
+ }
+ }
+
+ @Nullable
+ public MaxAttempt getMaxAttempts()
+ {
+ return !isEnabled() ? null : maxAttempts;
+ }
+
+ @Nullable
+ public LongMillisecondsBound getBaseSleepTime()
+ {
+ return !isEnabled() ? null : baseSleepTime;
+ }
+
+ public LongMillisecondsBound getMaxSleepTime()
+ {
+ return !isEnabled() ? null : maxSleepTime;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "RetrySpec{" +
+ "maxAttempts=" + maxAttempts +
+ ", baseSleepTime=" + baseSleepTime +
+ ", maxSleepTime=" + maxSleepTime +
+ '}';
+ }
+}
diff --git a/src/java/org/apache/cassandra/cql3/QueryProcessor.java b/src/java/org/apache/cassandra/cql3/QueryProcessor.java
index 02369f373a..fba424d7d9 100644
--- a/src/java/org/apache/cassandra/cql3/QueryProcessor.java
+++ b/src/java/org/apache/cassandra/cql3/QueryProcessor.java
@@ -478,7 +478,7 @@ public class QueryProcessor implements QueryHandler
}
Future<List<Message<ReadResponse>>> future = FutureCombiner.allOf(commands.stream()
.map(rc -> Message.out(rc.verb(), rc))
- .map(m -> MessagingService.instance().<ReadResponse>sendWithResult(m, address))
+ .map(m -> MessagingService.instance().<ReadCommand, ReadResponse>sendWithResult(m, address))
.collect(Collectors.toList()));
ResultSetBuilder result = new ResultSetBuilder(select.getResultMetadata(), select.getSelection().newSelectors(options), false);
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 97a4a27bdf..856542c946 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -2726,7 +2726,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
// since truncation can happen at different times on different nodes, we need to make sure
// that any repairs are aborted, otherwise we might clear the data on one node and then
// stream in data that is actually supposed to have been deleted
- ActiveRepairService.instance.abort((prs) -> prs.getTableIds().contains(metadata.id),
+ ActiveRepairService.instance().abort((prs) -> prs.getTableIds().contains(metadata.id),
"Stopping parent sessions {} due to truncation of tableId="+metadata.id);
data.notifyTruncated(truncatedAt);
diff --git a/src/java/org/apache/cassandra/db/Keyspace.java b/src/java/org/apache/cassandra/db/Keyspace.java
index a3f05e6048..b49eb3162a 100644
--- a/src/java/org/apache/cassandra/db/Keyspace.java
+++ b/src/java/org/apache/cassandra/db/Keyspace.java
@@ -674,7 +674,7 @@ public class Keyspace
public Iterable<ColumnFamilyStore> getValidColumnFamilies(boolean allowIndexes,
boolean autoAddIndexes,
- String... cfNames) throws IOException
+ String... cfNames)
{
Set<ColumnFamilyStore> valid = new HashSet<>();
diff --git a/src/java/org/apache/cassandra/db/ReadCommand.java b/src/java/org/apache/cassandra/db/ReadCommand.java
index 2ca47ea06f..671f400081 100644
--- a/src/java/org/apache/cassandra/db/ReadCommand.java
+++ b/src/java/org/apache/cassandra/db/ReadCommand.java
@@ -981,13 +981,13 @@ public abstract class ReadCommand extends AbstractReadQuery
TimeUUID pendingRepair = sstable.getPendingRepair();
if (pendingRepair != ActiveRepairService.NO_PENDING_REPAIR)
{
- if (ActiveRepairService.instance.consistent.local.isSessionFinalized(pendingRepair))
+ if (ActiveRepairService.instance().consistent.local.isSessionFinalized(pendingRepair))
return true;
// In the edge case where compaction is backed up long enough for the session to
// timeout and be purged by LocalSessions::cleanup, consider the sstable unrepaired
// as it will be marked unrepaired when compaction catches up
- if (!ActiveRepairService.instance.consistent.local.sessionExists(pendingRepair))
+ if (!ActiveRepairService.instance().consistent.local.sessionExists(pendingRepair))
return false;
repairedDataInfo.markInconclusive();
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index 44d6f4b833..ad68a09808 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@ -135,7 +135,7 @@ import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
* a set via Tracker. New scheduling attempts will ignore currently compacting
* sstables.
*/
-public class CompactionManager implements CompactionManagerMBean
+public class CompactionManager implements CompactionManagerMBean, ICompactionManager
{
public static final String MBEAN_OBJECT_NAME = "org.apache.cassandra.db:type=CompactionManager";
private static final Logger logger = LoggerFactory.getLogger(CompactionManager.class);
@@ -893,7 +893,7 @@ public class CompactionManager implements CompactionManagerMBean
ActiveRepairService.ParentRepairSession prs;
try
{
- prs = ActiveRepairService.instance.getParentRepairSession(sessionID);
+ prs = ActiveRepairService.instance().getParentRepairSession(sessionID);
}
catch (NoSuchRepairSessionException e)
{
@@ -2193,6 +2193,7 @@ public class CompactionManager implements CompactionManagerMBean
return metrics.totalCompactionsCompleted.getCount();
}
+ @Override
public int getPendingTasks()
{
return metrics.pendingTasks.getValue();
diff --git a/src/java/org/apache/cassandra/db/compaction/ICompactionManager.java b/src/java/org/apache/cassandra/db/compaction/ICompactionManager.java
new file mode 100644
index 0000000000..e8e7c6830e
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/compaction/ICompactionManager.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+public interface ICompactionManager
+{
+ int getPendingTasks();
+}
diff --git a/src/java/org/apache/cassandra/db/compaction/PendingRepairManager.java b/src/java/org/apache/cassandra/db/compaction/PendingRepairManager.java
index 87915c964a..2ddfab7060 100644
--- a/src/java/org/apache/cassandra/db/compaction/PendingRepairManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/PendingRepairManager.java
@@ -272,7 +272,7 @@ class PendingRepairManager
if (compactionStrategy == null)
return null;
Set<SSTableReader> sstables = compactionStrategy.getSSTables();
- long repairedAt = ActiveRepairService.instance.consistent.local.getFinalSessionRepairedAt(sessionID);
+ long repairedAt = ActiveRepairService.instance().consistent.local.getFinalSessionRepairedAt(sessionID);
LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.COMPACTION);
return txn == null ? null : new RepairFinishedCompactionTask(cfs, txn, sessionID, repairedAt);
}
@@ -426,7 +426,7 @@ class PendingRepairManager
boolean canCleanup(TimeUUID sessionID)
{
- return !ActiveRepairService.instance.consistent.local.isSessionInProgress(sessionID);
+ return !ActiveRepairService.instance().consistent.local.isSessionInProgress(sessionID);
}
@SuppressWarnings("resource")
diff --git a/src/java/org/apache/cassandra/db/repair/CassandraTableRepairManager.java b/src/java/org/apache/cassandra/db/repair/CassandraTableRepairManager.java
index 053342e12b..4e54d6ee77 100644
--- a/src/java/org/apache/cassandra/db/repair/CassandraTableRepairManager.java
+++ b/src/java/org/apache/cassandra/db/repair/CassandraTableRepairManager.java
@@ -32,6 +32,7 @@ import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.metrics.TopPartitionTracker;
+import org.apache.cassandra.repair.SharedContext;
import org.apache.cassandra.repair.TableRepairManager;
import org.apache.cassandra.repair.ValidationPartitionIterator;
import org.apache.cassandra.repair.NoSuchRepairSessionException;
@@ -41,16 +42,23 @@ import org.apache.cassandra.service.ActiveRepairService;
public class CassandraTableRepairManager implements TableRepairManager
{
private final ColumnFamilyStore cfs;
+ private final SharedContext ctx;
public CassandraTableRepairManager(ColumnFamilyStore cfs)
+ {
+ this(cfs, SharedContext.Global.instance);
+ }
+
+ public CassandraTableRepairManager(ColumnFamilyStore cfs, SharedContext ctx)
{
this.cfs = cfs;
+ this.ctx = ctx;
}
@Override
public ValidationPartitionIterator getValidationIterator(Collection<Range<Token>> ranges, TimeUUID parentId, TimeUUID sessionID, boolean isIncremental, long nowInSec, TopPartitionTracker.Collector topPartitionCollector) throws IOException, NoSuchRepairSessionException
{
- return new CassandraValidationIterator(cfs, ranges, parentId, sessionID, isIncremental, nowInSec, topPartitionCollector);
+ return new CassandraValidationIterator(cfs, ctx, ranges, parentId, sessionID, isIncremental, nowInSec, topPartitionCollector);
}
@Override
@@ -70,7 +78,7 @@ public class CassandraTableRepairManager implements TableRepairManager
{
try
{
- ActiveRepairService.instance.snapshotExecutor.submit(() -> {
+ ActiveRepairService.instance().snapshotExecutor.submit(() -> {
if (force || !cfs.snapshotExists(name))
{
cfs.snapshot(name, new Predicate<SSTableReader>()
diff --git a/src/java/org/apache/cassandra/db/repair/CassandraValidationIterator.java b/src/java/org/apache/cassandra/db/repair/CassandraValidationIterator.java
index 4e4f5600b2..a31a7038b0 100644
--- a/src/java/org/apache/cassandra/db/repair/CassandraValidationIterator.java
+++ b/src/java/org/apache/cassandra/db/repair/CassandraValidationIterator.java
@@ -52,6 +52,7 @@ import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.sstable.ISSTableScanner;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.metrics.TopPartitionTracker;
+import org.apache.cassandra.repair.SharedContext;
import org.apache.cassandra.repair.ValidationPartitionIterator;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.ActiveRepairService;
@@ -112,11 +113,11 @@ public class CassandraValidationIterator extends ValidationPartitionIterator
}
@VisibleForTesting
- public static synchronized Refs<SSTableReader> getSSTablesToValidate(ColumnFamilyStore cfs, Collection<Range<Token>> ranges, TimeUUID parentId, boolean isIncremental) throws NoSuchRepairSessionException
+ public static synchronized Refs<SSTableReader> getSSTablesToValidate(ColumnFamilyStore cfs, SharedContext ctx, Collection<Range<Token>> ranges, TimeUUID parentId, boolean isIncremental) throws NoSuchRepairSessionException
{
Refs<SSTableReader> sstables;
- ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(parentId);
+ ActiveRepairService.ParentRepairSession prs = ctx.repair().getParentRepairSession(parentId);
Set<SSTableReader> sstablesToValidate = new HashSet<>();
@@ -158,6 +159,7 @@ public class CassandraValidationIterator extends ValidationPartitionIterator
}
private final ColumnFamilyStore cfs;
+ private final SharedContext ctx;
private final Refs<SSTableReader> sstables;
private final String snapshotName;
private final boolean isGlobalSnapshotValidation;
@@ -172,9 +174,10 @@ public class CassandraValidationIterator extends ValidationPartitionIterator
private final long estimatedPartitions;
private final Map<Range<Token>, Long> rangePartitionCounts;
- public CassandraValidationIterator(ColumnFamilyStore cfs, Collection<Range<Token>> ranges, TimeUUID parentId, TimeUUID sessionID, boolean isIncremental, long nowInSec, TopPartitionTracker.Collector topPartitionCollector) throws IOException, NoSuchRepairSessionException
+ public CassandraValidationIterator(ColumnFamilyStore cfs, SharedContext ctx, Collection<Range<Token>> ranges, TimeUUID parentId, TimeUUID sessionID, boolean isIncremental, long nowInSec, TopPartitionTracker.Collector topPartitionCollector) throws IOException, NoSuchRepairSessionException
{
this.cfs = cfs;
+ this.ctx = ctx;
isGlobalSnapshotValidation = cfs.snapshotExists(parentId.toString());
if (isGlobalSnapshotValidation)
@@ -198,7 +201,7 @@ public class CassandraValidationIterator extends ValidationPartitionIterator
cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.VALIDATION);
// Note: we also flush for incremental repair during the anti-compaction process.
}
- sstables = getSSTablesToValidate(cfs, ranges, parentId, isIncremental);
+ sstables = getSSTablesToValidate(cfs, ctx, ranges, parentId, isIncremental);
}
// Persistent memtables will not flush or snapshot to sstables, make an sstable with their data.
@@ -208,7 +211,7 @@ public class CassandraValidationIterator extends ValidationPartitionIterator
Preconditions.checkArgument(sstables != null);
- ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(parentId);
+ ActiveRepairService.ParentRepairSession prs = ctx.repair().getParentRepairSession(parentId);
logger.info("{}, parentSessionId={}: Performing validation compaction on {} sstables in {}.{}",
prs.previewKind.logPrefix(sessionID),
parentId,
diff --git a/src/java/org/apache/cassandra/db/repair/PendingAntiCompaction.java b/src/java/org/apache/cassandra/db/repair/PendingAntiCompaction.java
index 55138e8277..2711ace7da 100644
--- a/src/java/org/apache/cassandra/db/repair/PendingAntiCompaction.java
+++ b/src/java/org/apache/cassandra/db/repair/PendingAntiCompaction.java
@@ -137,7 +137,7 @@ public class PendingAntiCompaction
// non-finalized sessions for a later error message
if (metadata.pendingRepair != NO_PENDING_REPAIR)
{
- if (!ActiveRepairService.instance.consistent.local.isSessionFinalized(metadata.pendingRepair))
+ if (!ActiveRepairService.instance().consistent.local.isSessionFinalized(metadata.pendingRepair))
{
String message = String.format("Prepare phase for incremental repair session %s has failed because it encountered " +
"intersecting sstables belonging to another incremental repair session (%s). This is " +
diff --git a/src/java/org/apache/cassandra/db/virtual/LocalRepairTables.java b/src/java/org/apache/cassandra/db/virtual/LocalRepairTables.java
index 1f440abc72..b88502988e 100644
--- a/src/java/org/apache/cassandra/db/virtual/LocalRepairTables.java
+++ b/src/java/org/apache/cassandra/db/virtual/LocalRepairTables.java
@@ -27,6 +27,7 @@ import java.util.UUID;
import java.util.stream.Collectors;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableSet;
import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
import org.apache.cassandra.db.DecoratedKey;
@@ -115,7 +116,7 @@ public class LocalRepairTables
public DataSet data()
{
SimpleDataSet result = new SimpleDataSet(metadata());
- ActiveRepairService.instance.coordinators().forEach(s -> updateDataset(result, s));
+ ActiveRepairService.instance().coordinators().forEach(s -> updateDataset(result, s));
return result;
}
@@ -123,7 +124,7 @@ public class LocalRepairTables
{
TimeUUID id = TimeUUIDType.instance.compose(partitionKey.getKey());
SimpleDataSet result = new SimpleDataSet(metadata());
- CoordinatorState state = ActiveRepairService.instance.coordinator(id);
+ CoordinatorState state = ActiveRepairService.instance().coordinator(id);
if (state != null)
updateDataset(result, state);
return result;
@@ -212,7 +213,7 @@ public class LocalRepairTables
public DataSet data()
{
SimpleDataSet result = new SimpleDataSet(metadata());
- ActiveRepairService.instance.coordinators().stream()
+ ActiveRepairService.instance().coordinators().stream()
.flatMap(s -> s.getSessions().stream())
.forEach(s -> updateDataset(result, s));
return result;
@@ -251,7 +252,7 @@ public class LocalRepairTables
public DataSet data()
{
SimpleDataSet result = new SimpleDataSet(metadata());
- ActiveRepairService.instance.coordinators().stream()
+ ActiveRepairService.instance().coordinators().stream()
.flatMap(s -> s.getSessions().stream())
.flatMap(s -> s.getJobs().stream())
.forEach(s -> updateDataset(result, s));
@@ -291,7 +292,7 @@ public class LocalRepairTables
public DataSet data()
{
SimpleDataSet result = new SimpleDataSet(metadata());
- ActiveRepairService.instance.participates().stream()
+ ActiveRepairService.instance().participates().stream()
.forEach(s -> updateDataset(result, s));
return result;
}
@@ -301,7 +302,7 @@ public class LocalRepairTables
{
TimeUUID id = TimeUUIDType.instance.compose(partitionKey.getKey());
SimpleDataSet result = new SimpleDataSet(metadata());
- ParticipateState state = ActiveRepairService.instance.participate(id);
+ ParticipateState state = ActiveRepairService.instance().participate(id);
if (state != null)
updateDataset(result, state);
return result;
@@ -322,7 +323,7 @@ public class LocalRepairTables
result.column("preview_kind", state.previewKind.name());
if (state.repairedAt != 0)
result.column("repaired_at", new Date(state.repairedAt));
- result.column("validations", state.validationIds());
+ result.column("validations", ImmutableSet.copyOf(state.validationIds()));
result.column("ranges", toStringList(state.ranges));
}
}
@@ -352,7 +353,7 @@ public class LocalRepairTables
public DataSet data()
{
SimpleDataSet result = new SimpleDataSet(metadata());
- ActiveRepairService.instance.validations().stream()
+ ActiveRepairService.instance().validations().stream()
.forEach(s -> updateDataset(result, s));
return result;
}
@@ -362,7 +363,7 @@ public class LocalRepairTables
{
UUID id = UUIDType.instance.compose(partitionKey.getKey());
SimpleDataSet result = new SimpleDataSet(metadata());
- ValidationState state = ActiveRepairService.instance.validation(id);
+ ValidationState state = ActiveRepairService.instance().validation(id);
if (state != null)
updateDataset(result, state);
return result;
diff --git a/src/java/org/apache/cassandra/exceptions/RepairException.java b/src/java/org/apache/cassandra/exceptions/RepairException.java
index eb93c37192..554bdfcee4 100644
--- a/src/java/org/apache/cassandra/exceptions/RepairException.java
+++ b/src/java/org/apache/cassandra/exceptions/RepairException.java
@@ -17,6 +17,8 @@
*/
package org.apache.cassandra.exceptions;
+import javax.annotation.Nullable;
+
import org.apache.cassandra.repair.RepairJobDesc;
import org.apache.cassandra.streaming.PreviewKind;
@@ -27,9 +29,9 @@ public class RepairException extends Exception
{
private final boolean shouldLogWarn;
- private RepairException(RepairJobDesc desc, PreviewKind previewKind, String message, boolean shouldLogWarn)
+ private RepairException(@Nullable RepairJobDesc desc, PreviewKind previewKind, String message, boolean shouldLogWarn)
{
- this(desc.toString(previewKind != null ? previewKind : PreviewKind.NONE) + ' ' + message, shouldLogWarn);
+ this((desc == null ? "" : desc.toString(previewKind != null ? previewKind : PreviewKind.NONE)) + ' ' + message, shouldLogWarn);
}
private RepairException(String msg, boolean shouldLogWarn)
@@ -38,12 +40,12 @@ public class RepairException extends Exception
this.shouldLogWarn = shouldLogWarn;
}
- public static RepairException error(RepairJobDesc desc, PreviewKind previewKind, String message)
+ public static RepairException error(@Nullable RepairJobDesc desc, PreviewKind previewKind, String message)
{
return new RepairException(desc, previewKind, message, false);
}
- public static RepairException warn(RepairJobDesc desc, PreviewKind previewKind, String message)
+ public static RepairException warn(@Nullable RepairJobDesc desc, PreviewKind previewKind, String message)
{
return new RepairException(desc, previewKind, message, true);
}
diff --git a/src/java/org/apache/cassandra/gms/Gossiper.java b/src/java/org/apache/cassandra/gms/Gossiper.java
index e51d8f7ff0..43ee0aeea5 100644
--- a/src/java/org/apache/cassandra/gms/Gossiper.java
+++ b/src/java/org/apache/cassandra/gms/Gossiper.java
@@ -116,7 +116,7 @@ import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort;
* This class is not threadsafe and any state changes should happen in the gossip stage.
*/
-public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
+public class Gossiper implements IFailureDetectionEventListener, GossiperMBean, IGossiper
{
public static final String MBEAN_NAME = "org.apache.cassandra.net:type=Gossiper";
@@ -448,6 +448,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
*
* @param subscriber module which implements the IEndpointStateChangeSubscriber
*/
+ @Override
public void register(IEndpointStateChangeSubscriber subscriber)
{
subscribers.add(subscriber);
@@ -458,6 +459,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
*
* @param subscriber module which implements the IEndpointStateChangeSubscriber
*/
+ @Override
public void unregister(IEndpointStateChangeSubscriber subscriber)
{
subscribers.remove(subscriber);
@@ -1147,6 +1149,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
return storedTime == null ? computeExpireTime() : storedTime;
}
+ @Override
public EndpointState getEndpointStateForEndpoint(InetAddressAndPort ep)
{
return endpointStateMap.get(ep);
@@ -2319,13 +2322,6 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
return currentTimeMillis() + aVeryLongTime;
}
- @Nullable
- public CassandraVersion getReleaseVersion(InetAddressAndPort ep)
- {
- EndpointState state = getEndpointStateForEndpoint(ep);
- return state != null ? state.getReleaseVersion() : null;
- }
-
public Map<String, List<String>> getReleaseVersionsWithPort()
{
Map<String, List<String>> results = new HashMap<>();
diff --git a/src/java/org/apache/cassandra/gms/IGossiper.java b/src/java/org/apache/cassandra/gms/IGossiper.java
new file mode 100644
index 0000000000..0e33526d22
--- /dev/null
+++ b/src/java/org/apache/cassandra/gms/IGossiper.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.gms;
+
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.utils.CassandraVersion;
+
+public interface IGossiper
+{
+ void register(IEndpointStateChangeSubscriber subscriber);
+ void unregister(IEndpointStateChangeSubscriber subscriber);
+
+ @Nullable
+ EndpointState getEndpointStateForEndpoint(InetAddressAndPort ep);
+ @Nullable
+ default CassandraVersion getReleaseVersion(InetAddressAndPort ep)
+ {
+ EndpointState state = getEndpointStateForEndpoint(ep);
+ return state != null ? state.getReleaseVersion() : null;
+ }
+}
diff --git a/src/java/org/apache/cassandra/net/Message.java b/src/java/org/apache/cassandra/net/Message.java
index da50727d6f..c60ebee972 100644
--- a/src/java/org/apache/cassandra/net/Message.java
+++ b/src/java/org/apache/cassandra/net/Message.java
@@ -102,7 +102,7 @@ public class Message<T>
return header.verb;
}
- boolean isFailureResponse()
+ public boolean isFailureResponse()
{
return verb() == Verb.FAILURE_RSP;
}
@@ -301,6 +301,11 @@ public class Message<T>
return new Message<>(header.withParam(ParamType.FORWARD_TO, peers), payload);
}
+ public Message<T> withFrom(InetAddressAndPort from)
+ {
+ return new Message<>(header.withFrom(from), payload);
+ }
+
public Message<T> withFlag(MessageFlag flag)
{
return new Message<>(header.withFlag(flag), payload);
@@ -434,6 +439,11 @@ public class Message<T>
this.params = params;
}
+ Header withFrom(InetAddressAndPort from)
+ {
+ return new Header(id, verb, from, createdAtNanos, expiresAtNanos, flags, params);
+ }
+
Header withFlag(MessageFlag flag)
{
return new Header(id, verb, from, createdAtNanos, expiresAtNanos, flag.addTo(flags), params);
diff --git a/src/java/org/apache/cassandra/net/MessageDelivery.java b/src/java/org/apache/cassandra/net/MessageDelivery.java
new file mode 100644
index 0000000000..dd8c6ceeda
--- /dev/null
+++ b/src/java/org/apache/cassandra/net/MessageDelivery.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.net;
+
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.utils.concurrent.Future;
+
+public interface MessageDelivery
+{
+ public <REQ> void send(Message<REQ> message, InetAddressAndPort to);
+ public <REQ, RSP> void sendWithCallback(Message<REQ> message, InetAddressAndPort to, RequestCallback<RSP> cb);
+ public <REQ, RSP> void sendWithCallback(Message<REQ> message, InetAddressAndPort to, RequestCallback<RSP> cb, ConnectionType specifyConnection);
+ public <REQ, RSP> Future<Message<RSP>> sendWithResult(Message<REQ> message, InetAddressAndPort to);
+ public <V> void respond(V response, Message<?> message);
+}
diff --git a/src/java/org/apache/cassandra/net/MessagingService.java b/src/java/org/apache/cassandra/net/MessagingService.java
index a3ae7d8d09..c306fd87f6 100644
--- a/src/java/org/apache/cassandra/net/MessagingService.java
+++ b/src/java/org/apache/cassandra/net/MessagingService.java
@@ -205,39 +205,11 @@ import static org.apache.cassandra.utils.Throwables.maybeFail;
* implemented in {@link org.apache.cassandra.db.virtual.InternodeInboundTable} and
* {@link org.apache.cassandra.db.virtual.InternodeOutboundTable} respectively.
*/
-public class MessagingService extends MessagingServiceMBeanImpl
+public class MessagingService extends MessagingServiceMBeanImpl implements MessageDelivery
{
private static final Logger logger = LoggerFactory.getLogger(MessagingService.class);
// 8 bits version, so don't waste versions
- @Deprecated
- public static final int VERSION_30 = 10;
- @Deprecated
- public static final int VERSION_3014 = 11;
- public static final int VERSION_40 = 12;
- public static final int VERSION_50 = 13; // c14227 TTL overflow, 'uint' timestamps
- public static final int minimum_version = VERSION_40;
- public static final int current_version = DatabaseDescriptor.getStorageCompatibilityMode().isBefore(5) ? VERSION_40 : VERSION_50;
- static AcceptVersions accept_messaging = new AcceptVersions(minimum_version, current_version);
- static AcceptVersions accept_streaming = new AcceptVersions(current_version, current_version);
- static Map<Integer, Integer> versionOrdinalMap = Arrays.stream(Version.values()).collect(Collectors.toMap(v -> v.value, v -> v.ordinal()));
-
- /**
- * This is an optimisation to speed up the translation of the serialization
- * version to the {@link Version} enum ordinal.
- *
- * @param version the serialization version
- * @return a {@link Version} ordinal value
- */
- public static int getVersionOrdinal(int version)
- {
- Integer ordinal = versionOrdinalMap.get(version);
- if (ordinal == null)
- throw new IllegalStateException("Unkown serialization version: " + version);
-
- return ordinal;
- }
-
public enum Version
{
@Deprecated
@@ -245,8 +217,11 @@ public class MessagingService extends MessagingServiceMBeanImpl
@Deprecated
VERSION_3014(11),
VERSION_40(12),
+ // c14227 TTL overflow, 'uint' timestamps
VERSION_50(13);
+ public static final Version CURRENT = DatabaseDescriptor.getStorageCompatibilityMode().isBefore(5) ? VERSION_40 : VERSION_50;
+
public final int value;
Version(int value)
@@ -264,6 +239,36 @@ public class MessagingService extends MessagingServiceMBeanImpl
return Collections.unmodifiableList(versions);
}
}
+ // Maintance Note:
+ // Try to keep Version enum in-sync for testing. By having the versions in the enum tests can get access without forcing this class
+ // to load, which adds a lot of costs to each test
+ @Deprecated
+ public static final int VERSION_30 = 10;
+ @Deprecated
+ public static final int VERSION_3014 = 11;
+ public static final int VERSION_40 = 12;
+ public static final int VERSION_50 = 13; // c14227 TTL overflow, 'uint' timestamps
+ public static final int minimum_version = VERSION_40;
+ public static final int current_version = Version.CURRENT.value;
+ static AcceptVersions accept_messaging = new AcceptVersions(minimum_version, current_version);
+ static AcceptVersions accept_streaming = new AcceptVersions(current_version, current_version);
+ static Map<Integer, Integer> versionOrdinalMap = Arrays.stream(Version.values()).collect(Collectors.toMap(v -> v.value, v -> v.ordinal()));
+
+ /**
+ * This is an optimisation to speed up the translation of the serialization
+ * version to the {@link Version} enum ordinal.
+ *
+ * @param version the serialization version
+ * @return a {@link Version} ordinal value
+ */
+ public static int getVersionOrdinal(int version)
+ {
+ Integer ordinal = versionOrdinalMap.get(version);
+ if (ordinal == null)
+ throw new IllegalStateException("Unkown serialization version: " + version);
+
+ return ordinal;
+ }
private static class MSHandle
{
@@ -312,13 +317,14 @@ public class MessagingService extends MessagingServiceMBeanImpl
OutboundConnections.scheduleUnusedConnectionMonitoring(this, ScheduledExecutors.scheduledTasks, 1L, TimeUnit.HOURS);
}
- public <T> org.apache.cassandra.utils.concurrent.Future<Message<T>> sendWithResult(Message message, InetAddressAndPort to)
+ @Override
+ public <REQ, RSP> org.apache.cassandra.utils.concurrent.Future<Message<RSP>> sendWithResult(Message<REQ> message, InetAddressAndPort to)
{
- AsyncPromise<Message<T>> promise = new AsyncPromise<>();
- MessagingService.instance().sendWithCallback(message, to, new RequestCallback<T>()
+ AsyncPromise<Message<RSP>> promise = new AsyncPromise<>();
+ sendWithCallback(message, to, new RequestCallback<RSP>()
{
@Override
- public void onResponse(Message<T> msg)
+ public void onResponse(Message<RSP> msg)
{
promise.trySuccess(msg);
}
diff --git a/src/java/org/apache/cassandra/net/Verb.java b/src/java/org/apache/cassandra/net/Verb.java
index e5b4983d77..2481674f09 100644
--- a/src/java/org/apache/cassandra/net/Verb.java
+++ b/src/java/org/apache/cassandra/net/Verb.java
@@ -159,22 +159,23 @@ public enum Verb
SCHEMA_VERSION_REQ (20, P1, rpcTimeout, MIGRATION, () -> NoPayload.serializer, () -> SchemaVersionVerbHandler.instance, SCHEMA_VERSION_RSP ),
// repair; mostly doesn't use callbacks and sends responses as their own request messages, with matching sessions by uuid; should eventually harmonize and make idiomatic
+ // for the repair messages that implement retry logic, use rpcTimeout so the single request fails faster, then retries can be used to recover
REPAIR_RSP (100, P1, repairTimeout, REQUEST_RESPONSE, () -> NoPayload.serializer, () -> ResponseVerbHandler.instance ),
- VALIDATION_RSP (102, P1, longTimeout, ANTI_ENTROPY, () -> ValidationResponse.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ),
- VALIDATION_REQ (101, P1, repairTimeout, ANTI_ENTROPY, () -> ValidationRequest.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ),
- SYNC_RSP (104, P1, repairTimeout, ANTI_ENTROPY, () -> SyncResponse.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ),
- SYNC_REQ (103, P1, repairTimeout, ANTI_ENTROPY, () -> SyncRequest.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ),
- PREPARE_MSG (105, P1, repairTimeout, ANTI_ENTROPY, () -> PrepareMessage.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ),
- SNAPSHOT_MSG (106, P1, repairTimeout, ANTI_ENTROPY, () -> SnapshotMessage.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ),
- CLEANUP_MSG (107, P1, repairTimeout, ANTI_ENTROPY, () -> CleanupMessage.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ),
- PREPARE_CONSISTENT_RSP (109, P1, repairTimeout, ANTI_ENTROPY, () -> PrepareConsistentResponse.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ),
- PREPARE_CONSISTENT_REQ (108, P1, repairTimeout, ANTI_ENTROPY, () -> PrepareConsistentRequest.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ),
- FINALIZE_PROPOSE_MSG (110, P1, repairTimeout, ANTI_ENTROPY, () -> FinalizePropose.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ),
- FINALIZE_PROMISE_MSG (111, P1, repairTimeout, ANTI_ENTROPY, () -> FinalizePromise.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ),
- FINALIZE_COMMIT_MSG (112, P1, repairTimeout, ANTI_ENTROPY, () -> FinalizeCommit.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ),
- FAILED_SESSION_MSG (113, P1, repairTimeout, ANTI_ENTROPY, () -> FailSession.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ),
- STATUS_RSP (115, P1, repairTimeout, ANTI_ENTROPY, () -> StatusResponse.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ),
- STATUS_REQ (114, P1, repairTimeout, ANTI_ENTROPY, () -> StatusRequest.serializer, () -> RepairMessageVerbHandler.instance, REPAIR_RSP ),
+ VALIDATION_RSP (102, P1, repairValidationRspTimeout, ANTI_ENTROPY, () -> ValidationResponse.serializer, () -> RepairMessageVerbHandler.instance(), REPAIR_RSP ),
+ VALIDATION_REQ (101, P1, repairWithBackoffTimeout, ANTI_ENTROPY, () -> ValidationRequest.serializer, () -> RepairMessageVerbHandler.instance(), REPAIR_RSP ),
+ SYNC_RSP (104, P1, repairWithBackoffTimeout, ANTI_ENTROPY, () -> SyncResponse.serializer, () -> RepairMessageVerbHandler.instance(), REPAIR_RSP ),
+ SYNC_REQ (103, P1, repairWithBackoffTimeout, ANTI_ENTROPY, () -> SyncRequest.serializer, () -> RepairMessageVerbHandler.instance(), REPAIR_RSP ),
+ PREPARE_MSG (105, P1, repairWithBackoffTimeout, ANTI_ENTROPY, () -> PrepareMessage.serializer, () -> RepairMessageVerbHandler.instance(), REPAIR_RSP ),
+ SNAPSHOT_MSG (106, P1, repairWithBackoffTimeout, ANTI_ENTROPY, () -> SnapshotMessage.serializer, () -> RepairMessageVerbHandler.instance(), REPAIR_RSP ),
+ CLEANUP_MSG (107, P1, repairWithBackoffTimeout, ANTI_ENTROPY, () -> CleanupMessage.serializer, () -> RepairMessageVerbHandler.instance(), REPAIR_RSP ),
+ PREPARE_CONSISTENT_RSP (109, P1, repairTimeout, ANTI_ENTROPY, () -> PrepareConsistentResponse.serializer, () -> RepairMessageVerbHandler.instance(), REPAIR_RSP ),
+ PREPARE_CONSISTENT_REQ (108, P1, repairTimeout, ANTI_ENTROPY, () -> PrepareConsistentRequest.serializer, () -> RepairMessageVerbHandler.instance(), REPAIR_RSP ),
+ FINALIZE_PROPOSE_MSG (110, P1, repairTimeout, ANTI_ENTROPY, () -> FinalizePropose.serializer, () -> RepairMessageVerbHandler.instance(), REPAIR_RSP ),
+ FINALIZE_PROMISE_MSG (111, P1, repairTimeout, ANTI_ENTROPY, () -> FinalizePromise.serializer, () -> RepairMessageVerbHandler.instance(), REPAIR_RSP ),
+ FINALIZE_COMMIT_MSG (112, P1, repairTimeout, ANTI_ENTROPY, () -> FinalizeCommit.serializer, () -> RepairMessageVerbHandler.instance(), REPAIR_RSP ),
+ FAILED_SESSION_MSG (113, P1, repairTimeout, ANTI_ENTROPY, () -> FailSession.serializer, () -> RepairMessageVerbHandler.instance(), REPAIR_RSP ),
+ STATUS_RSP (115, P1, repairTimeout, ANTI_ENTROPY, () -> StatusResponse.serializer, () -> RepairMessageVerbHandler.instance(), REPAIR_RSP ),
+ STATUS_REQ (114, P1, repairTimeout, ANTI_ENTROPY, () -> StatusRequest.serializer, () -> RepairMessageVerbHandler.instance(), REPAIR_RSP ),
REPLICATION_DONE_RSP (82, P0, rpcTimeout, MISC, () -> NoPayload.serializer, () -> ResponseVerbHandler.instance ),
REPLICATION_DONE_REQ (22, P0, rpcTimeout, MISC, () -> NoPayload.serializer, () -> ReplicationDoneVerbHandler.instance, REPLICATION_DONE_RSP),
@@ -475,4 +476,16 @@ class VerbTimeouts
static final ToLongFunction<TimeUnit> pingTimeout = DatabaseDescriptor::getPingTimeout;
static final ToLongFunction<TimeUnit> longTimeout = units -> Math.max(DatabaseDescriptor.getRpcTimeout(units), units.convert(5L, TimeUnit.MINUTES));
static final ToLongFunction<TimeUnit> noTimeout = units -> { throw new IllegalStateException(); };
+
+ // repair verbs need to have different timeouts based off if retries are enabled or not
+ static final ToLongFunction<TimeUnit> repairWithBackoffTimeout = units -> {
+ if (!DatabaseDescriptor.getRepairRetrySpec().isEnabled())
+ return repairTimeout.applyAsLong(units);
+ return rpcTimeout.applyAsLong(units);
+ };
+ static final ToLongFunction<TimeUnit> repairValidationRspTimeout = units -> {
+ if (!DatabaseDescriptor.getRepairRetrySpec().isMerkleTreeRetriesEnabled())
+ return longTimeout.applyAsLong(units);
+ return rpcTimeout.applyAsLong(units);
+ };
}
diff --git a/src/java/org/apache/cassandra/repair/AbstractRepairTask.java b/src/java/org/apache/cassandra/repair/AbstractRepairTask.java
index d2a6f1a786..c525597f5c 100644
--- a/src/java/org/apache/cassandra/repair/AbstractRepairTask.java
+++ b/src/java/org/apache/cassandra/repair/AbstractRepairTask.java
@@ -31,8 +31,8 @@ import org.slf4j.LoggerFactory;
import org.apache.cassandra.concurrent.ExecutorPlus;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.repair.messages.RepairOption;
-import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.utils.concurrent.Future;
import org.apache.cassandra.utils.concurrent.FutureCombiner;
@@ -41,15 +41,17 @@ public abstract class AbstractRepairTask implements RepairTask
{
protected static final Logger logger = LoggerFactory.getLogger(AbstractRepairTask.class);
+ protected final RepairCoordinator coordinator;
+ protected final InetAddressAndPort broadcastAddressAndPort;
protected final RepairOption options;
protected final String keyspace;
- protected final RepairNotifier notifier;
- protected AbstractRepairTask(RepairOption options, String keyspace, RepairNotifier notifier)
+ protected AbstractRepairTask(RepairCoordinator coordinator)
{
- this.options = Objects.requireNonNull(options);
- this.keyspace = Objects.requireNonNull(keyspace);
- this.notifier = Objects.requireNonNull(notifier);
+ this.coordinator = Objects.requireNonNull(coordinator);
+ this.broadcastAddressAndPort = coordinator.ctx.broadcastAddressAndPort();
+ this.options = Objects.requireNonNull(coordinator.state.options);
+ this.keyspace = Objects.requireNonNull(coordinator.state.keyspace);
}
private List<RepairSession> submitRepairSessions(TimeUUID parentSession,
@@ -63,18 +65,18 @@ public abstract class AbstractRepairTask implements RepairTask
for (CommonRange commonRange : commonRanges)
{
logger.info("Starting RepairSession for {}", commonRange);
- RepairSession session = ActiveRepairService.instance.submitRepairSession(parentSession,
- commonRange,
- keyspace,
- options.getParallelism(),
- isIncremental,
- options.isPullRepair(),
- options.getPreviewKind(),
- options.optimiseStreams(),
- options.repairPaxos(),
- options.paxosOnly(),
- executor,
- cfnames);
+ RepairSession session = coordinator.ctx.repair().submitRepairSession(parentSession,
+ commonRange,
+ keyspace,
+ options.getParallelism(),
+ isIncremental,
+ options.isPullRepair(),
+ options.getPreviewKind(),
+ options.optimiseStreams(),
+ options.repairPaxos(),
+ options.paxosOnly(),
+ executor,
+ cfnames);
if (session == null)
continue;
session.addCallback(new RepairSessionCallback(session));
@@ -107,18 +109,20 @@ public abstract class AbstractRepairTask implements RepairTask
this.session = session;
}
+ @Override
public void onSuccess(RepairSessionResult result)
{
String message = String.format("Repair session %s for range %s finished", session.getId(),
session.ranges().toString());
- notifier.notifyProgress(message);
+ coordinator.notifyProgress(message);
}
+ @Override
public void onFailure(Throwable t)
{
String message = String.format("Repair session %s for range %s failed with error %s",
session.getId(), session.ranges().toString(), t.getMessage());
- notifier.notifyError(new RuntimeException(message, t));
+ coordinator.notifyError(new RuntimeException(message, t));
}
}
}
diff --git a/src/java/org/apache/cassandra/repair/AsymmetricRemoteSyncTask.java b/src/java/org/apache/cassandra/repair/AsymmetricRemoteSyncTask.java
index c050f2c1b0..475c389678 100644
--- a/src/java/org/apache/cassandra/repair/AsymmetricRemoteSyncTask.java
+++ b/src/java/org/apache/cassandra/repair/AsymmetricRemoteSyncTask.java
@@ -29,7 +29,6 @@ import org.apache.cassandra.repair.messages.SyncRequest;
import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.streaming.SessionSummary;
import org.apache.cassandra.tracing.Tracing;
-import org.apache.cassandra.utils.FBUtilities;
/**
* AsymmetricRemoteSyncTask sends {@link SyncRequest} to target node to repair(stream)
@@ -39,14 +38,14 @@ import org.apache.cassandra.utils.FBUtilities;
*/
public class AsymmetricRemoteSyncTask extends SyncTask implements CompletableRemoteSyncTask
{
- public AsymmetricRemoteSyncTask(RepairJobDesc desc, InetAddressAndPort to, InetAddressAndPort from, List<Range<Token>> differences, PreviewKind previewKind)
+ public AsymmetricRemoteSyncTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort to, InetAddressAndPort from, List<Range<Token>> differences, PreviewKind previewKind)
{
- super(desc, to, from, differences, previewKind);
+ super(ctx, desc, to, from, differences, previewKind);
}
public void startSync()
{
- InetAddressAndPort local = FBUtilities.getBroadcastAddressAndPort();
+ InetAddressAndPort local = ctx.broadcastAddressAndPort();
SyncRequest request = new SyncRequest(desc, local, nodePair.coordinator, nodePair.peer, rangesToSync, previewKind, true);
String message = String.format("Forwarding streaming repair of %d ranges to %s (to be streamed with %s)", request.ranges.size(), request.src, request.dst);
Tracing.traceRepair(message);
diff --git a/src/java/org/apache/cassandra/repair/IValidationManager.java b/src/java/org/apache/cassandra/repair/IValidationManager.java
new file mode 100644
index 0000000000..fce4b62ad5
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/IValidationManager.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair;
+
+import java.util.concurrent.Future;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+
+public interface IValidationManager
+{
+ Future<?> submitValidation(ColumnFamilyStore cfs, Validator validator);
+}
diff --git a/src/java/org/apache/cassandra/repair/IncrementalRepairTask.java b/src/java/org/apache/cassandra/repair/IncrementalRepairTask.java
index af1a234d34..7f5e5630cc 100644
--- a/src/java/org/apache/cassandra/repair/IncrementalRepairTask.java
+++ b/src/java/org/apache/cassandra/repair/IncrementalRepairTask.java
@@ -25,26 +25,21 @@ import com.google.common.collect.ImmutableSet;
import org.apache.cassandra.concurrent.ExecutorPlus;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.repair.consistent.CoordinatorSession;
-import org.apache.cassandra.repair.messages.RepairOption;
-import org.apache.cassandra.service.ActiveRepairService;
-import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.utils.concurrent.Future;
public class IncrementalRepairTask extends AbstractRepairTask
{
private final TimeUUID parentSession;
- private final RepairRunnable.NeighborsAndRanges neighborsAndRanges;
+ private final RepairCoordinator.NeighborsAndRanges neighborsAndRanges;
private final String[] cfnames;
- protected IncrementalRepairTask(RepairOption options,
- String keyspace,
- RepairNotifier notifier,
+ protected IncrementalRepairTask(RepairCoordinator coordinator,
TimeUUID parentSession,
- RepairRunnable.NeighborsAndRanges neighborsAndRanges,
+ RepairCoordinator.NeighborsAndRanges neighborsAndRanges,
String[] cfnames)
{
- super(options, keyspace, notifier);
+ super(coordinator);
this.parentSession = parentSession;
this.neighborsAndRanges = neighborsAndRanges;
this.cfnames = cfnames;
@@ -62,12 +57,12 @@ public class IncrementalRepairTask extends AbstractRepairTask
// the local node also needs to be included in the set of participants, since coordinator sessions aren't persisted
Set<InetAddressAndPort> allParticipants = ImmutableSet.<InetAddressAndPort>builder()
.addAll(neighborsAndRanges.participants)
- .add(FBUtilities.getBroadcastAddressAndPort())
+ .add(broadcastAddressAndPort)
.build();
// Not necessary to include self for filtering. The common ranges only contains neighbhor node endpoints.
List<CommonRange> allRanges = neighborsAndRanges.filterCommonRanges(keyspace, cfnames);
- CoordinatorSession coordinatorSession = ActiveRepairService.instance.consistent.coordinated.registerSession(parentSession, allParticipants, neighborsAndRanges.shouldExcludeDeadParticipants);
+ CoordinatorSession coordinatorSession = coordinator.ctx.repair().consistent.coordinated.registerSession(parentSession, allParticipants, neighborsAndRanges.shouldExcludeDeadParticipants);
return coordinatorSession.execute(() -> runRepair(parentSession, true, executor, allRanges, cfnames));
diff --git a/src/java/org/apache/cassandra/repair/LocalSyncTask.java b/src/java/org/apache/cassandra/repair/LocalSyncTask.java
index 5df05c79c2..379ba4b2a1 100644
--- a/src/java/org/apache/cassandra/repair/LocalSyncTask.java
+++ b/src/java/org/apache/cassandra/repair/LocalSyncTask.java
@@ -64,13 +64,13 @@ public class LocalSyncTask extends SyncTask implements StreamEventHandler
private final AtomicBoolean active = new AtomicBoolean(true);
private final Promise<StreamPlan> planPromise = new AsyncPromise<>();
- public LocalSyncTask(RepairJobDesc desc, InetAddressAndPort local, InetAddressAndPort remote,
+ public LocalSyncTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort local, InetAddressAndPort remote,
List<Range<Token>> diff, TimeUUID pendingRepair,
boolean requestRanges, boolean transferRanges, PreviewKind previewKind)
{
- super(desc, local, remote, diff, previewKind);
+ super(ctx, desc, local, remote, diff, previewKind);
Preconditions.checkArgument(requestRanges || transferRanges, "Nothing to do in a sync job");
- Preconditions.checkArgument(local.equals(FBUtilities.getBroadcastAddressAndPort()));
+ Preconditions.checkArgument(local.equals(ctx.broadcastAddressAndPort()));
this.pendingRepair = pendingRepair;
this.requestRanges = requestRanges;
@@ -119,7 +119,7 @@ public class LocalSyncTask extends SyncTask implements StreamEventHandler
Tracing.traceRepair(message);
StreamPlan plan = createStreamPlan();
- plan.execute();
+ ctx.streamExecutor().execute(plan);
planPromise.setSuccess(plan);
}
}
@@ -130,6 +130,7 @@ public class LocalSyncTask extends SyncTask implements StreamEventHandler
return true;
}
+ @Override
public void handleStreamEvent(StreamEvent event)
{
if (state == null)
@@ -193,8 +194,9 @@ public class LocalSyncTask extends SyncTask implements StreamEventHandler
}
@Override
- public void abort()
+ public void abort(Throwable reason)
{
+ super.abort(reason);
planPromise.addCallback((plan, cause) ->
{
assert plan != null : "StreamPlan future should never be completed exceptionally";
diff --git a/src/java/org/apache/cassandra/repair/NormalRepairTask.java b/src/java/org/apache/cassandra/repair/NormalRepairTask.java
index 56a03f8816..bdc15cce53 100644
--- a/src/java/org/apache/cassandra/repair/NormalRepairTask.java
+++ b/src/java/org/apache/cassandra/repair/NormalRepairTask.java
@@ -20,7 +20,6 @@ package org.apache.cassandra.repair;
import java.util.List;
import org.apache.cassandra.concurrent.ExecutorPlus;
-import org.apache.cassandra.repair.messages.RepairOption;
import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.utils.concurrent.Future;
@@ -30,14 +29,12 @@ public class NormalRepairTask extends AbstractRepairTask
private final List<CommonRange> commonRanges;
private final String[] cfnames;
- protected NormalRepairTask(RepairOption options,
- String keyspace,
- RepairNotifier notifier,
+ protected NormalRepairTask(RepairCoordinator coordinator,
TimeUUID parentSession,
List<CommonRange> commonRanges,
String[] cfnames)
{
- super(options, keyspace, notifier);
+ super(coordinator);
this.parentSession = parentSession;
this.commonRanges = commonRanges;
this.cfnames = cfnames;
diff --git a/src/java/org/apache/cassandra/repair/PreviewRepairTask.java b/src/java/org/apache/cassandra/repair/PreviewRepairTask.java
index 728a813a2a..5b861f71b3 100644
--- a/src/java/org/apache/cassandra/repair/PreviewRepairTask.java
+++ b/src/java/org/apache/cassandra/repair/PreviewRepairTask.java
@@ -32,7 +32,6 @@ import org.apache.cassandra.dht.Token;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.metrics.RepairMetrics;
import org.apache.cassandra.repair.consistent.SyncStatSummary;
-import org.apache.cassandra.repair.messages.RepairOption;
import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.utils.DiagnosticSnapshotService;
import org.apache.cassandra.utils.TimeUUID;
@@ -45,9 +44,9 @@ public class PreviewRepairTask extends AbstractRepairTask
private final String[] cfnames;
private volatile String successMessage = name() + " completed successfully";
- protected PreviewRepairTask(RepairOption options, String keyspace, RepairNotifier notifier, TimeUUID parentSession, List<CommonRange> commonRanges, String[] cfnames)
+ protected PreviewRepairTask(RepairCoordinator coordinator, TimeUUID parentSession, List<CommonRange> commonRanges, String[] cfnames)
{
- super(options, keyspace, notifier);
+ super(coordinator);
this.parentSession = parentSession;
this.commonRanges = commonRanges;
this.cfnames = cfnames;
@@ -91,7 +90,7 @@ public class PreviewRepairTask extends AbstractRepairTask
maybeSnapshotReplicas(parentSession, keyspace, result.results.get()); // we know its present as summary used it
}
successMessage += "; " + message;
- notifier.notification(message);
+ coordinator.notification(message);
return result;
});
diff --git a/src/java/org/apache/cassandra/repair/RepairRunnable.java b/src/java/org/apache/cassandra/repair/RepairCoordinator.java
similarity index 82%
rename from src/java/org/apache/cassandra/repair/RepairRunnable.java
rename to src/java/org/apache/cassandra/repair/RepairCoordinator.java
index c56601010f..3d1eab3e9e 100644
--- a/src/java/org/apache/cassandra/repair/RepairRunnable.java
+++ b/src/java/org/apache/cassandra/repair/RepairCoordinator.java
@@ -17,7 +17,6 @@
*/
package org.apache.cassandra.repair;
-import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
@@ -29,6 +28,9 @@ import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import java.util.function.Supplier;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
@@ -38,7 +40,8 @@ import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
-import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.locator.RangesAtEndpoint;
+import org.apache.cassandra.utils.*;
import org.apache.cassandra.utils.concurrent.Future;
import org.apache.commons.lang3.time.DurationFormatUtils;
import org.slf4j.Logger;
@@ -57,7 +60,6 @@ import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.RepairException;
-import org.apache.cassandra.gms.FailureDetector;
import org.apache.cassandra.locator.EndpointsForRange;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.Replica;
@@ -74,44 +76,51 @@ import org.apache.cassandra.tracing.TraceKeyspace;
import org.apache.cassandra.tracing.TraceState;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.transport.messages.ResultMessage;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.Throwables;
-import org.apache.cassandra.utils.WrappedRunnable;
import org.apache.cassandra.utils.progress.ProgressEvent;
import org.apache.cassandra.utils.progress.ProgressEventNotifier;
import org.apache.cassandra.utils.progress.ProgressEventType;
import org.apache.cassandra.utils.progress.ProgressListener;
-import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
import static org.apache.cassandra.repair.state.AbstractState.COMPLETE;
import static org.apache.cassandra.repair.state.AbstractState.INIT;
import static org.apache.cassandra.service.QueryState.forInternalCalls;
-import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
-public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNotifier
+public class RepairCoordinator implements Runnable, ProgressEventNotifier, RepairNotifier
{
- private static final Logger logger = LoggerFactory.getLogger(RepairRunnable.class);
+ private static final Logger logger = LoggerFactory.getLogger(RepairCoordinator.class);
private static final AtomicInteger THREAD_COUNTER = new AtomicInteger(1);
public final CoordinatorState state;
- private final StorageService storageService;
-
private final String tag;
+ private final BiFunction<String, String[], Iterable<ColumnFamilyStore>> validColumnFamilies;
+ private final Function<String, RangesAtEndpoint> getLocalReplicas;
private final List<ProgressListener> listeners = new ArrayList<>();
private final AtomicReference<Throwable> firstError = new AtomicReference<>(null);
+ final SharedContext ctx;
private TraceState traceState;
- public RepairRunnable(StorageService storageService, int cmd, RepairOption options, String keyspace)
+ public RepairCoordinator(StorageService storageService, int cmd, RepairOption options, String keyspace)
{
- this.state = new CoordinatorState(cmd, keyspace, options);
- this.storageService = storageService;
+ this(SharedContext.Global.instance,
+ (ks, tables) -> storageService.getValidColumnFamilies(false, false, ks, tables),
+ storageService::getLocalReplicas,
+ cmd, options, keyspace);
+ }
+ RepairCoordinator(SharedContext ctx,
+ BiFunction<String, String[], Iterable<ColumnFamilyStore>> validColumnFamilies,
+ Function<String, RangesAtEndpoint> getLocalReplicas,
+ int cmd, RepairOption options, String keyspace)
+ {
+ this.ctx = ctx;
+ this.state = new CoordinatorState(ctx.clock(), cmd, keyspace, options);
this.tag = "repair:" + cmd;
- ActiveRepairService.instance.register(state);
+ this.validColumnFamilies = validColumnFamilies;
+ this.getLocalReplicas = getLocalReplicas;
+ ctx.repair().register(state);
}
@Override
@@ -187,7 +196,7 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNo
{
state.phase.success(msg);
fireProgressEvent(jmxEvent(ProgressEventType.SUCCESS, msg));
- ActiveRepairService.instance.recordRepairStatus(state.cmd, ActiveRepairService.ParentRepairStatus.COMPLETED,
+ ctx.repair().recordRepairStatus(state.cmd, ActiveRepairService.ParentRepairStatus.COMPLETED,
ImmutableList.of(msg));
complete(null);
}
@@ -197,14 +206,14 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNo
if (reason == null)
{
Throwable error = firstError.get();
- reason = error != null ? error.getMessage() : "Some repair failed";
+ reason = error != null ? error.toString() : "Some repair failed";
}
state.phase.fail(reason);
String completionMessage = String.format("Repair command #%d finished with error", state.cmd);
// Note we rely on the first message being the reason for the failure
// when inspecting this state from RepairRunner.queryForCompletedRepair
- ActiveRepairService.instance.recordRepairStatus(state.cmd, ParentRepairStatus.FAILED,
+ ctx.repair().recordRepairStatus(state.cmd, ParentRepairStatus.FAILED,
ImmutableList.of(reason, completionMessage));
complete(completionMessage);
@@ -222,7 +231,7 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNo
fireProgressEvent(jmxEvent(ProgressEventType.COMPLETE, msg));
logger.info(state.options.getPreviewKind().logPrefix(state.id) + msg);
- ActiveRepairService.instance.removeParentRepairSession(state.id);
+ ctx.repair().removeParentRepairSession(state.id);
TraceState localState = traceState;
if (state.options.isTraced() && localState != null)
{
@@ -251,17 +260,17 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNo
{
skip(e.getMessage());
}
- catch (Exception | Error e)
+ catch (Throwable e)
{
notifyError(e);
fail(e.getMessage());
}
}
- private void runMayThrow() throws Exception
+ private void runMayThrow() throws Throwable
{
state.phase.setup();
- ActiveRepairService.instance.recordRepairStatus(state.cmd, ParentRepairStatus.IN_PROGRESS, ImmutableList.of());
+ ctx.repair().recordRepairStatus(state.cmd, ParentRepairStatus.IN_PROGRESS, ImmutableList.of());
List<ColumnFamilyStore> columnFamilies = getColumnFamilies();
String[] cfnames = columnFamilies.stream().map(cfs -> cfs.name).toArray(String[]::new);
@@ -276,15 +285,36 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNo
maybeStoreParentRepairStart(cfnames);
- prepare(columnFamilies, neighborsAndRanges.participants, neighborsAndRanges.shouldExcludeDeadParticipants);
-
- repair(cfnames, neighborsAndRanges);
+ prepare(columnFamilies, neighborsAndRanges.participants, neighborsAndRanges.shouldExcludeDeadParticipants)
+ .flatMap(ignore -> repair(cfnames, neighborsAndRanges))
+ .addCallback((pair, failure) -> {
+ if (failure != null)
+ {
+ notifyError(failure);
+ fail(failure.getMessage());
+ }
+ else
+ {
+ state.phase.repairCompleted();
+ CoordinatedRepairResult result = pair.left;
+ maybeStoreParentRepairSuccess(result.successfulRanges);
+ if (result.hasFailed())
+ {
+ fail(null);
+ }
+ else
+ {
+ success(pair.right.get());
+ ctx.repair().cleanUp(state.id, neighborsAndRanges.participants);
+ }
+ }
+ });
}
- private List<ColumnFamilyStore> getColumnFamilies() throws IOException
+ private List<ColumnFamilyStore> getColumnFamilies()
{
String[] columnFamilies = state.options.getColumnFamilies().toArray(new String[state.options.getColumnFamilies().size()]);
- Iterable<ColumnFamilyStore> validColumnFamilies = storageService.getValidColumnFamilies(false, false, state.keyspace, columnFamilies);
+ Iterable<ColumnFamilyStore> validColumnFamilies = this.validColumnFamilies.apply(state.keyspace, columnFamilies);
if (Iterables.isEmpty(validColumnFamilies))
throw new SkipRepairException(String.format("%s Empty keyspace, skipping repair: %s", state.id, state.keyspace));
@@ -327,11 +357,11 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNo
//pre-calculate output of getLocalReplicas and pass it to getNeighbors to increase performance and prevent
//calculation multiple times
- Iterable<Range<Token>> keyspaceLocalRanges = storageService.getLocalReplicas(state.keyspace).ranges();
+ Iterable<Range<Token>> keyspaceLocalRanges = getLocalReplicas.apply(state.keyspace).ranges();
for (Range<Token> range : state.options.getRanges())
{
- EndpointsForRange neighbors = ActiveRepairService.getNeighbors(state.keyspace, keyspaceLocalRanges, range,
+ EndpointsForRange neighbors = ctx.repair().getNeighbors(state.keyspace, keyspaceLocalRanges, range,
state.options.getDataCenters(),
state.options.getHosts());
if (neighbors.isEmpty())
@@ -361,7 +391,7 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNo
if (shouldExcludeDeadParticipants)
{
- Set<InetAddressAndPort> actualNeighbors = Sets.newHashSet(Iterables.filter(allNeighbors, FailureDetector.instance::isAlive));
+ Set<InetAddressAndPort> actualNeighbors = Sets.newHashSet(Iterables.filter(allNeighbors, ctx.failureDetector()::isAlive));
shouldExcludeDeadParticipants = !allNeighbors.equals(actualNeighbors);
allNeighbors = actualNeighbors;
}
@@ -392,68 +422,46 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNo
}
}
- private void prepare(List<ColumnFamilyStore> columnFamilies, Set<InetAddressAndPort> allNeighbors, boolean force)
+ private Future<?> prepare(List<ColumnFamilyStore> columnFamilies, Set<InetAddressAndPort> allNeighbors, boolean force)
{
state.phase.prepareStart();
- try (Timer.Context ignore = Keyspace.open(state.keyspace).metric.repairPrepareTime.time())
- {
- ActiveRepairService.instance.prepareForRepair(state.id, FBUtilities.getBroadcastAddressAndPort(), allNeighbors, state.options, force, columnFamilies);
- }
- state.phase.prepareComplete();
+ Timer timer = Keyspace.open(state.keyspace).metric.repairPrepareTime;
+ long startNanos = ctx.clock().nanoTime();
+ return ctx.repair().prepareForRepair(state.id, ctx.broadcastAddressAndPort(), allNeighbors, state.options, force, columnFamilies)
+ .map(ignore -> {
+ timer.update(ctx.clock().nanoTime() - startNanos, TimeUnit.NANOSECONDS);
+ state.phase.prepareComplete();
+ return null;
+ });
}
- private void repair(String[] cfnames, NeighborsAndRanges neighborsAndRanges)
+ private Future<Pair<CoordinatedRepairResult, Supplier<String>>> repair(String[] cfnames, NeighborsAndRanges neighborsAndRanges)
{
RepairTask task;
if (state.options.isPreview())
{
- task = new PreviewRepairTask(state.options, state.keyspace, this, state.id, neighborsAndRanges.filterCommonRanges(state.keyspace, cfnames), cfnames);
+ task = new PreviewRepairTask(this, state.id, neighborsAndRanges.filterCommonRanges(state.keyspace, cfnames), cfnames);
}
else if (state.options.isIncremental())
{
- task = new IncrementalRepairTask(state.options, state.keyspace, this, state.id, neighborsAndRanges, cfnames);
+ task = new IncrementalRepairTask(this, state.id, neighborsAndRanges, cfnames);
}
else
{
- task = new NormalRepairTask(state.options, state.keyspace, this, state.id, neighborsAndRanges.filterCommonRanges(state.keyspace, cfnames), cfnames);
+ task = new NormalRepairTask(this, state.id, neighborsAndRanges.filterCommonRanges(state.keyspace, cfnames), cfnames);
}
ExecutorPlus executor = createExecutor();
state.phase.repairSubmitted();
- Future<CoordinatedRepairResult> f = task.perform(executor);
- f.addCallback((result, failure) -> {
- state.phase.repairCompleted();
- try
- {
- if (failure != null)
- {
- notifyError(failure);
- fail(failure.getMessage());
- }
- else
- {
- maybeStoreParentRepairSuccess(result.successfulRanges);
- if (result.hasFailed())
- {
- fail(null);
- }
- else
- {
- success(task.successMessage());
- ActiveRepairService.instance.cleanUp(state.id, neighborsAndRanges.participants);
- }
- }
- }
- finally
- {
- executor.shutdown();
- }
- });
+ return task.perform(executor)
+ // after adding the callback java could no longer infer the type...
+ .<Pair<CoordinatedRepairResult, Supplier<String>>>map(r -> Pair.create(r, task::successMessage))
+ .addCallback((s, f) -> executor.shutdown());
}
private ExecutorPlus createExecutor()
{
- return executorFactory()
+ return ctx.executorFactory()
.localAware()
.withJmxInternal()
.pooled("Repair#" + state.cmd, state.options.getJobThreads());
@@ -480,7 +488,7 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNo
private Thread createQueryThread(final TimeUUID sessionId)
{
- return executorFactory().startThread("Repair-Runnable-" + THREAD_COUNTER.incrementAndGet(), new WrappedRunnable()
+ return ctx.executorFactory().startThread("Repair-Runnable-" + THREAD_COUNTER.incrementAndGet(), new WrappedRunnable()
{
// Query events within a time interval that overlaps the last by one second. Ignore duplicates. Ignore local traces.
// Wake up upon local trace activity. Query when notified of trace activity with a timeout that doubles every two timeouts.
@@ -495,13 +503,13 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNo
SelectStatement statement = (SelectStatement) QueryProcessor.parseStatement(query).prepare(ClientState.forInternalCalls());
ByteBuffer sessionIdBytes = sessionId.toBytes();
- InetAddressAndPort source = FBUtilities.getBroadcastAddressAndPort();
+ InetAddressAndPort source = ctx.broadcastAddressAndPort();
HashSet<UUID>[] seen = new HashSet[]{ new HashSet<>(), new HashSet<>() };
int si = 0;
UUID uuid;
- long tlast = currentTimeMillis(), tcur;
+ long tlast = ctx.clock().currentTimeMillis(), tcur;
TraceState.Status status;
long minWaitMillis = 125;
@@ -522,11 +530,11 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNo
shouldDouble = false;
}
ByteBuffer tminBytes = TimeUUID.minAtUnixMillis(tlast - 1000).toBytes();
- ByteBuffer tmaxBytes = TimeUUID.maxAtUnixMillis(tcur = currentTimeMillis()).toBytes();
+ ByteBuffer tmaxBytes = TimeUUID.maxAtUnixMillis(tcur = ctx.clock().currentTimeMillis()).toBytes();
QueryOptions options = QueryOptions.forInternalCalls(ConsistencyLevel.ONE, Lists.newArrayList(sessionIdBytes,
tminBytes,
tmaxBytes));
- ResultMessage.Rows rows = statement.execute(forInternalCalls(), options, nanoTime());
+ ResultMessage.Rows rows = statement.execute(forInternalCalls(), options, ctx.clock().nanoTime());
UntypedResultSet result = UntypedResultSet.create(rows.result);
for (UntypedResultSet.Row r : result)
diff --git a/src/java/org/apache/cassandra/repair/RepairJob.java b/src/java/org/apache/cassandra/repair/RepairJob.java
index 7d11b0597c..8fec3e9d54 100644
--- a/src/java/org/apache/cassandra/repair/RepairJob.java
+++ b/src/java/org/apache/cassandra/repair/RepairJob.java
@@ -19,16 +19,19 @@ package org.apache.cassandra.repair;
import java.util.*;
import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.Executor;
import java.util.function.Predicate;
import java.util.function.Function;
import java.util.stream.Collectors;
+import javax.annotation.Nullable;
+
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.common.util.concurrent.*;
-import org.apache.cassandra.concurrent.ExecutorPlus;
+
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.repair.state.JobState;
@@ -45,7 +48,6 @@ import org.apache.cassandra.repair.asymmetric.PreferedNodeFilter;
import org.apache.cassandra.repair.asymmetric.ReduceHelper;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.schema.SystemDistributedKeyspace;
-import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
@@ -60,7 +62,6 @@ import org.apache.cassandra.utils.concurrent.ImmediateFuture;
import static org.apache.cassandra.config.DatabaseDescriptor.paxosRepairEnabled;
import static org.apache.cassandra.service.paxos.Paxos.useV2;
-import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
/**
* RepairJob runs repair on given ColumnFamily.
@@ -69,11 +70,12 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
{
private static final Logger logger = LoggerFactory.getLogger(RepairJob.class);
+ private final SharedContext ctx;
public final JobState state;
private final RepairJobDesc desc;
private final RepairSession session;
private final RepairParallelism parallelismDegree;
- private final ExecutorPlus taskExecutor;
+ private final Executor taskExecutor;
@VisibleForTesting
final List<ValidationTask> validationTasks = new CopyOnWriteArrayList<>();
@@ -88,16 +90,17 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
*/
public RepairJob(RepairSession session, String columnFamily)
{
+ this.ctx = session.ctx;
this.session = session;
this.taskExecutor = session.taskExecutor;
this.parallelismDegree = session.parallelismDegree;
this.desc = new RepairJobDesc(session.state.parentRepairSession, session.getId(), session.state.keyspace, columnFamily, session.state.commonRange.ranges);
- this.state = new JobState(desc, session.state.commonRange.endpoints);
+ this.state = new JobState(ctx.clock(), desc, session.state.commonRange.endpoints);
}
public long getNowInSeconds()
{
- long nowInSeconds = FBUtilities.nowInSeconds();
+ long nowInSeconds = ctx.clock().nowInSeconds();
if (session.previewKind == PreviewKind.REPAIRED)
{
return nowInSeconds + DatabaseDescriptor.getValidationPreviewPurgeHeadStartInSec();
@@ -121,7 +124,7 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
ColumnFamilyStore cfs = ks.getColumnFamilyStore(desc.columnFamily);
cfs.metric.repairsStarted.inc();
List<InetAddressAndPort> allEndpoints = new ArrayList<>(session.state.commonRange.endpoints);
- allEndpoints.add(FBUtilities.getBroadcastAddressAndPort());
+ allEndpoints.add(ctx.broadcastAddressAndPort());
Future<List<TreeResponse>> treeResponses;
Future<Void> paxosRepair;
@@ -176,7 +179,7 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
state.phase.snapshotsSubmitted();
for (InetAddressAndPort endpoint : allEndpoints)
{
- SnapshotTask snapshotTask = new SnapshotTask(desc, endpoint);
+ SnapshotTask snapshotTask = new SnapshotTask(ctx, desc, endpoint);
snapshotTasks.add(snapshotTask);
taskExecutor.execute(snapshotTask);
}
@@ -231,9 +234,7 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
public void onFailure(Throwable t)
{
state.phase.fail(t);
- // Make sure all validation tasks have cleaned up the off-heap Merkle trees they might contain.
- validationTasks.forEach(ValidationTask::abort);
- syncTasks.forEach(SyncTask::abort);
+ abort(t);
if (!session.previewKind.isPreview())
{
@@ -248,6 +249,17 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
}, taskExecutor);
}
+ public synchronized void abort(@Nullable Throwable reason)
+ {
+ if (reason == null)
+ reason = new RuntimeException("Abort");
+ // Make sure all validation tasks have cleaned up the off-heap Merkle trees they might contain.
+ for (ValidationTask v : validationTasks)
+ v.abort(reason);
+ for (SyncTask s : syncTasks)
+ s.abort(reason);
+ }
+
private boolean isTransient(InetAddressAndPort ep)
{
return session.state.commonRange.transEndpoints.contains(ep);
@@ -255,10 +267,9 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
private Future<List<SyncStat>> standardSyncing(List<TreeResponse> trees)
{
- state.phase.streamSubmitted();
- List<SyncTask> syncTasks = createStandardSyncTasks(desc,
+ List<SyncTask> syncTasks = createStandardSyncTasks(ctx, desc,
trees,
- FBUtilities.getLocalAddressAndPort(),
+ ctx.broadcastAddressAndPort(),
this::isTransient,
session.isIncremental,
session.pullRepair,
@@ -266,7 +277,8 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
return executeTasks(syncTasks);
}
- static List<SyncTask> createStandardSyncTasks(RepairJobDesc desc,
+ static List<SyncTask> createStandardSyncTasks(SharedContext ctx,
+ RepairJobDesc desc,
List<TreeResponse> trees,
InetAddressAndPort local,
Predicate<InetAddressAndPort> isTransient,
@@ -274,7 +286,7 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
boolean pullRepair,
PreviewKind previewKind)
{
- long startedAt = currentTimeMillis();
+ long startedAt = ctx.clock().currentTimeMillis();
List<SyncTask> syncTasks = new ArrayList<>();
// We need to difference all trees one against another
for (int i = 0; i < trees.size() - 1; ++i)
@@ -309,7 +321,7 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
if (!requestRanges && !transferRanges)
continue;
- task = new LocalSyncTask(desc, self.endpoint, remote.endpoint, differences, isIncremental ? desc.parentSessionId : null,
+ task = new LocalSyncTask(ctx, desc, self.endpoint, remote.endpoint, differences, isIncremental ? desc.parentSessionId : null,
requestRanges, transferRanges, previewKind);
}
else if (isTransient.test(r1.endpoint) || isTransient.test(r2.endpoint))
@@ -317,11 +329,11 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
// Stream only from transient replica
TreeResponse streamFrom = isTransient.test(r1.endpoint) ? r1 : r2;
TreeResponse streamTo = isTransient.test(r1.endpoint) ? r2 : r1;
- task = new AsymmetricRemoteSyncTask(desc, streamTo.endpoint, streamFrom.endpoint, differences, previewKind);
+ task = new AsymmetricRemoteSyncTask(ctx, desc, streamTo.endpoint, streamFrom.endpoint, differences, previewKind);
}
else
{
- task = new SymmetricRemoteSyncTask(desc, r1.endpoint, r2.endpoint, differences, previewKind);
+ task = new SymmetricRemoteSyncTask(ctx, desc, r1.endpoint, r2.endpoint, differences, previewKind);
}
syncTasks.add(task);
}
@@ -329,14 +341,14 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
}
trees.get(trees.size() - 1).trees.release();
logger.info("Created {} sync tasks based on {} merkle tree responses for {} (took: {}ms)",
- syncTasks.size(), trees.size(), desc.parentSessionId, currentTimeMillis() - startedAt);
+ syncTasks.size(), trees.size(), desc.parentSessionId, ctx.clock().currentTimeMillis() - startedAt);
return syncTasks;
}
private Future<List<SyncStat>> optimisedSyncing(List<TreeResponse> trees)
{
- state.phase.streamSubmitted();
- List<SyncTask> syncTasks = createOptimisedSyncingSyncTasks(desc,
+ List<SyncTask> syncTasks = createOptimisedSyncingSyncTasks(ctx,
+ desc,
trees,
FBUtilities.getLocalAddressAndPort(),
this::isTransient,
@@ -352,9 +364,12 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
{
try
{
- ActiveRepairService.instance.getParentRepairSession(desc.parentSessionId);
+ ctx.repair().getParentRepairSession(desc.parentSessionId);
syncTasks.addAll(tasks);
+ if (!tasks.isEmpty())
+ state.phase.streamSubmitted();
+
for (SyncTask task : tasks)
{
if (!task.isLocal())
@@ -385,7 +400,8 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
}
}
- static List<SyncTask> createOptimisedSyncingSyncTasks(RepairJobDesc desc,
+ static List<SyncTask> createOptimisedSyncingSyncTasks(SharedContext ctx,
+ RepairJobDesc desc,
List<TreeResponse> trees,
InetAddressAndPort local,
Predicate<InetAddressAndPort> isTransient,
@@ -393,7 +409,7 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
boolean isIncremental,
PreviewKind previewKind)
{
- long startedAt = currentTimeMillis();
+ long startedAt = ctx.clock().currentTimeMillis();
List<SyncTask> syncTasks = new ArrayList<>();
// We need to difference all trees one against another
DifferenceHolder diffHolder = new DifferenceHolder(trees);
@@ -427,12 +443,12 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
SyncTask task;
if (address.equals(local))
{
- task = new LocalSyncTask(desc, address, fetchFrom, toFetch, isIncremental ? desc.parentSessionId : null,
+ task = new LocalSyncTask(ctx, desc, address, fetchFrom, toFetch, isIncremental ? desc.parentSessionId : null,
true, false, previewKind);
}
else
{
- task = new AsymmetricRemoteSyncTask(desc, address, fetchFrom, toFetch, previewKind);
+ task = new AsymmetricRemoteSyncTask(ctx, desc, address, fetchFrom, toFetch, previewKind);
}
syncTasks.add(task);
@@ -444,14 +460,14 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
}
}
logger.info("Created {} optimised sync tasks based on {} merkle tree responses for {} (took: {}ms)",
- syncTasks.size(), trees.size(), desc.parentSessionId, currentTimeMillis() - startedAt);
+ syncTasks.size(), trees.size(), desc.parentSessionId, ctx.clock().currentTimeMillis() - startedAt);
logger.trace("Optimised sync tasks for {}: {}", desc.parentSessionId, syncTasks);
return syncTasks;
}
private String getDC(InetAddressAndPort address)
{
- return DatabaseDescriptor.getEndpointSnitch().getDatacenter(address);
+ return ctx.snitch().getDatacenter(address);
}
/**
@@ -582,7 +598,7 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
private ValidationTask newValidationTask(InetAddressAndPort endpoint, long nowInSec)
{
- ValidationTask task = new ValidationTask(desc, endpoint, nowInSec, session.previewKind);
+ ValidationTask task = new ValidationTask(session.ctx, desc, endpoint, nowInSec, session.previewKind);
validationTasks.add(task);
return task;
}
diff --git a/src/java/org/apache/cassandra/repair/RepairJobDesc.java b/src/java/org/apache/cassandra/repair/RepairJobDesc.java
index c6b1898f79..dba336b5e7 100644
--- a/src/java/org/apache/cassandra/repair/RepairJobDesc.java
+++ b/src/java/org/apache/cassandra/repair/RepairJobDesc.java
@@ -21,10 +21,9 @@ import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Objects;
import java.util.UUID;
-import com.google.common.base.Objects;
-
import org.apache.commons.lang3.ArrayUtils;
import org.apache.cassandra.db.TypeSizes;
@@ -96,11 +95,11 @@ public class RepairJobDesc
RepairJobDesc that = (RepairJobDesc) o;
- if (!columnFamily.equals(that.columnFamily)) return false;
+ if (!Objects.equals(parentSessionId, that.parentSessionId)) return false;
+ if (!sessionId.equals(that.sessionId)) return false;
if (!keyspace.equals(that.keyspace)) return false;
+ if (!columnFamily.equals(that.columnFamily)) return false;
if (ranges != null ? that.ranges == null || (ranges.size() != that.ranges.size()) || (ranges.size() == that.ranges.size() && !ranges.containsAll(that.ranges)) : that.ranges != null) return false;
- if (!sessionId.equals(that.sessionId)) return false;
- if (parentSessionId != null ? !parentSessionId.equals(that.parentSessionId) : that.parentSessionId != null) return false;
return true;
}
@@ -108,7 +107,7 @@ public class RepairJobDesc
@Override
public int hashCode()
{
- return Objects.hashCode(sessionId, keyspace, columnFamily, ranges);
+ return Objects.hash(parentSessionId, sessionId, keyspace, columnFamily, ranges);
}
private static class RepairJobDescSerializer implements IVersionedSerializer<RepairJobDesc>
diff --git a/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java b/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java
index 58612f778a..988159860d 100644
--- a/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java
+++ b/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java
@@ -18,6 +18,8 @@
package org.apache.cassandra.repair;
import java.util.*;
+import java.util.function.BiFunction;
+import java.util.function.Function;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -26,17 +28,19 @@ import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.exceptions.RequestFailureReason;
import org.apache.cassandra.net.IVerbHandler;
import org.apache.cassandra.net.Message;
-import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.repair.messages.*;
+import org.apache.cassandra.repair.state.AbstractCompletable;
+import org.apache.cassandra.repair.state.AbstractState;
+import org.apache.cassandra.repair.state.Completable;
import org.apache.cassandra.repair.state.ParticipateState;
+import org.apache.cassandra.repair.state.SyncState;
import org.apache.cassandra.repair.state.ValidationState;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.TimeUUID;
-import static org.apache.cassandra.net.Verb.VALIDATION_RSP;
-
/**
* Handles all repair related message.
*
@@ -44,18 +48,38 @@ import static org.apache.cassandra.net.Verb.VALIDATION_RSP;
*/
public class RepairMessageVerbHandler implements IVerbHandler<RepairMessage>
{
- public static RepairMessageVerbHandler instance = new RepairMessageVerbHandler();
+ private static class Holder
+ {
+ private static final RepairMessageVerbHandler instance = new RepairMessageVerbHandler();
+ }
+
+ public static RepairMessageVerbHandler instance()
+ {
+ return Holder.instance;
+ }
+
+ private final SharedContext ctx;
+
+ private RepairMessageVerbHandler()
+ {
+ this(SharedContext.Global.instance);
+ }
+
+ public RepairMessageVerbHandler(SharedContext ctx)
+ {
+ this.ctx = ctx;
+ }
private static final Logger logger = LoggerFactory.getLogger(RepairMessageVerbHandler.class);
private boolean isIncremental(TimeUUID sessionID)
{
- return ActiveRepairService.instance.consistent.local.isSessionInProgress(sessionID);
+ return ctx.repair().consistent.local.isSessionInProgress(sessionID);
}
private PreviewKind previewKind(TimeUUID sessionID) throws NoSuchRepairSessionException
{
- ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(sessionID);
+ ActiveRepairService.ParentRepairSession prs = ctx.repair().getParentRepairSession(sessionID);
return prs != null ? prs.previewKind : PreviewKind.NONE;
}
@@ -71,16 +95,17 @@ public class RepairMessageVerbHandler implements IVerbHandler<RepairMessage>
{
PrepareMessage prepareMessage = (PrepareMessage) message.payload;
logger.debug("Preparing, {}", prepareMessage);
- ParticipateState state = new ParticipateState(message.from(), prepareMessage);
- if (!ActiveRepairService.instance.register(state))
+ ParticipateState state = new ParticipateState(ctx.clock(), message.from(), prepareMessage);
+ if (!ctx.repair().register(state))
{
- logger.debug("Duplicate prepare message found for {}", state.id);
+ replyDedup(ctx.repair().participate(state.id), message);
return;
}
- if (!ActiveRepairService.verifyCompactionsPendingThreshold(prepareMessage.parentRepairSession, prepareMessage.previewKind))
+ if (!ctx.repair().verifyCompactionsPendingThreshold(prepareMessage.parentRepairSession, prepareMessage.previewKind))
{
// error is logged in verifyCompactionsPendingThreshold
state.phase.fail("Too many pending compactions");
+
sendFailureResponse(message);
return;
}
@@ -99,22 +124,23 @@ public class RepairMessageVerbHandler implements IVerbHandler<RepairMessage>
}
columnFamilyStores.add(columnFamilyStore);
}
- ActiveRepairService.instance.registerParentRepairSession(prepareMessage.parentRepairSession,
- message.from(),
- columnFamilyStores,
- prepareMessage.ranges,
- prepareMessage.isIncremental,
- prepareMessage.repairedAt,
- prepareMessage.isGlobal,
- prepareMessage.previewKind);
- MessagingService.instance().send(message.emptyResponse(), message.from());
+ state.phase.accept();
+ ctx.repair().registerParentRepairSession(prepareMessage.parentRepairSession,
+ message.from(),
+ columnFamilyStores,
+ prepareMessage.ranges,
+ prepareMessage.isIncremental,
+ prepareMessage.repairedAt,
+ prepareMessage.isGlobal,
+ prepareMessage.previewKind);
+ sendAck(message);
}
break;
case SNAPSHOT_MSG:
{
logger.debug("Snapshotting {}", desc);
- ParticipateState state = ActiveRepairService.instance.participate(desc.parentSessionId);
+ ParticipateState state = ctx.repair().participate(desc.parentSessionId);
if (state == null)
{
logErrorAndSendFailureResponse("Unknown repair " + desc.parentSessionId, message);
@@ -130,56 +156,65 @@ public class RepairMessageVerbHandler implements IVerbHandler<RepairMessage>
return;
}
- ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(desc.parentSessionId);
- prs.setHasSnapshots();
- TableRepairManager repairManager = cfs.getRepairManager();
- if (prs.isGlobal)
+ ActiveRepairService.ParentRepairSession prs = ctx.repair().getParentRepairSession(desc.parentSessionId);
+ if (prs.setHasSnapshots())
{
- repairManager.snapshot(desc.parentSessionId.toString(), prs.getRanges(), false);
- }
- else
- {
- repairManager.snapshot(desc.parentSessionId.toString(), desc.ranges, true);
+ state.getOrCreateJob(desc).snapshot();
+ TableRepairManager repairManager = cfs.getRepairManager();
+ if (prs.isGlobal)
+ {
+ repairManager.snapshot(desc.parentSessionId.toString(), prs.getRanges(), false);
+ }
+ else
+ {
+ repairManager.snapshot(desc.parentSessionId.toString(), desc.ranges, true);
+ }
+ logger.debug("Enqueuing response to snapshot request {} to {}", desc.sessionId, message.from());
}
- logger.debug("Enqueuing response to snapshot request {} to {}", desc.sessionId, message.from());
- MessagingService.instance().send(message.emptyResponse(), message.from());
+ sendAck(message);
}
break;
case VALIDATION_REQ:
{
- // notify initiator that the message has been received, allowing this method to take as long as it needs to
- MessagingService.instance().send(message.emptyResponse(), message.from());
ValidationRequest validationRequest = (ValidationRequest) message.payload;
logger.debug("Validating {}", validationRequest);
- ParticipateState participate = ActiveRepairService.instance.participate(desc.parentSessionId);
+ ParticipateState participate = ctx.repair().participate(desc.parentSessionId);
if (participate == null)
{
logErrorAndSendFailureResponse("Unknown repair " + desc.parentSessionId, message);
return;
}
- ValidationState vState = new ValidationState(desc, message.from());
- if (!participate.register(vState))
- {
- logger.debug("Duplicate validation message found for parent={}, validation={}", participate.id, vState.id);
+ ValidationState vState = new ValidationState(ctx.clock(), desc, message.from());
+ if (!register(message, participate, vState,
+ participate::register,
+ (d, i) -> participate.validation(d)))
return;
- }
try
{
// trigger read-only compaction
ColumnFamilyStore store = ColumnFamilyStore.getIfExists(desc.keyspace, desc.columnFamily);
if (store == null)
{
- logger.error("Table {}.{} was dropped during validation phase of repair {}",
- desc.keyspace, desc.columnFamily, desc.parentSessionId);
- vState.phase.fail(String.format("Table %s.%s was dropped", desc.keyspace, desc.columnFamily));
- MessagingService.instance().send(Message.out(VALIDATION_RSP, new ValidationResponse(desc)), message.from());
+ String msg = String.format("Table %s.%s was dropped during validation phase of repair %s", desc.keyspace, desc.columnFamily, desc.parentSessionId);
+ vState.phase.fail(msg);
+ logErrorAndSendFailureResponse(msg, message);
return;
}
- ActiveRepairService.instance.consistent.local.maybeSetRepairing(desc.parentSessionId);
+ try
+ {
+ ctx.repair().consistent.local.maybeSetRepairing(desc.parentSessionId);
+ }
+ catch (Throwable t)
+ {
+ JVMStabilityInspector.inspectThrowable(t);
+ vState.phase.fail(t.toString());
+ logErrorAndSendFailureResponse(t.toString(), message);
+ return;
+ }
PreviewKind previewKind;
try
{
@@ -189,13 +224,15 @@ public class RepairMessageVerbHandler implements IVerbHandler<RepairMessage>
{
logger.warn("Parent repair session {} has been removed, failing repair", desc.parentSessionId);
vState.phase.fail(e);
- MessagingService.instance().send(Message.out(VALIDATION_RSP, new ValidationResponse(desc)), message.from());
+ sendFailureResponse(message);
return;
}
+ vState.phase.accept();
+ sendAck(message);
- Validator validator = new Validator(vState, validationRequest.nowInSec,
+ Validator validator = new Validator(ctx, vState, validationRequest.nowInSec,
isIncremental(desc.parentSessionId), previewKind);
- ValidationManager.instance.submitValidation(store, validator);
+ ctx.validationManager().submitValidation(store, validator);
}
catch (Throwable t)
{
@@ -207,12 +244,23 @@ public class RepairMessageVerbHandler implements IVerbHandler<RepairMessage>
case SYNC_REQ:
{
- // notify initiator that the message has been received, allowing this method to take as long as it needs to
- MessagingService.instance().send(message.emptyResponse(), message.from());
// forwarded sync request
SyncRequest request = (SyncRequest) message.payload;
logger.debug("Syncing {}", request);
- StreamingRepairTask task = new StreamingRepairTask(desc,
+
+ ParticipateState participate = ctx.repair().participate(desc.parentSessionId);
+ if (participate == null)
+ {
+ logErrorAndSendFailureResponse("Unknown repair " + desc.parentSessionId, message);
+ return;
+ }
+ SyncState state = new SyncState(ctx.clock(), desc, request.initiator, request.src, request.dst);
+ if (!register(message, participate, state,
+ participate::register,
+ participate::sync))
+ return;
+ state.phase.accept();
+ StreamingRepairTask task = new StreamingRepairTask(ctx, state, desc,
request.initiator,
request.src,
request.dst,
@@ -221,6 +269,7 @@ public class RepairMessageVerbHandler implements IVerbHandler<RepairMessage>
request.previewKind,
request.asymmetric);
task.run();
+ sendAck(message);
}
break;
@@ -228,50 +277,50 @@ public class RepairMessageVerbHandler implements IVerbHandler<RepairMessage>
{
logger.debug("cleaning up repair");
CleanupMessage cleanup = (CleanupMessage) message.payload;
- ParticipateState state = ActiveRepairService.instance.participate(cleanup.parentRepairSession);
+ ParticipateState state = ctx.repair().participate(cleanup.parentRepairSession);
if (state != null)
state.phase.success("Cleanup message recieved");
- ActiveRepairService.instance.removeParentRepairSession(cleanup.parentRepairSession);
- MessagingService.instance().send(message.emptyResponse(), message.from());
+ ctx.repair().removeParentRepairSession(cleanup.parentRepairSession);
+ sendAck(message);
}
break;
case PREPARE_CONSISTENT_REQ:
- ActiveRepairService.instance.consistent.local.handlePrepareMessage(message.from(), (PrepareConsistentRequest) message.payload);
+ ctx.repair().consistent.local.handlePrepareMessage(message.from(), (PrepareConsistentRequest) message.payload);
break;
case PREPARE_CONSISTENT_RSP:
- ActiveRepairService.instance.consistent.coordinated.handlePrepareResponse((PrepareConsistentResponse) message.payload);
+ ctx.repair().consistent.coordinated.handlePrepareResponse((PrepareConsistentResponse) message.payload);
break;
case FINALIZE_PROPOSE_MSG:
- ActiveRepairService.instance.consistent.local.handleFinalizeProposeMessage(message.from(), (FinalizePropose) message.payload);
+ ctx.repair().consistent.local.handleFinalizeProposeMessage(message.from(), (FinalizePropose) message.payload);
break;
case FINALIZE_PROMISE_MSG:
- ActiveRepairService.instance.consistent.coordinated.handleFinalizePromiseMessage((FinalizePromise) message.payload);
+ ctx.repair().consistent.coordinated.handleFinalizePromiseMessage((FinalizePromise) message.payload);
break;
case FINALIZE_COMMIT_MSG:
- ActiveRepairService.instance.consistent.local.handleFinalizeCommitMessage(message.from(), (FinalizeCommit) message.payload);
+ ctx.repair().consistent.local.handleFinalizeCommitMessage(message.from(), (FinalizeCommit) message.payload);
break;
case FAILED_SESSION_MSG:
FailSession failure = (FailSession) message.payload;
- ActiveRepairService.instance.consistent.coordinated.handleFailSessionMessage(failure);
- ActiveRepairService.instance.consistent.local.handleFailSessionMessage(message.from(), failure);
+ ctx.repair().consistent.coordinated.handleFailSessionMessage(failure);
+ ctx.repair().consistent.local.handleFailSessionMessage(message.from(), failure);
break;
case STATUS_REQ:
- ActiveRepairService.instance.consistent.local.handleStatusRequest(message.from(), (StatusRequest) message.payload);
+ ctx.repair().consistent.local.handleStatusRequest(message.from(), (StatusRequest) message.payload);
break;
case STATUS_RSP:
- ActiveRepairService.instance.consistent.local.handleStatusResponse(message.from(), (StatusResponse) message.payload);
+ ctx.repair().consistent.local.handleStatusResponse(message.from(), (StatusResponse) message.payload);
break;
default:
- ActiveRepairService.instance.handleMessage(message);
+ ctx.repair().handleMessage(message);
break;
}
}
@@ -280,15 +329,79 @@ public class RepairMessageVerbHandler implements IVerbHandler<RepairMessage>
logger.error("Got error, removing parent repair session");
if (desc != null && desc.parentSessionId != null)
{
- ParticipateState parcipate = ActiveRepairService.instance.participate(desc.parentSessionId);
+ ParticipateState parcipate = ctx.repair().participate(desc.parentSessionId);
if (parcipate != null)
parcipate.phase.fail(e);
- ActiveRepairService.instance.removeParentRepairSession(desc.parentSessionId);
+ ctx.repair().removeParentRepairSession(desc.parentSessionId);
}
throw new RuntimeException(e);
}
}
+ private <I, T extends AbstractState<?, I>> boolean register(Message<RepairMessage> message,
+ ParticipateState participate,
+ T vState,
+ Function<T, ParticipateState.RegisterStatus> register,
+ BiFunction<RepairJobDesc, I, T> getter)
+ {
+ ParticipateState.RegisterStatus registerStatus = register.apply(vState);
+ switch (registerStatus)
+ {
+ case ACCEPTED:
+ return true;
+ case EXISTS:
+ logger.debug("Duplicate validation message found for parent={}, validation={}", participate.id, vState.id);
+ replyDedup(getter.apply(message.payload.desc, vState.id), message);
+ return false;
+ case ALREADY_COMPLETED:
+ case STATUS_REJECTED:
+ // the repair is complete (most likely failed as we don't know success always), or is at a later phase such as sync
+ // so send a nack saying that the validation could not be accepted
+ sendFailureResponse(message);
+ return false;
+ default:
+ throw new IllegalStateException("Unexpected status: " + registerStatus);
+ }
+ }
+
+ private enum DedupResult { UNKNOWN, ACCEPT, REJECT }
+
+ private static DedupResult dedupResult(AbstractCompletable<?> state)
+ {
+ AbstractCompletable.Status status = state.getCompletionStatus();
+ switch (status)
+ {
+ case INIT:
+ return DedupResult.UNKNOWN;
+ case ACCEPTED:
+ return DedupResult.ACCEPT;
+ case COMPLETED:
+ return state.getResult().kind == Completable.Result.Kind.FAILURE ? DedupResult.REJECT: DedupResult.ACCEPT;
+ default:
+ throw new IllegalStateException("Unknown status: " + state);
+ }
+ }
+
+ private void replyDedup(AbstractCompletable<?> state, Message<RepairMessage> message)
+ {
+ if (state == null)
+ throw new IllegalStateException("State is null");
+ DedupResult result = dedupResult(state);
+ switch (result)
+ {
+ case ACCEPT:
+ sendAck(message);
+ break;
+ case REJECT:
+ sendFailureResponse(message);
+ break;
+ case UNKNOWN:
+ break;
+ default:
+ throw new IllegalStateException("Unknown result: " + result);
+ }
+ }
+
private void logErrorAndSendFailureResponse(String errorMessage, Message<?> respondTo)
{
logger.error(errorMessage);
@@ -298,6 +411,11 @@ public class RepairMessageVerbHandler implements IVerbHandler<RepairMessage>
private void sendFailureResponse(Message<?> respondTo)
{
Message<?> reply = respondTo.failureResponse(RequestFailureReason.UNKNOWN);
- MessagingService.instance().send(reply, respondTo.from());
+ ctx.messaging().send(reply, respondTo.from());
+ }
+
+ private void sendAck(Message<RepairMessage> message)
+ {
+ ctx.messaging().send(message.emptyResponse(), message.from());
}
}
diff --git a/src/java/org/apache/cassandra/repair/RepairSession.java b/src/java/org/apache/cassandra/repair/RepairSession.java
index 6fb455b47a..e0e104ff8e 100644
--- a/src/java/org/apache/cassandra/repair/RepairSession.java
+++ b/src/java/org/apache/cassandra/repair/RepairSession.java
@@ -27,20 +27,21 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.Executor;
+import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.atomic.AtomicBoolean;
+import javax.annotation.Nullable;
+
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
import com.google.common.util.concurrent.*;
-import org.apache.cassandra.concurrent.ExecutorFactory;
-import org.apache.cassandra.concurrent.ExecutorPlus;
-import org.apache.cassandra.utils.TimeUUID;
-import org.apache.cassandra.repair.state.SessionState;
-import org.apache.cassandra.utils.concurrent.AsyncFuture;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.concurrent.ExecutorPlus;
+import org.apache.cassandra.concurrent.Stage;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
@@ -49,19 +50,23 @@ import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.RepairException;
import org.apache.cassandra.gms.*;
import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.Message;
import org.apache.cassandra.repair.consistent.ConsistentSession;
import org.apache.cassandra.repair.consistent.LocalSession;
import org.apache.cassandra.repair.consistent.LocalSessions;
+import org.apache.cassandra.repair.messages.SyncResponse;
+import org.apache.cassandra.repair.messages.ValidationResponse;
+import org.apache.cassandra.repair.state.SessionState;
import org.apache.cassandra.schema.SystemDistributedKeyspace;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.streaming.PreviewKind;
-import org.apache.cassandra.streaming.SessionSummary;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.MerkleTrees;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.Throwables;
-import org.apache.cassandra.utils.concurrent.Future;
+import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.utils.concurrent.AsyncFuture;
/**
* Coordinates the (active) repair of a list of non overlapping token ranges.
@@ -125,8 +130,10 @@ public class RepairSession extends AsyncFuture<RepairSessionResult> implements I
private final ConcurrentMap<Pair<RepairJobDesc, SyncNodePair>, CompletableRemoteSyncTask> syncingTasks = new ConcurrentHashMap<>();
// Tasks(snapshot, validate request, differencing, ...) are run on taskExecutor
- public final ExecutorPlus taskExecutor;
+ public final SafeExecutor taskExecutor;
public final boolean optimiseStreams;
+ public final SharedContext ctx;
+ private volatile List<RepairJob> jobs = Collections.emptyList();
private volatile boolean terminated = false;
@@ -141,7 +148,8 @@ public class RepairSession extends AsyncFuture<RepairSessionResult> implements I
* @param paxosOnly true if we should only complete paxos operations, not run a normal repair
* @param cfnames names of columnfamilies
*/
- public RepairSession(TimeUUID parentRepairSession,
+ public RepairSession(SharedContext ctx,
+ TimeUUID parentRepairSession,
CommonRange commonRange,
String keyspace,
RepairParallelism parallelismDegree,
@@ -153,21 +161,23 @@ public class RepairSession extends AsyncFuture<RepairSessionResult> implements I
boolean paxosOnly,
String... cfnames)
{
+ this.ctx = ctx;
this.repairPaxos = repairPaxos;
this.paxosOnly = paxosOnly;
assert cfnames.length > 0 : "Repairing no column families seems pointless, doesn't it";
- this.state = new SessionState(parentRepairSession, keyspace, cfnames, commonRange);
+ this.state = new SessionState(ctx.clock(), parentRepairSession, keyspace, cfnames, commonRange);
this.parallelismDegree = parallelismDegree;
this.isIncremental = isIncremental;
this.previewKind = previewKind;
this.pullRepair = pullRepair;
this.optimiseStreams = optimiseStreams;
- this.taskExecutor = createExecutor();
+ this.taskExecutor = new SafeExecutor(createExecutor(ctx));
}
- protected ExecutorPlus createExecutor()
+ @VisibleForTesting
+ protected ExecutorPlus createExecutor(SharedContext ctx)
{
- return ExecutorFactory.Global.executorFactory().pooled("RepairJobTask", Integer.MAX_VALUE);
+ return ctx.executorFactory().pooled("RepairJobTask", Integer.MAX_VALUE);
}
public TimeUUID getId()
@@ -185,13 +195,20 @@ public class RepairSession extends AsyncFuture<RepairSessionResult> implements I
return state.commonRange.endpoints;
}
- public void trackValidationCompletion(Pair<RepairJobDesc, InetAddressAndPort> key, ValidationTask task)
+ public synchronized void trackValidationCompletion(Pair<RepairJobDesc, InetAddressAndPort> key, ValidationTask task)
{
+ if (terminated)
+ {
+ task.abort(new RuntimeException("Session terminated"));
+ return;
+ }
validating.put(key, task);
}
- public void trackSyncCompletion(Pair<RepairJobDesc, SyncNodePair> key, CompletableRemoteSyncTask task)
+ public synchronized void trackSyncCompletion(Pair<RepairJobDesc, SyncNodePair> key, CompletableRemoteSyncTask task)
{
+ if (terminated)
+ return;
syncingTasks.put(key, task);
}
@@ -199,26 +216,28 @@ public class RepairSession extends AsyncFuture<RepairSessionResult> implements I
* Receive merkle tree response or failed response from {@code endpoint} for current repair job.
*
* @param desc repair job description
- * @param endpoint endpoint that sent merkle tree
- * @param trees calculated merkle trees, or null if validation failed
+ * @param message containing the merkle trees or an error
*/
- public void validationComplete(RepairJobDesc desc, InetAddressAndPort endpoint, MerkleTrees trees)
+ public void validationComplete(RepairJobDesc desc, Message<ValidationResponse> message)
{
+ InetAddressAndPort endpoint = message.from();
+ MerkleTrees trees = message.payload.trees;
ValidationTask task = validating.remove(Pair.create(desc, endpoint));
+ // replies without a callback get dropped, so if in mixed mode this should be ignored
+ ctx.messaging().send(message.emptyResponse(), message.from());
if (task == null)
{
- assert terminated : "The repair session should be terminated if the validation we're completing no longer exists.";
-
// The trees may be off-heap, and will therefore need to be released.
if (trees != null)
trees.release();
-
+
+ // either the session completed so the validation is no longer needed, or this is a retry; in both cases there is nothing to do
return;
}
- String message = String.format("Received merkle tree for %s from %s", desc.columnFamily, endpoint);
- logger.info("{} {}", previewKind.logPrefix(getId()), message);
- Tracing.traceRepair(message);
+ String msg = String.format("Received merkle tree for %s from %s", desc.columnFamily, endpoint);
+ logger.info("{} {}", previewKind.logPrefix(getId()), msg);
+ Tracing.traceRepair(msg);
task.treesReceived(trees);
}
@@ -226,21 +245,20 @@ public class RepairSession extends AsyncFuture<RepairSessionResult> implements I
* Notify this session that sync completed/failed with given {@code SyncNodePair}.
*
* @param desc synced repair job
- * @param nodes nodes that completed sync
- * @param success true if sync succeeded
+ * @param message nodes that completed sync and if they were successful
*/
- public void syncComplete(RepairJobDesc desc, SyncNodePair nodes, boolean success, List<SessionSummary> summaries)
+ public void syncComplete(RepairJobDesc desc, Message<SyncResponse> message)
{
+ SyncNodePair nodes = message.payload.nodes;
CompletableRemoteSyncTask task = syncingTasks.remove(Pair.create(desc, nodes));
+ // replies without a callback get dropped, so if in mixed mode this should be ignored
+ ctx.messaging().send(message.emptyResponse(), message.from());
if (task == null)
- {
- assert terminated : "The repair session should be terminated if the sync task we're completing no longer exists.";
return;
- }
if (logger.isDebugEnabled())
logger.debug("{} Repair completed between {} and {} on {}", previewKind.logPrefix(getId()), nodes.coordinator, nodes.peer, desc.columnFamily);
- task.syncComplete(success, summaries);
+ task.syncComplete(message.payload.success, message.payload.summaries);
}
@VisibleForTesting
@@ -297,7 +315,7 @@ public class RepairSession extends AsyncFuture<RepairSessionResult> implements I
// Checking all nodes are live
for (InetAddressAndPort endpoint : state.commonRange.endpoints)
{
- if (!FailureDetector.instance.isAlive(endpoint) && !state.commonRange.hasSkippedReplicas)
+ if (!ctx.failureDetector().isAlive(endpoint) && !state.commonRange.hasSkippedReplicas)
{
message = String.format("Cannot proceed on repair because a neighbor (%s) is dead: session failed", endpoint);
state.phase.fail(message);
@@ -314,7 +332,7 @@ public class RepairSession extends AsyncFuture<RepairSessionResult> implements I
// Create and submit RepairJob for each ColumnFamily
state.phase.jobsSubmitted();
- List<Future<RepairResult>> jobs = new ArrayList<>(state.cfnames.length);
+ List<RepairJob> jobs = new ArrayList<>(state.cfnames.length);
for (String cfname : state.cfnames)
{
RepairJob job = new RepairJob(this, cfname);
@@ -322,6 +340,7 @@ public class RepairSession extends AsyncFuture<RepairSessionResult> implements I
executor.execute(job);
jobs.add(job);
}
+ this.jobs = jobs;
// When all RepairJobs are done without error, cleanup and set the final result
FBUtilities.allOf(jobs).addCallback(new FutureCallback<List<RepairResult>>()
@@ -334,10 +353,9 @@ public class RepairSession extends AsyncFuture<RepairSessionResult> implements I
Tracing.traceRepair("Completed sync of range {}", state.commonRange);
trySuccess(new RepairSessionResult(state.id, state.keyspace, state.commonRange.ranges, results, state.commonRange.hasSkippedReplicas));
- taskExecutor.shutdown();
// mark this session as terminated
- terminate();
- awaitTaskExecutorTermination();
+ terminate(null);
+ taskExecutor.shutdown();
}
public void onFailure(Throwable t)
@@ -351,12 +369,19 @@ public class RepairSession extends AsyncFuture<RepairSessionResult> implements I
Tracing.traceRepair("Session completed with the following error: {}", t);
forceShutdown(t);
}
- }, executor);
+ }, taskExecutor);
}
- public void terminate()
+ public synchronized void terminate(@Nullable Throwable reason)
{
terminated = true;
+ List<RepairJob> jobs = this.jobs;
+ if (jobs != null)
+ {
+ for (RepairJob job : jobs)
+ job.abort(reason);
+ }
+ this.jobs = null;
validating.clear();
syncingTasks.clear();
}
@@ -369,26 +394,9 @@ public class RepairSession extends AsyncFuture<RepairSessionResult> implements I
public void forceShutdown(Throwable reason)
{
tryFailure(reason);
+ terminate(reason);
taskExecutor.shutdown();
- terminate();
- awaitTaskExecutorTermination();
}
-
- private void awaitTaskExecutorTermination()
- {
- try
- {
- if (taskExecutor.awaitTermination(30, TimeUnit.SECONDS))
- logger.debug("{} session task executor shut down gracefully", previewKind.logPrefix(getId()));
- else
- logger.warn("{} session task executor unable to shut down gracefully", previewKind.logPrefix(getId()));
- }
- catch (InterruptedException e)
- {
- Thread.currentThread().interrupt();
- }
- }
-
public void onRemove(InetAddressAndPort endpoint)
{
convict(endpoint, Double.MAX_VALUE);
@@ -452,4 +460,33 @@ public class RepairSession extends AsyncFuture<RepairSessionResult> implements I
}
return false;
}
+
+ private static class SafeExecutor implements Executor
+ {
+ private final ExecutorPlus delegate;
+
+ private SafeExecutor(ExecutorPlus delegate)
+ {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public void execute(Runnable command)
+ {
+ try
+ {
+ delegate.execute(command);
+ }
+ catch (RejectedExecutionException e)
+ {
+ // task executor was shutdown, so fall back to a known good executor to finish callbacks
+ Stage.INTERNAL_RESPONSE.execute(command);
+ }
+ }
+
+ public void shutdown()
+ {
+ delegate.shutdown();
+ }
+ }
}
diff --git a/src/java/org/apache/cassandra/repair/SharedContext.java b/src/java/org/apache/cassandra/repair/SharedContext.java
new file mode 100644
index 0000000000..1dc0607cfc
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/SharedContext.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair;
+
+import java.util.Random;
+import java.util.function.Supplier;
+
+import org.apache.cassandra.concurrent.ExecutorFactory;
+import org.apache.cassandra.concurrent.ScheduledExecutorPlus;
+import org.apache.cassandra.concurrent.ScheduledExecutors;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.ICompactionManager;
+import org.apache.cassandra.gms.FailureDetector;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.gms.IFailureDetector;
+import org.apache.cassandra.gms.IGossiper;
+import org.apache.cassandra.locator.IEndpointSnitch;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.MessageDelivery;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.streaming.StreamPlan;
+import org.apache.cassandra.utils.Clock;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.MBeanWrapper;
+
+/**
+ * Access methods to shared resources and services.
+ * <p>
+ * In many parts of the code base we reach into the global space to pull out singletons, but this makes testing much harder; the main goals for this type is to make users easier to test.
+ *
+ * @see {@link Global#instance} for the main production path
+ */
+public interface SharedContext
+{
+ InetAddressAndPort broadcastAddressAndPort();
+ Supplier<Random> random();
+ Clock clock();
+ ExecutorFactory executorFactory();
+ MBeanWrapper mbean();
+ ScheduledExecutorPlus optionalTasks();
+
+ MessageDelivery messaging();
+ IFailureDetector failureDetector();
+ IEndpointSnitch snitch();
+ IGossiper gossiper();
+ ICompactionManager compactionManager();
+ ActiveRepairService repair();
+ IValidationManager validationManager();
+ TableRepairManager repairManager(ColumnFamilyStore store);
+ StreamExecutor streamExecutor();
+
+ class Global implements SharedContext
+ {
+ public static final Global instance = new Global();
+
+ @Override
+ public InetAddressAndPort broadcastAddressAndPort()
+ {
+ return FBUtilities.getBroadcastAddressAndPort();
+ }
+
+ @Override
+ public Supplier<Random> random()
+ {
+ return Random::new;
+ }
+
+ @Override
+ public Clock clock()
+ {
+ return Clock.Global.clock();
+ }
+
+ @Override
+ public ExecutorFactory executorFactory()
+ {
+ return ExecutorFactory.Global.executorFactory();
+ }
+
+ @Override
+ public MBeanWrapper mbean()
+ {
+ return MBeanWrapper.instance;
+ }
+
+ @Override
+ public ScheduledExecutorPlus optionalTasks()
+ {
+ return ScheduledExecutors.optionalTasks;
+ }
+
+ @Override
+ public MessageDelivery messaging()
+ {
+ return MessagingService.instance();
+ }
+
+ @Override
+ public IFailureDetector failureDetector()
+ {
+ return FailureDetector.instance;
+ }
+
+ @Override
+ public IEndpointSnitch snitch()
+ {
+ return DatabaseDescriptor.getEndpointSnitch();
+ }
+
+ @Override
+ public IGossiper gossiper()
+ {
+ return Gossiper.instance;
+ }
+
+ @Override
+ public ICompactionManager compactionManager()
+ {
+ return CompactionManager.instance;
+ }
+
+ @Override
+ public ActiveRepairService repair()
+ {
+ return ActiveRepairService.instance();
+ }
+
+ @Override
+ public IValidationManager validationManager()
+ {
+ return ValidationManager.instance;
+ }
+
+ @Override
+ public TableRepairManager repairManager(ColumnFamilyStore store)
+ {
+ return store.getRepairManager();
+ }
+
+ @Override
+ public StreamExecutor streamExecutor()
+ {
+ return StreamPlan::execute;
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/repair/SnapshotTask.java b/src/java/org/apache/cassandra/repair/SnapshotTask.java
index 7f5dcd7fae..ad45070cfb 100644
--- a/src/java/org/apache/cassandra/repair/SnapshotTask.java
+++ b/src/java/org/apache/cassandra/repair/SnapshotTask.java
@@ -23,11 +23,12 @@ import org.apache.cassandra.exceptions.RequestFailureReason;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.RequestCallback;
import org.apache.cassandra.net.Message;
-import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.repair.messages.RepairMessage;
import org.apache.cassandra.repair.messages.SnapshotMessage;
import org.apache.cassandra.utils.concurrent.AsyncFuture;
import static org.apache.cassandra.net.Verb.SNAPSHOT_MSG;
+import static org.apache.cassandra.repair.messages.RepairMessage.notDone;
/**
* SnapshotTask is a task that sends snapshot request.
@@ -36,18 +37,18 @@ public class SnapshotTask extends AsyncFuture<InetAddressAndPort> implements Run
{
private final RepairJobDesc desc;
private final InetAddressAndPort endpoint;
+ private final SharedContext ctx;
- SnapshotTask(RepairJobDesc desc, InetAddressAndPort endpoint)
+ SnapshotTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort endpoint)
{
+ this.ctx = ctx;
this.desc = desc;
this.endpoint = endpoint;
}
public void run()
{
- MessagingService.instance().sendWithCallback(Message.out(SNAPSHOT_MSG, new SnapshotMessage(desc)),
- endpoint,
- new SnapshotCallback(this));
+ RepairMessage.sendMessageWithRetries(ctx, notDone(this), new SnapshotMessage(desc), SNAPSHOT_MSG, endpoint, new SnapshotCallback(this));
}
/**
diff --git a/src/java/org/apache/cassandra/repair/StreamExecutor.java b/src/java/org/apache/cassandra/repair/StreamExecutor.java
new file mode 100644
index 0000000000..cb70d1ab75
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/StreamExecutor.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair;
+
+import org.apache.cassandra.streaming.StreamPlan;
+import org.apache.cassandra.streaming.StreamResultFuture;
+
+public interface StreamExecutor
+{
+ StreamResultFuture execute(StreamPlan plan);
+}
diff --git a/src/java/org/apache/cassandra/repair/StreamingRepairTask.java b/src/java/org/apache/cassandra/repair/StreamingRepairTask.java
index ea50a50068..0f84d66893 100644
--- a/src/java/org/apache/cassandra/repair/StreamingRepairTask.java
+++ b/src/java/org/apache/cassandra/repair/StreamingRepairTask.java
@@ -21,6 +21,7 @@ import java.util.Collections;
import java.util.Collection;
import com.google.common.annotations.VisibleForTesting;
+
import org.apache.cassandra.locator.RangesAtEndpoint;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -28,9 +29,9 @@ import org.slf4j.LoggerFactory;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.net.Message;
-import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.repair.messages.RepairMessage;
import org.apache.cassandra.repair.messages.SyncResponse;
+import org.apache.cassandra.repair.state.SyncState;
import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.streaming.StreamEvent;
import org.apache.cassandra.streaming.StreamEventHandler;
@@ -53,6 +54,8 @@ public class StreamingRepairTask implements Runnable, StreamEventHandler
{
private static final Logger logger = LoggerFactory.getLogger(StreamingRepairTask.class);
+ private final SharedContext ctx;
+ private final SyncState state;
private final RepairJobDesc desc;
private final boolean asymmetric;
private final InetAddressAndPort initiator;
@@ -62,8 +65,10 @@ public class StreamingRepairTask implements Runnable, StreamEventHandler
private final TimeUUID pendingRepair;
private final PreviewKind previewKind;
- public StreamingRepairTask(RepairJobDesc desc, InetAddressAndPort initiator, InetAddressAndPort src, InetAddressAndPort dst, Collection<Range<Token>> ranges, TimeUUID pendingRepair, PreviewKind previewKind, boolean asymmetric)
+ public StreamingRepairTask(SharedContext ctx, SyncState state, RepairJobDesc desc, InetAddressAndPort initiator, InetAddressAndPort src, InetAddressAndPort dst, Collection<Range<Token>> ranges, TimeUUID pendingRepair, PreviewKind previewKind, boolean asymmetric)
{
+ this.ctx = ctx;
+ this.state = state;
this.desc = desc;
this.initiator = initiator;
this.src = src;
@@ -80,12 +85,14 @@ public class StreamingRepairTask implements Runnable, StreamEventHandler
long start = approxTime.now();
StreamPlan streamPlan = createStreamPlan(dst);
logger.info("[streaming task #{}] Stream plan created in {}ms", desc.sessionId, MILLISECONDS.convert(approxTime.now() - start, NANOSECONDS));
- streamPlan.execute();
+ state.phase.start();
+ ctx.streamExecutor().execute(streamPlan);
}
@VisibleForTesting
StreamPlan createStreamPlan(InetAddressAndPort dest)
{
+ state.phase.planning();
StreamPlan sp = new StreamPlan(StreamOperation.REPAIR, 1, false, pendingRepair, previewKind)
.listeners(this)
.flushBeforeTransfer(pendingRepair == null) // sstables are isolated at the beginning of an incremental repair session, so flushing isn't neccessary
@@ -98,6 +105,7 @@ public class StreamingRepairTask implements Runnable, StreamEventHandler
return sp;
}
+ @Override
public void handleStreamEvent(StreamEvent event)
{
// Nothing to do here, all we care about is the final success or failure and that's handled by
@@ -107,17 +115,21 @@ public class StreamingRepairTask implements Runnable, StreamEventHandler
/**
* If we succeeded on both stream in and out, respond back to coordinator
*/
+ @Override
public void onSuccess(StreamState state)
{
logger.info("[repair #{}] streaming task succeed, returning response to {}", desc.sessionId, initiator);
- MessagingService.instance().send(Message.out(SYNC_RSP, new SyncResponse(desc, src, dst, true, state.createSummaries())), initiator);
+ this.state.phase.success();
+ RepairMessage.sendMessageWithRetries(ctx, new SyncResponse(desc, src, dst, true, state.createSummaries()), SYNC_RSP, initiator);
}
/**
* If we failed on either stream in or out, respond fail to coordinator
*/
+ @Override
public void onFailure(Throwable t)
{
- MessagingService.instance().send(Message.out(SYNC_RSP, new SyncResponse(desc, src, dst, false, Collections.emptyList())), initiator);
+ this.state.phase.fail(t);
+ RepairMessage.sendMessageWithRetries(ctx, new SyncResponse(desc, src, dst, false, Collections.emptyList()), SYNC_RSP, initiator);
}
}
diff --git a/src/java/org/apache/cassandra/repair/SymmetricRemoteSyncTask.java b/src/java/org/apache/cassandra/repair/SymmetricRemoteSyncTask.java
index 96f07f3fb8..95b8959407 100644
--- a/src/java/org/apache/cassandra/repair/SymmetricRemoteSyncTask.java
+++ b/src/java/org/apache/cassandra/repair/SymmetricRemoteSyncTask.java
@@ -31,7 +31,6 @@ import org.apache.cassandra.repair.messages.SyncRequest;
import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.streaming.SessionSummary;
import org.apache.cassandra.tracing.Tracing;
-import org.apache.cassandra.utils.FBUtilities;
/**
* SymmetricRemoteSyncTask sends {@link SyncRequest} to remote(non-coordinator) node
@@ -43,15 +42,15 @@ public class SymmetricRemoteSyncTask extends SyncTask implements CompletableRemo
{
private static final Logger logger = LoggerFactory.getLogger(SymmetricRemoteSyncTask.class);
- public SymmetricRemoteSyncTask(RepairJobDesc desc, InetAddressAndPort r1, InetAddressAndPort r2, List<Range<Token>> differences, PreviewKind previewKind)
+ public SymmetricRemoteSyncTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort r1, InetAddressAndPort r2, List<Range<Token>> differences, PreviewKind previewKind)
{
- super(desc, r1, r2, differences, previewKind);
+ super(ctx, desc, r1, r2, differences, previewKind);
}
@Override
protected void startSync()
{
- InetAddressAndPort local = FBUtilities.getBroadcastAddressAndPort();
+ InetAddressAndPort local = ctx.broadcastAddressAndPort();
SyncRequest request = new SyncRequest(desc, local, nodePair.coordinator, nodePair.peer, rangesToSync, previewKind, false);
Preconditions.checkArgument(nodePair.coordinator.equals(request.src));
String message = String.format("Forwarding streaming repair of %d ranges to %s (to be streamed with %s)", request.ranges.size(), request.src, request.dst);
diff --git a/src/java/org/apache/cassandra/repair/SyncTask.java b/src/java/org/apache/cassandra/repair/SyncTask.java
index 7393effd2d..a3b1a57493 100644
--- a/src/java/org/apache/cassandra/repair/SyncTask.java
+++ b/src/java/org/apache/cassandra/repair/SyncTask.java
@@ -37,13 +37,14 @@ import org.apache.cassandra.repair.messages.SyncRequest;
import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.tracing.Tracing;
-import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
import static org.apache.cassandra.net.Verb.SYNC_REQ;
+import static org.apache.cassandra.repair.messages.RepairMessage.notDone;
public abstract class SyncTask extends AsyncFuture<SyncStat> implements Runnable
{
private static final Logger logger = LoggerFactory.getLogger(SyncTask.class);
+ protected final SharedContext ctx;
protected final RepairJobDesc desc;
@VisibleForTesting
public final List<Range<Token>> rangesToSync;
@@ -53,9 +54,10 @@ public abstract class SyncTask extends AsyncFuture<SyncStat> implements Runnable
protected volatile long startTime = Long.MIN_VALUE;
protected final SyncStat stat;
- protected SyncTask(RepairJobDesc desc, InetAddressAndPort primaryEndpoint, InetAddressAndPort peer, List<Range<Token>> rangesToSync, PreviewKind previewKind)
+ protected SyncTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort primaryEndpoint, InetAddressAndPort peer, List<Range<Token>> rangesToSync, PreviewKind previewKind)
{
Preconditions.checkArgument(!peer.equals(primaryEndpoint), "Sending and receiving node are the same: %s", peer);
+ this.ctx = ctx;
this.desc = desc;
this.rangesToSync = rangesToSync;
this.nodePair = new SyncNodePair(primaryEndpoint, peer);
@@ -75,8 +77,7 @@ public abstract class SyncTask extends AsyncFuture<SyncStat> implements Runnable
*/
public final void run()
{
- startTime = currentTimeMillis();
-
+ startTime = ctx.clock().currentTimeMillis();
// choose a repair method based on the significance of the difference
String format = String.format("%s Endpoints %s and %s %%s for %s", previewKind.logPrefix(desc.sessionId), nodePair.coordinator, nodePair.peer, desc.columnFamily);
@@ -102,14 +103,17 @@ public abstract class SyncTask extends AsyncFuture<SyncStat> implements Runnable
protected void finished()
{
if (startTime != Long.MIN_VALUE)
- Keyspace.open(desc.keyspace).getColumnFamilyStore(desc.columnFamily).metric.repairSyncTime.update(currentTimeMillis() - startTime, TimeUnit.MILLISECONDS);
+ Keyspace.open(desc.keyspace).getColumnFamilyStore(desc.columnFamily).metric.repairSyncTime.update(ctx.clock().currentTimeMillis() - startTime, TimeUnit.MILLISECONDS);
}
- public void abort() {}
+ public void abort(Throwable reason)
+ {
+ tryFailure(reason);
+ }
void sendRequest(SyncRequest request, InetAddressAndPort to)
{
- RepairMessage.sendMessageWithFailureCB(request,
+ RepairMessage.sendMessageWithFailureCB(ctx, notDone(this), request,
SYNC_REQ,
to,
this::tryFailure);
diff --git a/src/java/org/apache/cassandra/repair/ValidationManager.java b/src/java/org/apache/cassandra/repair/ValidationManager.java
index a6c9fa9ef2..cc796c5f5b 100644
--- a/src/java/org/apache/cassandra/repair/ValidationManager.java
+++ b/src/java/org/apache/cassandra/repair/ValidationManager.java
@@ -37,13 +37,12 @@ import org.apache.cassandra.dht.Token;
import org.apache.cassandra.metrics.TableMetrics;
import org.apache.cassandra.metrics.TopPartitionTracker;
import org.apache.cassandra.repair.state.ValidationState;
+import org.apache.cassandra.utils.Clock;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.MerkleTree;
import org.apache.cassandra.utils.MerkleTrees;
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
-
-public class ValidationManager
+public class ValidationManager implements IValidationManager
{
private static final Logger logger = LoggerFactory.getLogger(ValidationManager.class);
@@ -99,8 +98,10 @@ public class ValidationManager
* but without writing the merge result
*/
@SuppressWarnings("resource")
- private void doValidation(ColumnFamilyStore cfs, Validator validator) throws IOException, NoSuchRepairSessionException
+ public static void doValidation(ColumnFamilyStore cfs, Validator validator) throws IOException, NoSuchRepairSessionException
{
+ SharedContext ctx = validator.ctx;
+ Clock clock = ctx.clock();
// this isn't meant to be race-proof, because it's not -- it won't cause bugs for a CFS to be dropped
// mid-validation, or to attempt to validate a droped CFS. this is just a best effort to avoid useless work,
// particularly in the scenario where a validation is submitted before the drop, and there are compactions
@@ -119,8 +120,8 @@ public class ValidationManager
// Create Merkle trees suitable to hold estimated partitions for the given ranges.
// We blindly assume that a partition is evenly distributed on all sstables for now.
- long start = nanoTime();
- try (ValidationPartitionIterator vi = getValidationIterator(cfs.getRepairManager(), validator, topPartitionCollector))
+ long start = clock.nanoTime();
+ try (ValidationPartitionIterator vi = getValidationIterator(ctx.repairManager(cfs), validator, topPartitionCollector))
{
state.phase.start(vi.estimatedPartitions(), vi.getEstimatedBytes());
MerkleTrees trees = createMerkleTrees(vi, validator.desc.ranges, cfs);
@@ -148,7 +149,7 @@ public class ValidationManager
}
if (logger.isDebugEnabled())
{
- long duration = TimeUnit.NANOSECONDS.toMillis(nanoTime() - start);
+ long duration = TimeUnit.NANOSECONDS.toMillis(clock.nanoTime() - start);
logger.debug("Validation of {} partitions (~{}) finished in {} msec, for {}",
state.partitionsProcessed,
FBUtilities.prettyPrintMemory(state.estimatedTotalBytes),
@@ -177,6 +178,7 @@ public class ValidationManager
/**
* Does not mutate data, so is not scheduled.
*/
+ @Override
public Future<?> submitValidation(ColumnFamilyStore cfs, Validator validator)
{
Callable<Object> validation = new Callable<Object>()
diff --git a/src/java/org/apache/cassandra/repair/ValidationTask.java b/src/java/org/apache/cassandra/repair/ValidationTask.java
index 003e736b80..322e07cd2d 100644
--- a/src/java/org/apache/cassandra/repair/ValidationTask.java
+++ b/src/java/org/apache/cassandra/repair/ValidationTask.java
@@ -28,6 +28,7 @@ import org.apache.cassandra.utils.MerkleTrees;
import org.apache.cassandra.utils.concurrent.AsyncFuture;
import static org.apache.cassandra.net.Verb.VALIDATION_REQ;
+import static org.apache.cassandra.repair.messages.RepairMessage.notDone;
/**
* ValidationTask sends {@link ValidationRequest} to a replica.
@@ -39,11 +40,11 @@ public class ValidationTask extends AsyncFuture<TreeResponse> implements Runnabl
private final InetAddressAndPort endpoint;
private final long nowInSec;
private final PreviewKind previewKind;
-
- private boolean active = true;
+ private final SharedContext ctx;
- public ValidationTask(RepairJobDesc desc, InetAddressAndPort endpoint, long nowInSec, PreviewKind previewKind)
+ public ValidationTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort endpoint, long nowInSec, PreviewKind previewKind)
{
+ this.ctx = ctx;
this.desc = desc;
this.endpoint = endpoint;
this.nowInSec = nowInSec;
@@ -55,7 +56,8 @@ public class ValidationTask extends AsyncFuture<TreeResponse> implements Runnabl
*/
public void run()
{
- RepairMessage.sendMessageWithFailureCB(new ValidationRequest(desc, nowInSec),
+ RepairMessage.sendMessageWithFailureCB(ctx, notDone(this),
+ new ValidationRequest(desc, nowInSec),
VALIDATION_REQ,
endpoint,
this::tryFailure);
@@ -70,18 +72,12 @@ public class ValidationTask extends AsyncFuture<TreeResponse> implements Runnabl
{
if (trees == null)
{
- active = false;
tryFailure(RepairException.warn(desc, previewKind, "Validation failed in " + endpoint));
}
- else if (active)
- {
- trySuccess(new TreeResponse(endpoint, trees));
- }
- else
+ else if (!trySuccess(new TreeResponse(endpoint, trees)))
{
- // If the task has already been aborted, just release the possibly off-heap trees and move along.
+ // If the task is done, just release the possibly off-heap trees and move along.
trees.release();
- trySuccess(null);
}
}
@@ -89,37 +85,32 @@ public class ValidationTask extends AsyncFuture<TreeResponse> implements Runnabl
* Release any trees already received by this task, and place it a state where any trees
* received subsequently will be properly discarded.
*/
- public synchronized void abort()
+ public synchronized void abort(Throwable reason)
{
- if (active)
+ if (!tryFailure(reason) && isSuccess())
{
- if (isDone())
+ try
+ {
+ // If we're done, this should return immediately.
+ TreeResponse response = get();
+
+ if (response.trees != null)
+ response.trees.release();
+ }
+ catch (InterruptedException e)
+ {
+ // Restore the interrupt.
+ Thread.currentThread().interrupt();
+ }
+ catch (ExecutionException e)
{
- try
- {
- // If we're done, this should return immediately.
- TreeResponse response = get();
-
- if (response.trees != null)
- response.trees.release();
- }
- catch (InterruptedException e)
- {
- // Restore the interrupt.
- Thread.currentThread().interrupt();
- }
- catch (ExecutionException e)
- {
- // Do nothing here. If an exception was set, there were no trees to release.
- }
+ // Do nothing here. If an exception was set, there were no trees to release.
}
-
- active = false;
}
}
public synchronized boolean isActive()
{
- return active;
+ return !isDone();
}
}
diff --git a/src/java/org/apache/cassandra/repair/Validator.java b/src/java/org/apache/cassandra/repair/Validator.java
index 4912fa93a2..d8ba929de6 100644
--- a/src/java/org/apache/cassandra/repair/Validator.java
+++ b/src/java/org/apache/cassandra/repair/Validator.java
@@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Random;
+import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -37,10 +38,10 @@ import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.metrics.TopPartitionTracker;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.repair.messages.RepairMessage;
import org.apache.cassandra.repair.messages.ValidationResponse;
import org.apache.cassandra.repair.state.ValidationState;
import org.apache.cassandra.streaming.PreviewKind;
-import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.MerkleTree;
@@ -66,6 +67,7 @@ public class Validator implements Runnable
public final long nowInSec;
private final boolean evenTreeDistribution;
public final boolean isIncremental;
+ public final SharedContext ctx;
// null when all rows with the min token have been consumed
private long validated;
@@ -83,16 +85,22 @@ public class Validator implements Runnable
public Validator(ValidationState state, long nowInSec, PreviewKind previewKind)
{
- this(state, nowInSec, false, false, previewKind);
+ this(SharedContext.Global.instance, state, nowInSec, false, false, previewKind);
+ }
+
+ public Validator(SharedContext ctx, ValidationState state, long nowInSec, boolean isIncremental, PreviewKind previewKind)
+ {
+ this(ctx, state, nowInSec, false, isIncremental, previewKind);
}
public Validator(ValidationState state, long nowInSec, boolean isIncremental, PreviewKind previewKind)
{
- this(state, nowInSec, false, isIncremental, previewKind);
+ this(SharedContext.Global.instance, state, nowInSec, false, isIncremental, previewKind);
}
- public Validator(ValidationState state, long nowInSec, boolean evenTreeDistribution, boolean isIncremental, PreviewKind previewKind)
+ public Validator(SharedContext ctx, ValidationState state, long nowInSec, boolean evenTreeDistribution, boolean isIncremental, PreviewKind previewKind)
{
+ this.ctx = ctx;
this.state = state;
this.desc = state.desc;
this.initiator = state.initiator;
@@ -118,7 +126,7 @@ public class Validator implements Runnable
else
{
List<DecoratedKey> keys = new ArrayList<>();
- Random random = new Random();
+ Random random = ctx.random().get();
for (Range<Token> range : trees.ranges())
{
@@ -269,11 +277,12 @@ public class Validator implements Runnable
return !FBUtilities.getBroadcastAddressAndPort().equals(initiator);
}
- private void respond(ValidationResponse response)
+ @VisibleForTesting
+ void respond(ValidationResponse response)
{
if (initiatorIsRemote())
{
- MessagingService.instance().send(Message.out(VALIDATION_RSP, response), initiator);
+ RepairMessage.sendMessageWithRetries(ctx, response, VALIDATION_RSP, initiator);
return;
}
@@ -294,7 +303,7 @@ public class Validator implements Runnable
{
logger.error("Failed to move local merkle tree for {} off heap", desc, e);
}
- ActiveRepairService.instance.handleMessage(Message.out(VALIDATION_RSP, movedResponse));
+ ctx.repair().handleMessage(Message.out(VALIDATION_RSP, movedResponse));
});
}
}
diff --git a/src/java/org/apache/cassandra/repair/consistent/ConsistentSession.java b/src/java/org/apache/cassandra/repair/consistent/ConsistentSession.java
index 86ecfb76f0..76645f1adc 100644
--- a/src/java/org/apache/cassandra/repair/consistent/ConsistentSession.java
+++ b/src/java/org/apache/cassandra/repair/consistent/ConsistentSession.java
@@ -44,6 +44,7 @@ import org.apache.cassandra.repair.messages.RepairOption;
import org.apache.cassandra.repair.messages.StatusRequest;
import org.apache.cassandra.repair.messages.StatusResponse;
import org.apache.cassandra.repair.messages.ValidationRequest;
+import org.apache.cassandra.repair.SharedContext;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.tools.nodetool.RepairAdmin;
@@ -187,6 +188,7 @@ public abstract class ConsistentSession
}
private volatile State state;
+ public final SharedContext ctx;
public final TimeUUID sessionID;
public final InetAddressAndPort coordinator;
public final ImmutableSet<TableId> tableIds;
@@ -197,6 +199,7 @@ public abstract class ConsistentSession
ConsistentSession(AbstractBuilder builder)
{
builder.validate();
+ this.ctx = builder.ctx;
this.state = builder.state;
this.sessionID = builder.sessionID;
this.coordinator = builder.coordinator;
@@ -264,6 +267,7 @@ public abstract class ConsistentSession
abstract static class AbstractBuilder
{
+ private final SharedContext ctx;
private State state;
private TimeUUID sessionID;
private InetAddressAndPort coordinator;
@@ -272,6 +276,11 @@ public abstract class ConsistentSession
private Collection<Range<Token>> ranges;
private Set<InetAddressAndPort> participants;
+ protected AbstractBuilder(SharedContext ctx)
+ {
+ this.ctx = ctx;
+ }
+
void withState(State state)
{
this.state = state;
diff --git a/src/java/org/apache/cassandra/repair/consistent/CoordinatorSession.java b/src/java/org/apache/cassandra/repair/consistent/CoordinatorSession.java
index 88f30029c3..0ca1576df1 100644
--- a/src/java/org/apache/cassandra/repair/consistent/CoordinatorSession.java
+++ b/src/java/org/apache/cassandra/repair/consistent/CoordinatorSession.java
@@ -29,6 +29,7 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.Iterables;
import org.apache.cassandra.concurrent.ImmediateExecutor;
+import org.apache.cassandra.repair.SharedContext;
import org.apache.cassandra.repair.CoordinatedRepairResult;
import org.apache.cassandra.utils.concurrent.AsyncPromise;
import org.apache.cassandra.utils.concurrent.Future;
@@ -40,7 +41,6 @@ import org.slf4j.LoggerFactory;
import org.apache.cassandra.exceptions.RepairException;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.Message;
-import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.net.Verb;
import org.apache.cassandra.repair.SomeRepairFailedException;
import org.apache.cassandra.repair.messages.FailSession;
@@ -51,8 +51,6 @@ import org.apache.cassandra.repair.messages.RepairMessage;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.utils.concurrent.ImmediateFuture;
-import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
-
/**
* Coordinator side logic and state of a consistent repair session. Like {@link ActiveRepairService.ParentRepairSession},
* there is only one {@code CoordinatorSession} per user repair command, regardless of the number of tables and token
@@ -62,6 +60,7 @@ public class CoordinatorSession extends ConsistentSession
{
private static final Logger logger = LoggerFactory.getLogger(CoordinatorSession.class);
+ private final SharedContext ctx;
private final Map<InetAddressAndPort, State> participantStates = new HashMap<>();
private final AsyncPromise<Void> prepareFuture = AsyncPromise.uncancellable();
private final AsyncPromise<Void> finalizeProposeFuture = AsyncPromise.uncancellable();
@@ -73,6 +72,7 @@ public class CoordinatorSession extends ConsistentSession
public CoordinatorSession(Builder builder)
{
super(builder);
+ ctx = builder.ctx == null ? SharedContext.Global.instance : builder.ctx;
for (InetAddressAndPort participant : participants)
{
participantStates.put(participant, State.PREPARING);
@@ -81,6 +81,19 @@ public class CoordinatorSession extends ConsistentSession
public static class Builder extends AbstractBuilder
{
+ private SharedContext ctx;
+
+ public Builder(SharedContext ctx)
+ {
+ super(ctx);
+ }
+
+ public Builder withContext(SharedContext ctx)
+ {
+ this.ctx = ctx;
+ return this;
+ }
+
public CoordinatorSession build()
{
validate();
@@ -88,9 +101,9 @@ public class CoordinatorSession extends ConsistentSession
}
}
- public static Builder builder()
+ public static Builder builder(SharedContext ctx)
{
- return new Builder();
+ return new Builder(ctx);
}
public void setState(State state)
@@ -144,7 +157,8 @@ public class CoordinatorSession extends ConsistentSession
protected void sendMessage(InetAddressAndPort destination, Message<RepairMessage> message)
{
logger.trace("Sending {} to {}", message.payload, destination);
- MessagingService.instance().send(message, destination);
+
+ ctx.messaging().send(message, destination);
}
public Future<Void> prepare()
@@ -297,12 +311,12 @@ public class CoordinatorSession extends ConsistentSession
{
logger.info("Beginning coordination of incremental repair session {}", sessionID);
- sessionStart = currentTimeMillis();
+ sessionStart = ctx.clock().currentTimeMillis();
Future<Void> prepareResult = prepare();
// run repair sessions normally
Future<CoordinatedRepairResult> repairSessionResults = prepareResult.flatMap(ignore -> {
- repairStart = currentTimeMillis();
+ repairStart = ctx.clock().currentTimeMillis();
if (logger.isDebugEnabled())
logger.debug("Incremental repair {} prepare phase completed in {}", sessionID, formatDuration(sessionStart, repairStart));
setRepairing();
@@ -311,7 +325,7 @@ public class CoordinatorSession extends ConsistentSession
// if any session failed, then fail the future
Future<CoordinatedRepairResult> onlySuccessSessionResults = repairSessionResults.flatMap(result -> {
- finalizeStart = currentTimeMillis();
+ finalizeStart = ctx.clock().currentTimeMillis();
if (result.hasFailed())
{
if (logger.isDebugEnabled())
@@ -324,10 +338,10 @@ public class CoordinatorSession extends ConsistentSession
// mark propose finalization and commit
Future<CoordinatedRepairResult> proposeFuture = onlySuccessSessionResults.flatMap(results -> finalizePropose().map(ignore -> {
if (logger.isDebugEnabled())
- logger.debug("Incremental repair {} finalization phase completed in {}", sessionID, formatDuration(finalizeStart, currentTimeMillis()));
+ logger.debug("Incremental repair {} finalization phase completed in {}", sessionID, formatDuration(finalizeStart, ctx.clock().currentTimeMillis()));
finalizeCommit();
if (logger.isDebugEnabled())
- logger.debug("Incremental repair {} phase completed in {}", sessionID, formatDuration(sessionStart, currentTimeMillis()));
+ logger.debug("Incremental repair {} phase completed in {}", sessionID, formatDuration(sessionStart, ctx.clock().currentTimeMillis()));
return results;
}));
@@ -335,7 +349,7 @@ public class CoordinatorSession extends ConsistentSession
if (failure != null)
{
if (logger.isDebugEnabled())
- logger.debug("Incremental repair {} phase failed in {}", sessionID, formatDuration(sessionStart, currentTimeMillis()));
+ logger.debug("Incremental repair {} phase failed in {}", sessionID, formatDuration(sessionStart, ctx.clock().currentTimeMillis()));
fail();
}
}, ImmediateExecutor.INSTANCE);
diff --git a/src/java/org/apache/cassandra/repair/consistent/CoordinatorSessions.java b/src/java/org/apache/cassandra/repair/consistent/CoordinatorSessions.java
index 328c16e645..1abab8c476 100644
--- a/src/java/org/apache/cassandra/repair/consistent/CoordinatorSessions.java
+++ b/src/java/org/apache/cassandra/repair/consistent/CoordinatorSessions.java
@@ -24,6 +24,7 @@ import java.util.Set;
import com.google.common.base.Preconditions;
+import org.apache.cassandra.repair.SharedContext;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.repair.messages.FailSession;
import org.apache.cassandra.repair.messages.FinalizePromise;
@@ -37,8 +38,15 @@ import org.apache.cassandra.utils.TimeUUID;
*/
public class CoordinatorSessions
{
+ private final SharedContext ctx;
+
private final Map<TimeUUID, CoordinatorSession> sessions = new HashMap<>();
+ public CoordinatorSessions(SharedContext ctx)
+ {
+ this.ctx = ctx;
+ }
+
protected CoordinatorSession buildSession(CoordinatorSession.Builder builder)
{
return new CoordinatorSession(builder);
@@ -46,14 +54,14 @@ public class CoordinatorSessions
public synchronized CoordinatorSession registerSession(TimeUUID sessionId, Set<InetAddressAndPort> participants, boolean isForced) throws NoSuchRepairSessionException
{
- ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(sessionId);
+ ActiveRepairService.ParentRepairSession prs = ctx.repair().getParentRepairSession(sessionId);
Preconditions.checkArgument(!sessions.containsKey(sessionId),
"A coordinator already exists for session %s", sessionId);
Preconditions.checkArgument(!isForced || prs.repairedAt == ActiveRepairService.UNREPAIRED_SSTABLE,
"cannot promote data for forced incremental repairs");
- CoordinatorSession.Builder builder = CoordinatorSession.builder();
+ CoordinatorSession.Builder builder = CoordinatorSession.builder(ctx);
builder.withState(ConsistentSession.State.PREPARING);
builder.withSessionID(sessionId);
builder.withCoordinator(prs.coordinator);
@@ -62,6 +70,7 @@ public class CoordinatorSessions
builder.withRepairedAt(prs.repairedAt);
builder.withRanges(prs.getRanges());
builder.withParticipants(participants);
+ builder.withContext(ctx);
CoordinatorSession session = buildSession(builder);
sessions.put(session.sessionID, session);
return session;
diff --git a/src/java/org/apache/cassandra/repair/consistent/LocalSession.java b/src/java/org/apache/cassandra/repair/consistent/LocalSession.java
index a0d1ac05c8..06c6755562 100644
--- a/src/java/org/apache/cassandra/repair/consistent/LocalSession.java
+++ b/src/java/org/apache/cassandra/repair/consistent/LocalSession.java
@@ -22,7 +22,7 @@ import java.util.Objects;
import com.google.common.base.Preconditions;
-import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.repair.SharedContext;
/**
* Basically just a record of a local session. All of the local session logic is implemented in {@link LocalSessions}
@@ -57,7 +57,7 @@ public class LocalSession extends ConsistentSession
public void setLastUpdate()
{
- lastUpdate = FBUtilities.nowInSeconds();
+ lastUpdate = ctx.clock().nowInSeconds();
}
public boolean equals(Object o)
@@ -97,6 +97,11 @@ public class LocalSession extends ConsistentSession
private long startedAt;
private long lastUpdate;
+ public Builder(SharedContext ctx)
+ {
+ super(ctx);
+ }
+
public Builder withStartedAt(long startedAt)
{
this.startedAt = startedAt;
@@ -123,8 +128,8 @@ public class LocalSession extends ConsistentSession
}
}
- public static Builder builder()
+ public static Builder builder(SharedContext ctx)
{
- return new Builder();
+ return new Builder(ctx);
}
}
diff --git a/src/java/org/apache/cassandra/repair/consistent/LocalSessions.java b/src/java/org/apache/cassandra/repair/consistent/LocalSessions.java
index 31df2e69bb..85bdf15923 100644
--- a/src/java/org/apache/cassandra/repair/consistent/LocalSessions.java
+++ b/src/java/org/apache/cassandra/repair/consistent/LocalSessions.java
@@ -74,11 +74,9 @@ import org.apache.cassandra.db.marshal.UUIDType;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.gms.FailureDetector;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.net.Message;
-import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.repair.messages.FailSession;
import org.apache.cassandra.repair.messages.FinalizeCommit;
import org.apache.cassandra.repair.messages.FinalizePromise;
@@ -88,16 +86,15 @@ import org.apache.cassandra.repair.messages.PrepareConsistentResponse;
import org.apache.cassandra.repair.messages.RepairMessage;
import org.apache.cassandra.repair.messages.StatusRequest;
import org.apache.cassandra.repair.messages.StatusResponse;
+import org.apache.cassandra.repair.SharedContext;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.repair.NoSuchRepairSessionException;
import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Throwables;
import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.utils.concurrent.Future;
-import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
import static org.apache.cassandra.config.CassandraRelevantProperties.REPAIR_CLEANUP_INTERVAL_SECONDS;
import static org.apache.cassandra.config.CassandraRelevantProperties.REPAIR_DELETE_TIMEOUT_SECONDS;
import static org.apache.cassandra.config.CassandraRelevantProperties.REPAIR_FAIL_TIMEOUT_SECONDS;
@@ -154,10 +151,16 @@ public class LocalSessions
private final String keyspace = SchemaConstants.SYSTEM_KEYSPACE_NAME;
private final String table = SystemKeyspace.REPAIRS;
+ private final SharedContext ctx;
private boolean started = false;
private volatile ImmutableMap<TimeUUID, LocalSession> sessions = ImmutableMap.of();
private volatile ImmutableMap<TableId, RepairedState> repairedStates = ImmutableMap.of();
+ public LocalSessions(SharedContext ctx)
+ {
+ this.ctx = ctx;
+ }
+
@VisibleForTesting
int getNumSessions()
{
@@ -167,13 +170,13 @@ public class LocalSessions
@VisibleForTesting
protected InetAddressAndPort getBroadcastAddressAndPort()
{
- return FBUtilities.getBroadcastAddressAndPort();
+ return ctx.broadcastAddressAndPort();
}
@VisibleForTesting
protected boolean isAlive(InetAddressAndPort address)
{
- return FailureDetector.instance.isAlive(address);
+ return ctx.failureDetector().isAlive(address);
}
@VisibleForTesting
@@ -443,7 +446,7 @@ public class LocalSessions
{
synchronized (session)
{
- long now = FBUtilities.nowInSeconds();
+ long now = ctx.clock().nowInSeconds();
if (shouldFail(session, now))
{
logger.warn("Auto failing timed out repair session {}", session);
@@ -564,7 +567,7 @@ public class LocalSessions
private LocalSession load(UntypedResultSet.Row row)
{
- LocalSession.Builder builder = LocalSession.builder();
+ LocalSession.Builder builder = LocalSession.builder(ctx);
builder.withState(ConsistentSession.State.valueOf(row.getInt("state")));
builder.withSessionID(row.getTimeUUID("parent_id"));
InetAddressAndPort coordinator = InetAddressAndPort.getByAddressOverrideDefaults(
@@ -662,7 +665,7 @@ public class LocalSessions
@VisibleForTesting
LocalSession createSessionUnsafe(TimeUUID sessionId, ActiveRepairService.ParentRepairSession prs, Set<InetAddressAndPort> peers)
{
- LocalSession.Builder builder = LocalSession.builder();
+ LocalSession.Builder builder = LocalSession.builder(ctx);
builder.withState(ConsistentSession.State.PREPARING);
builder.withSessionID(sessionId);
builder.withCoordinator(prs.coordinator);
@@ -672,7 +675,7 @@ public class LocalSessions
builder.withRanges(prs.getRanges());
builder.withParticipants(peers);
- long now = FBUtilities.nowInSeconds();
+ long now = ctx.clock().nowInSeconds();
builder.withStartedAt(now);
builder.withLastUpdate(now);
@@ -681,13 +684,14 @@ public class LocalSessions
protected ActiveRepairService.ParentRepairSession getParentRepairSession(TimeUUID sessionID) throws NoSuchRepairSessionException
{
- return ActiveRepairService.instance.getParentRepairSession(sessionID);
+
+ return ctx.repair().getParentRepairSession(sessionID);
}
protected void sendMessage(InetAddressAndPort destination, Message<? extends RepairMessage> message)
{
logger.trace("sending {} to {}", message.payload, destination);
- MessagingService.instance().send(message, destination);
+ ctx.messaging().send(message, destination);
}
@VisibleForTesting
@@ -821,7 +825,7 @@ public class LocalSessions
putSessionUnsafe(session);
logger.info("Beginning local incremental repair session {}", session);
- ExecutorService executor = executorFactory().pooled("Repair-" + sessionID, parentSession.getColumnFamilyStores().size());
+ ExecutorService executor = ctx.executorFactory().pooled("Repair-" + sessionID, parentSession.getColumnFamilyStores().size());
KeyspaceRepairManager repairManager = parentSession.getKeyspace().getRepairManager();
RangesAtEndpoint tokenRanges = filterLocalRanges(parentSession.getKeyspace().getName(), parentSession.getRanges());
@@ -1099,6 +1103,12 @@ public class LocalSessions
listeners.remove(listener);
}
+ @VisibleForTesting
+ public static void unsafeClearListeners()
+ {
+ listeners.clear();
+ }
+
public interface Listener
{
void onIRStateChange(LocalSession session);
diff --git a/src/java/org/apache/cassandra/repair/messages/CleanupMessage.java b/src/java/org/apache/cassandra/repair/messages/CleanupMessage.java
index 0e1a74f3d7..add59060a7 100644
--- a/src/java/org/apache/cassandra/repair/messages/CleanupMessage.java
+++ b/src/java/org/apache/cassandra/repair/messages/CleanupMessage.java
@@ -40,6 +40,12 @@ public class CleanupMessage extends RepairMessage
this.parentRepairSession = parentRepairSession;
}
+ @Override
+ public TimeUUID parentRepairSession()
+ {
+ return parentRepairSession;
+ }
+
@Override
public boolean equals(Object o)
{
diff --git a/src/java/org/apache/cassandra/repair/messages/PrepareMessage.java b/src/java/org/apache/cassandra/repair/messages/PrepareMessage.java
index 7064227e11..bf76190fc3 100644
--- a/src/java/org/apache/cassandra/repair/messages/PrepareMessage.java
+++ b/src/java/org/apache/cassandra/repair/messages/PrepareMessage.java
@@ -61,6 +61,12 @@ public class PrepareMessage extends RepairMessage
this.previewKind = previewKind;
}
+ @Override
+ public TimeUUID parentRepairSession()
+ {
+ return parentRepairSession;
+ }
+
@Override
public boolean equals(Object o)
{
diff --git a/src/java/org/apache/cassandra/repair/messages/RepairMessage.java b/src/java/org/apache/cassandra/repair/messages/RepairMessage.java
index 00ce888696..d6d4d9b536 100644
--- a/src/java/org/apache/cassandra/repair/messages/RepairMessage.java
+++ b/src/java/org/apache/cassandra/repair/messages/RepairMessage.java
@@ -17,21 +17,34 @@
*/
package org.apache.cassandra.repair.messages;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.EnumSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Supplier;
+
+import javax.annotation.Nullable;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.RepairRetrySpec;
+import org.apache.cassandra.config.RetrySpec;
+import org.apache.cassandra.repair.SharedContext;
import org.apache.cassandra.exceptions.RepairException;
import org.apache.cassandra.exceptions.RequestFailureReason;
-import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.Message;
-import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.net.RequestCallback;
import org.apache.cassandra.net.Verb;
import org.apache.cassandra.repair.RepairJobDesc;
import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.utils.Backoff;
import org.apache.cassandra.utils.CassandraVersion;
import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.utils.concurrent.Future;
import static org.apache.cassandra.net.MessageFlag.CALL_BACK_ON_FAILURE;
@@ -42,29 +55,121 @@ import static org.apache.cassandra.net.MessageFlag.CALL_BACK_ON_FAILURE;
*/
public abstract class RepairMessage
{
- private static final CassandraVersion SUPPORTS_TIMEOUTS = new CassandraVersion("4.0.7-SNAPSHOT");
+ private enum ErrorHandling { NONE, TIMEOUT, RETRY }
+ private static final CassandraVersion SUPPORTS_RETRY = new CassandraVersion("5.0.0-alpha2.SNAPSHOT");
+ private static final Map<Verb, CassandraVersion> VERB_TIMEOUT_VERSIONS;
+
+ static
+ {
+ CassandraVersion timeoutVersion = new CassandraVersion("4.0.7-SNAPSHOT");
+ EnumMap<Verb, CassandraVersion> map = new EnumMap<>(Verb.class);
+ map.put(Verb.VALIDATION_REQ, timeoutVersion);
+ map.put(Verb.SYNC_REQ, timeoutVersion);
+ map.put(Verb.VALIDATION_RSP, SUPPORTS_RETRY);
+ map.put(Verb.SYNC_RSP, SUPPORTS_RETRY);
+ VERB_TIMEOUT_VERSIONS = Collections.unmodifiableMap(map);
+ }
+ private static final Set<Verb> SUPPORTS_RETRY_WITHOUT_VERSION_CHECK = Collections.unmodifiableSet(EnumSet.of(Verb.CLEANUP_MSG));
+
private static final Logger logger = LoggerFactory.getLogger(RepairMessage.class);
+ @Nullable
public final RepairJobDesc desc;
- protected RepairMessage(RepairJobDesc desc)
+ protected RepairMessage(@Nullable RepairJobDesc desc)
{
this.desc = desc;
}
+ public TimeUUID parentRepairSession()
+ {
+ return desc.parentSessionId;
+ }
+
public interface RepairFailureCallback
{
void onFailure(Exception e);
}
- public static void sendMessageWithFailureCB(RepairMessage request, Verb verb, InetAddressAndPort endpoint, RepairFailureCallback failureCallback)
+ private static Backoff backoff(SharedContext ctx, Verb verb)
+ {
+ RepairRetrySpec retrySpec = DatabaseDescriptor.getRepairRetrySpec();
+ RetrySpec spec = verb == Verb.VALIDATION_RSP ? retrySpec.getMerkleTreeResponseSpec() : retrySpec;
+ if (!spec.isEnabled())
+ return Backoff.None.INSTANCE;
+ return new Backoff.ExponentialBackoff(spec.maxAttempts.value, spec.baseSleepTime.toMilliseconds(), spec.maxSleepTime.toMilliseconds(), ctx.random().get()::nextDouble);
+ }
+
+ public static Supplier<Boolean> notDone(Future<?> f)
{
- RequestCallback<?> callback = new RequestCallback<Object>()
+ return () -> !f.isDone();
+ }
+
+ private static Supplier<Boolean> always()
+ {
+ return () -> true;
+ }
+
+ public static <T> void sendMessageWithRetries(SharedContext ctx, Supplier<Boolean> allowRetry, RepairMessage request, Verb verb, InetAddressAndPort endpoint, RequestCallback<T> finalCallback)
+ {
+ sendMessageWithRetries(ctx, backoff(ctx, verb), allowRetry, request, verb, endpoint, finalCallback, 0);
+ }
+
+ public static <T> void sendMessageWithRetries(SharedContext ctx, RepairMessage request, Verb verb, InetAddressAndPort endpoint, RequestCallback<T> finalCallback)
+ {
+ sendMessageWithRetries(ctx, backoff(ctx, verb), always(), request, verb, endpoint, finalCallback, 0);
+ }
+
+ public static void sendMessageWithRetries(SharedContext ctx, RepairMessage request, Verb verb, InetAddressAndPort endpoint)
+ {
+ sendMessageWithRetries(ctx, backoff(ctx, verb), always(), request, verb, endpoint, new RequestCallback<>()
{
@Override
public void onResponse(Message<Object> msg)
{
- logger.info("[#{}] {} received by {}", request.desc.parentSessionId, verb, endpoint);
- // todo: at some point we should make repair messages follow the normal path, actually using this
+ }
+
+ @Override
+ public void onFailure(InetAddressAndPort from, RequestFailureReason failureReason)
+ {
+ }
+ }, 0);
+ }
+
+ private static <T> void sendMessageWithRetries(SharedContext ctx, Backoff backoff, Supplier<Boolean> allowRetry, RepairMessage request, Verb verb, InetAddressAndPort endpoint, RequestCallback<T> finalCallback, int attempt)
+ {
+ RequestCallback<T> callback = new RequestCallback<>()
+ {
+ @Override
+ public void onResponse(Message<T> msg)
+ {
+ finalCallback.onResponse(msg);
+ }
+
+ @Override
+ public void onFailure(InetAddressAndPort from, RequestFailureReason failureReason)
+ {
+ ErrorHandling allowed = errorHandlingSupported(ctx, endpoint, verb, request.parentRepairSession());
+ switch (allowed)
+ {
+ case NONE:
+ logger.error("[#{}] {} failed on {}: {}", request.parentRepairSession(), verb, from, failureReason);
+ return;
+ case TIMEOUT:
+ finalCallback.onFailure(from, failureReason);
+ return;
+ case RETRY:
+ int maxAttempts = backoff.maxAttempts();
+ if (failureReason == RequestFailureReason.TIMEOUT && attempt < maxAttempts && allowRetry.get())
+ {
+ ctx.optionalTasks().schedule(() -> sendMessageWithRetries(ctx, backoff, allowRetry, request, verb, endpoint, finalCallback, attempt + 1),
+ backoff.computeWaitTime(attempt), backoff.unit());
+ return;
+ }
+ finalCallback.onFailure(from, failureReason);
+ return;
+ default:
+ throw new AssertionError("Unknown error handler: " + allowed);
+ }
}
@Override
@@ -72,27 +177,59 @@ public abstract class RepairMessage
{
return true;
}
+ };
+ ctx.messaging().sendWithCallback(Message.outWithFlag(verb, request, CALL_BACK_ON_FAILURE),
+ endpoint,
+ callback);
+ }
+ public static void sendMessageWithFailureCB(SharedContext ctx, Supplier<Boolean> allowRetry, RepairMessage request, Verb verb, InetAddressAndPort endpoint, RepairFailureCallback failureCallback)
+ {
+ RequestCallback<?> callback = new RequestCallback<>()
+ {
+ @Override
+ public void onResponse(Message<Object> msg)
+ {
+ logger.info("[#{}] {} received by {}", request.parentRepairSession(), verb, endpoint);
+ // todo: at some point we should make repair messages follow the normal path, actually using this
+ }
+
+ @Override
public void onFailure(InetAddressAndPort from, RequestFailureReason failureReason)
{
- logger.error("[#{}] {} failed on {}: {}", request.desc.parentSessionId, verb, from, failureReason);
+ failureCallback.onFailure(RepairException.error(request.desc, PreviewKind.NONE, String.format("Got %s failure from %s: %s", verb, from, failureReason)));
+ }
- if (supportsTimeouts(from, request.desc.parentSessionId))
- failureCallback.onFailure(RepairException.error(request.desc, PreviewKind.NONE, String.format("Got %s failure from %s: %s", verb, from, failureReason)));
+ @Override
+ public boolean invokeOnFailure()
+ {
+ return true;
}
};
-
- MessagingService.instance().sendWithCallback(Message.outWithFlag(verb, request, CALL_BACK_ON_FAILURE),
- endpoint,
- callback);
+ sendMessageWithRetries(ctx, allowRetry, request, verb, endpoint, callback);
}
- private static boolean supportsTimeouts(InetAddressAndPort from, TimeUUID parentSessionId)
+ private static ErrorHandling errorHandlingSupported(SharedContext ctx, InetAddressAndPort from, Verb verb, TimeUUID parentSessionId)
{
- CassandraVersion remoteVersion = Gossiper.instance.getReleaseVersion(from);
- if (remoteVersion != null && remoteVersion.compareTo(SUPPORTS_TIMEOUTS) >= 0)
- return true;
- logger.warn("[#{}] Not failing repair due to remote host {} not supporting repair message timeouts (version = {})", parentSessionId, from, remoteVersion);
- return false;
+ if (SUPPORTS_RETRY_WITHOUT_VERSION_CHECK.contains(verb))
+ return ErrorHandling.RETRY;
+ // Repair in mixed mode isn't fully supported, but also not activally blocked... so in the common case all participants
+ // will be on the same version as this instance, so can avoid the lookup from gossip
+ CassandraVersion remoteVersion = ctx.gossiper().getReleaseVersion(from);
+ if (remoteVersion == null)
+ {
+ if (VERB_TIMEOUT_VERSIONS.containsKey(verb))
+ {
+ logger.warn("[#{}] Not failing repair due to remote host {} not supporting repair message timeouts (version is unknown)", parentSessionId, from);
+ return ErrorHandling.NONE;
+ }
+ return ErrorHandling.TIMEOUT;
+ }
+ if (remoteVersion.compareTo(SUPPORTS_RETRY) >= 0)
+ return ErrorHandling.RETRY;
+ CassandraVersion timeoutVersion = VERB_TIMEOUT_VERSIONS.get(verb);
+ if (timeoutVersion == null || remoteVersion.compareTo(timeoutVersion) >= 0)
+ return ErrorHandling.TIMEOUT;
+ return ErrorHandling.NONE;
}
}
diff --git a/src/java/org/apache/cassandra/repair/state/AbstractCompletable.java b/src/java/org/apache/cassandra/repair/state/AbstractCompletable.java
index 20e59d65d5..e919dc314b 100644
--- a/src/java/org/apache/cassandra/repair/state/AbstractCompletable.java
+++ b/src/java/org/apache/cassandra/repair/state/AbstractCompletable.java
@@ -24,19 +24,35 @@ import com.google.common.base.Throwables;
import org.apache.cassandra.utils.Clock;
-public class AbstractCompletable<I> implements Completable<I>
+public abstract class AbstractCompletable<I> implements Completable<I>
{
- private final long creationTimeMillis = Clock.Global.currentTimeMillis(); // used to convert from nanos to millis
- private final long creationTimeNanos = Clock.Global.nanoTime();
+ public enum Status { INIT, ACCEPTED, COMPLETED }
+
+ private final long creationTimeMillis; // used to convert from nanos to millis
+ private final long creationTimeNanos;
+ protected final Clock clock;
private final AtomicReference<Result> result = new AtomicReference<>(null);
public final I id;
protected volatile long lastUpdatedAtNs;
- public AbstractCompletable(I id)
+ public AbstractCompletable(Clock clock, I id)
{
+ this.creationTimeMillis = clock.currentTimeMillis();
+ this.creationTimeNanos = clock.nanoTime();
+ this.clock = clock;
this.id = id;
}
+ public abstract boolean isAccepted();
+
+ public Status getCompletionStatus()
+ {
+ Result result = getResult();
+ if (result != null)
+ return Status.COMPLETED;
+ return isAccepted() ? Status.ACCEPTED : Status.INIT;
+ }
+
@Override
public I getId()
{
@@ -75,7 +91,7 @@ public class AbstractCompletable<I> implements Completable<I>
public void updated()
{
- lastUpdatedAtNs = Clock.Global.nanoTime();
+ lastUpdatedAtNs = clock.nanoTime();
}
protected boolean tryResult(Result result)
@@ -83,7 +99,7 @@ public class AbstractCompletable<I> implements Completable<I>
if (!this.result.compareAndSet(null, result))
return false;
onComplete();
- lastUpdatedAtNs = Clock.Global.nanoTime();
+ lastUpdatedAtNs = clock.nanoTime();
return true;
}
diff --git a/src/java/org/apache/cassandra/repair/state/AbstractState.java b/src/java/org/apache/cassandra/repair/state/AbstractState.java
index f9b0a93645..08e172ce95 100644
--- a/src/java/org/apache/cassandra/repair/state/AbstractState.java
+++ b/src/java/org/apache/cassandra/repair/state/AbstractState.java
@@ -23,6 +23,27 @@ import org.apache.cassandra.utils.Clock;
public abstract class AbstractState<T extends Enum<T>, I> extends AbstractCompletable<I> implements State<T, I>
{
+ protected enum UpdateType
+ {
+ NO_CHANGE, ACCEPTED,
+ LARGER_STATE_SEEN, ALREADY_COMPLETED;
+
+ protected boolean isRejected()
+ {
+ switch (this)
+ {
+ case NO_CHANGE:
+ case ACCEPTED:
+ return false;
+ case LARGER_STATE_SEEN:
+ case ALREADY_COMPLETED:
+ return true;
+ default:
+ throw new IllegalStateException("Unknown type: " + this);
+ }
+ }
+ }
+
public static final int INIT = -1;
public static final int COMPLETE = -2;
@@ -30,13 +51,19 @@ public abstract class AbstractState<T extends Enum<T>, I> extends AbstractComple
protected final long[] stateTimesNanos;
protected int currentState = INIT;
- public AbstractState(I id, Class<T> klass)
+ public AbstractState(Clock clock, I id, Class<T> klass)
{
- super(id);
+ super(clock, id);
this.klass = klass;
this.stateTimesNanos = new long[klass.getEnumConstants().length];
}
+ @Override
+ public boolean isAccepted()
+ {
+ return currentState == INIT ? false : true;
+ }
+
@Override
public T getStatus()
{
@@ -46,6 +73,27 @@ public abstract class AbstractState<T extends Enum<T>, I> extends AbstractComple
return klass.getEnumConstants()[current];
}
+ public String status()
+ {
+ T state = getStatus();
+ Result result = getResult();
+ if (result != null)
+ return result.kind.name();
+ if (state == null)
+ return "init";
+ return state.name();
+ }
+
+ @Override
+ public String toString()
+ {
+ return getClass().getSimpleName() + "{" +
+ "id=" + id +
+ ", status=" + status() +
+ ", lastUpdatedAtNs=" + lastUpdatedAtNs +
+ '}';
+ }
+
public int getCurrentState()
{
return currentState;
@@ -85,11 +133,22 @@ public abstract class AbstractState<T extends Enum<T>, I> extends AbstractComple
protected void updateState(T state)
{
- int currentState = this.currentState;
- if (currentState >= state.ordinal())
+ if (maybeUpdateState(state).isRejected())
throw new IllegalStateException("State went backwards; current=" + klass.getEnumConstants()[currentState] + ", desired=" + state);
- long now = Clock.Global.nanoTime();
+ }
+
+ protected UpdateType maybeUpdateState(T state)
+ {
+ int currentState = this.currentState;
+ if (currentState == COMPLETE)
+ return UpdateType.ALREADY_COMPLETED;
+ if (currentState == state.ordinal())
+ return UpdateType.NO_CHANGE;
+ if (currentState > state.ordinal())
+ return UpdateType.LARGER_STATE_SEEN;
+ long now = clock.nanoTime();
stateTimesNanos[this.currentState = state.ordinal()] = now;
lastUpdatedAtNs = now;
+ return UpdateType.ACCEPTED;
}
}
diff --git a/src/java/org/apache/cassandra/repair/state/Completable.java b/src/java/org/apache/cassandra/repair/state/Completable.java
index 61813c6974..1cf496d124 100644
--- a/src/java/org/apache/cassandra/repair/state/Completable.java
+++ b/src/java/org/apache/cassandra/repair/state/Completable.java
@@ -17,6 +17,7 @@
*/
package org.apache.cassandra.repair.state;
+import java.util.Objects;
import java.util.concurrent.TimeUnit;
import org.apache.cassandra.utils.Clock;
@@ -78,24 +79,48 @@ public interface Completable<I>
this.message = message;
}
- protected static Result success()
+ public static Result success()
{
return new Result(Result.Kind.SUCCESS, null);
}
- protected static Result success(String msg)
+ public static Result success(String msg)
{
return new Result(Result.Kind.SUCCESS, msg);
}
- protected static Result skip(String msg)
+ public static Result skip(String msg)
{
return new Result(Result.Kind.SKIPPED, msg);
}
- protected static Result fail(String msg)
+ public static Result fail(String msg)
{
return new Result(Result.Kind.FAILURE, msg);
}
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ Result result = (Result) o;
+ return kind == result.kind && Objects.equals(message, result.message);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(kind, message);
+ }
+
+ @Override
+ public String toString()
+ {
+ return "Result{" +
+ "kind=" + kind +
+ ", message='" + message + '\'' +
+ '}';
+ }
}
}
diff --git a/src/java/org/apache/cassandra/repair/state/CoordinatorState.java b/src/java/org/apache/cassandra/repair/state/CoordinatorState.java
index 74ca3e393e..5bc8a9e5d8 100644
--- a/src/java/org/apache/cassandra/repair/state/CoordinatorState.java
+++ b/src/java/org/apache/cassandra/repair/state/CoordinatorState.java
@@ -17,18 +17,21 @@
*/
package org.apache.cassandra.repair.state;
+import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import java.util.stream.Collectors;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.repair.CommonRange;
-import org.apache.cassandra.repair.RepairRunnable;
+import org.apache.cassandra.repair.RepairCoordinator;
import org.apache.cassandra.repair.messages.RepairOption;
+import org.apache.cassandra.utils.Clock;
import org.apache.cassandra.utils.TimeUUID;
import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
@@ -48,14 +51,14 @@ public class CoordinatorState extends AbstractState<CoordinatorState.State, Time
private final ConcurrentMap<TimeUUID, SessionState> sessions = new ConcurrentHashMap<>();
private List<ColumnFamilyStore> columnFamilies = null;
- private RepairRunnable.NeighborsAndRanges neighborsAndRanges = null;
+ private RepairCoordinator.NeighborsAndRanges neighborsAndRanges = null;
// API to split function calls for phase changes from getting the state
public final Phase phase = new Phase();
- public CoordinatorState(int cmd, String keyspace, RepairOption options)
+ public CoordinatorState(Clock clock, int cmd, String keyspace, RepairOption options)
{
- super(nextTimeUUID(), State.class);
+ super(clock, nextTimeUUID(), State.class);
this.cmd = cmd;
this.keyspace = Objects.requireNonNull(keyspace);
this.options = Objects.requireNonNull(options);
@@ -88,7 +91,7 @@ public class CoordinatorState extends AbstractState<CoordinatorState.State, Time
return columnFamilies.stream().map(ColumnFamilyStore::getTableName).toArray(String[]::new);
}
- public RepairRunnable.NeighborsAndRanges getNeighborsAndRanges()
+ public RepairCoordinator.NeighborsAndRanges getNeighborsAndRanges()
{
return neighborsAndRanges;
}
@@ -114,6 +117,32 @@ public class CoordinatorState extends AbstractState<CoordinatorState.State, Time
return neighborsAndRanges.filterCommonRanges(keyspace, getColumnFamilyNames());
}
+ @Override
+ public String status()
+ {
+ State currentState = getStatus();
+ Result result = getResult();
+ if (result != null)
+ return result.kind.name();
+ else if (currentState == null)
+ return "init";
+ else if (currentState == State.REPAIR_START)
+ return currentState.name() + " " + sessions.entrySet().stream().map(e -> e.getKey() + " -> " + e.getValue().status()).collect(Collectors.toList());
+ else
+ return currentState.name();
+ }
+
+ @Override
+ public String toString()
+ {
+ return "CoordinatorState{" +
+ "id=" + id +
+ ", stateTimesNanos=" + Arrays.toString(stateTimesNanos) +
+ ", status=" + status() +
+ ", lastUpdatedAtNs=" + lastUpdatedAtNs +
+ '}';
+ }
+
public final class Phase extends BaseSkipPhase
{
public void setup()
@@ -121,7 +150,7 @@ public class CoordinatorState extends AbstractState<CoordinatorState.State, Time
updateState(State.SETUP);
}
- public void start(List<ColumnFamilyStore> columnFamilies, RepairRunnable.NeighborsAndRanges neighborsAndRanges)
+ public void start(List<ColumnFamilyStore> columnFamilies, RepairCoordinator.NeighborsAndRanges neighborsAndRanges)
{
CoordinatorState.this.columnFamilies = Objects.requireNonNull(columnFamilies);
CoordinatorState.this.neighborsAndRanges = Objects.requireNonNull(neighborsAndRanges);
diff --git a/src/java/org/apache/cassandra/repair/state/JobState.java b/src/java/org/apache/cassandra/repair/state/JobState.java
index f8d42c350e..6c02b6d73d 100644
--- a/src/java/org/apache/cassandra/repair/state/JobState.java
+++ b/src/java/org/apache/cassandra/repair/state/JobState.java
@@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableSet;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.repair.RepairJobDesc;
+import org.apache.cassandra.utils.Clock;
public class JobState extends AbstractState<JobState.State, UUID>
{
@@ -40,9 +41,9 @@ public class JobState extends AbstractState<JobState.State, UUID>
public final Phase phase = new Phase();
- public JobState(RepairJobDesc desc, ImmutableSet<InetAddressAndPort> endpoints)
+ public JobState(Clock clock, RepairJobDesc desc, ImmutableSet<InetAddressAndPort> endpoints)
{
- super(desc.determanisticId(), State.class);
+ super(clock, desc.determanisticId(), State.class);
this.desc = desc;
this.endpoints = endpoints;
}
diff --git a/src/java/org/apache/cassandra/repair/state/ParticipateState.java b/src/java/org/apache/cassandra/repair/state/ParticipateState.java
index c1faa4e939..ede9cc2024 100644
--- a/src/java/org/apache/cassandra/repair/state/ParticipateState.java
+++ b/src/java/org/apache/cassandra/repair/state/ParticipateState.java
@@ -22,17 +22,26 @@ import java.util.List;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import javax.annotation.Nullable;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.repair.RepairJobDesc;
import org.apache.cassandra.repair.messages.PrepareMessage;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.utils.Clock;
import org.apache.cassandra.utils.TimeUUID;
public class ParticipateState extends AbstractCompletable<TimeUUID>
{
+ public enum RegisterStatus
+ { ACCEPTED, EXISTS, STATUS_REJECTED, ALREADY_COMPLETED }
public final InetAddressAndPort initiator;
public final List<TableId> tableIds;
public final Collection<Range<Token>> ranges;
@@ -40,14 +49,21 @@ public class ParticipateState extends AbstractCompletable<TimeUUID>
public final long repairedAt;
public final boolean global;
public final PreviewKind previewKind;
-
- private final ConcurrentMap<UUID, ValidationState> validations = new ConcurrentHashMap<>();
+ private volatile boolean accepted = false;
public final Phase phase = new Phase();
- public ParticipateState(InetAddressAndPort initiator, PrepareMessage msg)
+ @Override
+ public boolean isAccepted()
{
- super(msg.parentRepairSession);
+ return accepted;
+ }
+
+ public final ConcurrentMap<RepairJobDesc, Job> jobs = new ConcurrentHashMap<>();
+
+ public ParticipateState(Clock clock, InetAddressAndPort initiator, PrepareMessage msg)
+ {
+ super(clock, msg.parentRepairSession);
this.initiator = initiator;
this.tableIds = msg.tableIds;
this.ranges = msg.ranges;
@@ -57,24 +73,150 @@ public class ParticipateState extends AbstractCompletable<TimeUUID>
this.previewKind = msg.previewKind;
}
- public boolean register(ValidationState state)
+ @Nullable
+ public Job job(RepairJobDesc desc)
+ {
+ return jobs.get(desc);
+ }
+
+ public Job getOrCreateJob(RepairJobDesc desc)
+ {
+ return jobs.computeIfAbsent(desc, d -> new Job(clock, d));
+ }
+
+ @Nullable
+ public ValidationState validation(RepairJobDesc desc)
+ {
+ Job job = job(desc);
+ if (job == null)
+ return null;
+ return job.validation();
+ }
+
+ public RegisterStatus register(ValidationState state)
+ {
+ return getOrCreateJob(state.desc).register(state);
+ }
+
+ @Nullable
+ public SyncState sync(RepairJobDesc desc, SyncState.Id id)
{
- ValidationState current = validations.putIfAbsent(state.id, state);
- return current == null;
+ Job job = job(desc);
+ if (job == null)
+ return null;
+ return job.sync(id);
+ }
+
+ public RegisterStatus register(SyncState state)
+ {
+ return getOrCreateJob(state.id.desc).register(state);
}
public Collection<ValidationState> validations()
{
- return validations.values();
+ return jobs.values().stream()
+ .map(j -> j.validation())
+ .filter(f -> f != null)
+ .collect(Collectors.toList());
}
public Collection<UUID> validationIds()
{
- return validations.keySet();
+ return jobs.values().stream()
+ .map(j -> j.validation())
+ .filter(f -> f != null)
+ .map(v -> v.id)
+ .collect(Collectors.toList());
+ }
+
+ @Override
+ public String toString()
+ {
+ Result result = getResult();
+ return "ParticipateState{" +
+ "initiator=" + initiator +
+ ", status=" + (result == null ? "pending" : result.toString()) +
+ ", jobs=" + jobs.values() +
+ '}';
}
public class Phase extends BasePhase
{
+ public void accept()
+ {
+ accepted = true;
+ }
+ }
+
+ public static class Job extends AbstractState<Job.State, RepairJobDesc>
+ {
+ public enum State { ACCEPT, SNAPSHOT, VALIDATION, SYNC }
+
+ private final AtomicReference<ValidationState> validation = new AtomicReference<>(null);
+ private final ConcurrentMap<SyncState.Id, SyncState> syncs = new ConcurrentHashMap<>();
+
+ public Job(Clock clock, RepairJobDesc desc)
+ {
+ super(clock, desc, State.class);
+ }
+
+ @Override
+ protected synchronized UpdateType maybeUpdateState(State state)
+ {
+ return super.maybeUpdateState(state);
+ }
+
+ public void snapshot()
+ {
+ updateState(State.SNAPSHOT);
+ }
+
+ public RegisterStatus register(ValidationState state)
+ {
+ return register(s -> validation.compareAndSet(null, s) ? null : validation(), State.VALIDATION, state);
+ }
+
+ @Nullable
+ public ValidationState validation()
+ {
+ return validation.get();
+ }
+
+ public RegisterStatus register(SyncState state)
+ {
+ return register(s -> syncs.putIfAbsent(s.id, s), State.SYNC, state);
+ }
+
+ private <I, S extends AbstractState<?, I>> RegisterStatus register(Function<S, S> putter, State state, S value)
+ {
+ UpdateType updateType = maybeUpdateState(state);
+ switch (updateType)
+ {
+ case ALREADY_COMPLETED:
+ return RegisterStatus.ALREADY_COMPLETED;
+ case LARGER_STATE_SEEN:
+ return RegisterStatus.STATUS_REJECTED;
+ case ACCEPTED:
+ case NO_CHANGE:
+ // allow
+ break;
+ default:
+ throw new IllegalStateException("Unknown status: " + updateType);
+ }
+ S current = putter.apply(value);
+ return current == null ? RegisterStatus.ACCEPTED : RegisterStatus.EXISTS;
+ }
+
+ @Nullable
+ public SyncState sync(SyncState.Id id)
+ {
+ return syncs.get(id);
+ }
+ @Override
+ public String toString()
+ {
+ return super.toString();
+ }
}
}
diff --git a/src/java/org/apache/cassandra/repair/state/SessionState.java b/src/java/org/apache/cassandra/repair/state/SessionState.java
index 8e8e6485be..32be089350 100644
--- a/src/java/org/apache/cassandra/repair/state/SessionState.java
+++ b/src/java/org/apache/cassandra/repair/state/SessionState.java
@@ -22,9 +22,11 @@ import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import java.util.stream.Collectors;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.repair.CommonRange;
+import org.apache.cassandra.utils.Clock;
import org.apache.cassandra.utils.TimeUUID;
import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
@@ -44,9 +46,9 @@ public class SessionState extends AbstractState<SessionState.State, TimeUUID>
public final Phase phase = new Phase();
- public SessionState(TimeUUID parentRepairSession, String keyspace, String[] cfnames, CommonRange commonRange)
+ public SessionState(Clock clock, TimeUUID parentRepairSession, String keyspace, String[] cfnames, CommonRange commonRange)
{
- super(nextTimeUUID(), State.class);
+ super(clock, nextTimeUUID(), State.class);
this.parentRepairSession = parentRepairSession;
this.keyspace = keyspace;
this.cfnames = cfnames;
@@ -73,6 +75,21 @@ public class SessionState extends AbstractState<SessionState.State, TimeUUID>
return commonRange.endpoints;
}
+ @Override
+ public String status()
+ {
+ State state = getStatus();
+ Result result = getResult();
+ if (result != null)
+ return result.kind.name();
+ else if (state == null)
+ return "init";
+ else if (state == State.JOBS_START)
+ return state.name() + " " + jobs.entrySet().stream().map(e -> e.getKey() + " -> " + e.getValue().status()).collect(Collectors.toList());
+ else
+ return state.name();
+ }
+
public void register(JobState state)
{
jobs.put(state.id, state);
diff --git a/src/java/org/apache/cassandra/repair/state/SyncState.java b/src/java/org/apache/cassandra/repair/state/SyncState.java
new file mode 100644
index 0000000000..9c88aa1877
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/state/SyncState.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.state;
+
+import java.util.Objects;
+
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.repair.RepairJobDesc;
+import org.apache.cassandra.utils.Clock;
+
+public class SyncState extends AbstractState<SyncState.State, SyncState.Id>
+{
+ public enum State
+ { ACCEPT, PLANNING, START }
+
+ public final Phase phase = new Phase();
+
+ public SyncState(Clock clock, RepairJobDesc desc, InetAddressAndPort initiator, InetAddressAndPort src, InetAddressAndPort dst)
+ {
+ super(clock, new Id(desc, initiator, src, dst), State.class);
+ }
+
+ public final class Phase extends BaseSkipPhase
+ {
+ public void accept()
+ {
+ updateState(State.ACCEPT);
+ }
+
+ public void planning()
+ {
+ updateState(State.PLANNING);
+ }
+
+ public void start()
+ {
+ updateState(State.START);
+ }
+ }
+
+ public static class Id
+ {
+ public final RepairJobDesc desc;
+ public final InetAddressAndPort initiator, src, dst;
+
+ public Id(RepairJobDesc desc, InetAddressAndPort initiator, InetAddressAndPort src, InetAddressAndPort dst)
+ {
+ this.desc = desc;
+ this.initiator = initiator;
+ this.src = src;
+ this.dst = dst;
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ Id id = (Id) o;
+ return desc.equals(id.desc) && initiator.equals(id.initiator) && src.equals(id.src) && dst.equals(id.dst);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(desc, initiator, src, dst);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/repair/state/ValidationState.java b/src/java/org/apache/cassandra/repair/state/ValidationState.java
index ff2ca537df..8dc63ec245 100644
--- a/src/java/org/apache/cassandra/repair/state/ValidationState.java
+++ b/src/java/org/apache/cassandra/repair/state/ValidationState.java
@@ -21,11 +21,12 @@ import java.util.UUID;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.repair.RepairJobDesc;
+import org.apache.cassandra.utils.Clock;
public class ValidationState extends AbstractState<ValidationState.State, UUID>
{
public enum State
- { START, SENDING_TREES }
+ { ACCEPT, START, SENDING_TREES }
public final Phase phase = new Phase();
public final RepairJobDesc desc;
@@ -35,9 +36,10 @@ public class ValidationState extends AbstractState<ValidationState.State, UUID>
public long partitionsProcessed;
public long bytesRead;
- public ValidationState(RepairJobDesc desc, InetAddressAndPort initiator)
+ public ValidationState(Clock clock, RepairJobDesc desc, InetAddressAndPort initiator)
{
- super(desc.determanisticId(), State.class);
+ // UUID is used to make the validations table easier for users to lookup by a single key rather than a composite key
+ super(clock, desc.determanisticId(), State.class);
this.desc = desc;
this.initiator = initiator;
}
@@ -56,6 +58,11 @@ public class ValidationState extends AbstractState<ValidationState.State, UUID>
public final class Phase extends BaseSkipPhase
{
+ public void accept()
+ {
+ updateState(State.ACCEPT);
+ }
+
public void start(long estimatedPartitions, long estimatedTotalBytes)
{
updateState(State.START);
diff --git a/src/java/org/apache/cassandra/schema/Tables.java b/src/java/org/apache/cassandra/schema/Tables.java
index 33ed1b8025..0f8f6b3190 100644
--- a/src/java/org/apache/cassandra/schema/Tables.java
+++ b/src/java/org/apache/cassandra/schema/Tables.java
@@ -123,7 +123,7 @@ public final class Tables implements Iterable<TableMetadata>
}
@Nullable
- TableMetadata getNullable(TableId id)
+ public TableMetadata getNullable(TableId id)
{
return tablesById.get(id);
}
diff --git a/src/java/org/apache/cassandra/service/ActiveRepairService.java b/src/java/org/apache/cassandra/service/ActiveRepairService.java
index 738551d7c3..755301dcff 100644
--- a/src/java/org/apache/cassandra/service/ActiveRepairService.java
+++ b/src/java/org/apache/cassandra/service/ActiveRepairService.java
@@ -26,8 +26,8 @@ import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicBoolean;
import javax.management.openmbean.CompositeData;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Predicate;
import java.util.stream.Collectors;
@@ -44,7 +44,7 @@ import com.google.common.collect.Multimap;
import org.apache.cassandra.concurrent.ExecutorPlus;
import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DurationSpec;
-import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.repair.SharedContext;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.locator.AbstractReplicationStrategy;
import org.apache.cassandra.locator.EndpointsByRange;
@@ -59,11 +59,10 @@ import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.utils.TimeUUID;
-import org.apache.cassandra.utils.concurrent.CountDownLatch;
+import org.apache.cassandra.utils.concurrent.AsyncPromise;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.concurrent.ScheduledExecutors;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
@@ -73,10 +72,8 @@ import org.apache.cassandra.exceptions.RequestFailureReason;
import org.apache.cassandra.gms.ApplicationState;
import org.apache.cassandra.gms.EndpointState;
import org.apache.cassandra.gms.FailureDetector;
-import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.gms.IEndpointStateChangeSubscriber;
import org.apache.cassandra.gms.IFailureDetectionEventListener;
-import org.apache.cassandra.gms.IFailureDetector;
import org.apache.cassandra.gms.VersionedValue;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.TokenMetadata;
@@ -84,7 +81,6 @@ import org.apache.cassandra.metrics.RepairMetrics;
import org.apache.cassandra.net.RequestCallback;
import org.apache.cassandra.net.Verb;
import org.apache.cassandra.net.Message;
-import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.repair.CommonRange;
import org.apache.cassandra.repair.NoSuchRepairSessionException;
import org.apache.cassandra.service.paxos.PaxosRepair;
@@ -106,8 +102,6 @@ import org.apache.cassandra.repair.messages.RepairOption;
import org.apache.cassandra.repair.messages.SyncResponse;
import org.apache.cassandra.repair.messages.ValidationResponse;
import org.apache.cassandra.schema.TableId;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.MBeanWrapper;
import org.apache.cassandra.utils.MerkleTrees;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.concurrent.Future;
@@ -126,12 +120,9 @@ import static org.apache.cassandra.config.CassandraRelevantProperties.SKIP_PAXOS
import static org.apache.cassandra.config.CassandraRelevantProperties.SKIP_PAXOS_REPAIR_ON_TOPOLOGY_CHANGE_KEYSPACES;
import static org.apache.cassandra.config.Config.RepairCommandPoolFullStrategy.reject;
import static org.apache.cassandra.config.DatabaseDescriptor.*;
-import static org.apache.cassandra.net.Message.out;
import static org.apache.cassandra.net.Verb.PREPARE_MSG;
-import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
+import static org.apache.cassandra.repair.messages.RepairMessage.notDone;
import static org.apache.cassandra.utils.Simulate.With.MONITORS;
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
-import static org.apache.cassandra.utils.concurrent.CountDownLatch.newCountDownLatch;
/**
* ActiveRepairService is the starting point for manual "active" repairs.
@@ -150,6 +141,7 @@ import static org.apache.cassandra.utils.concurrent.CountDownLatch.newCountDownL
@Simulate(with = MONITORS)
public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFailureDetectionEventListener, ActiveRepairServiceMBean
{
+
public enum ParentRepairStatus
{
IN_PROGRESS, COMPLETED, FAILED
@@ -157,21 +149,35 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
public static class ConsistentSessions
{
- public final LocalSessions local = new LocalSessions();
- public final CoordinatorSessions coordinated = new CoordinatorSessions();
+ public final LocalSessions local;
+ public final CoordinatorSessions coordinated;
+
+ public ConsistentSessions(SharedContext ctx)
+ {
+ local = new LocalSessions(ctx);
+ coordinated = new CoordinatorSessions(ctx);
+ }
}
- public final ConsistentSessions consistent = new ConsistentSessions();
+ public final ConsistentSessions consistent;
private boolean registeredForEndpointChanges = false;
private static final Logger logger = LoggerFactory.getLogger(ActiveRepairService.class);
- // singleton enforcement
- public static final ActiveRepairService instance = new ActiveRepairService(FailureDetector.instance, Gossiper.instance);
public static final long UNREPAIRED_SSTABLE = 0;
public static final TimeUUID NO_PENDING_REPAIR = null;
+ public static ActiveRepairService instance()
+ {
+ return Holder.instance;
+ }
+
+ private static class Holder
+ {
+ private static final ActiveRepairService instance = new ActiveRepairService();
+ }
+
/**
* A map of active coordinator session.
*/
@@ -182,6 +188,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
private final Cache<TimeUUID, CoordinatorState> repairs;
// map of top level repair id (parent repair id) -> participate state
private final Cache<TimeUUID, ParticipateState> participates;
+ public final SharedContext ctx;
private volatile ScheduledFuture<?> irCleanup;
@@ -213,18 +220,22 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
return RepairCommandExecutorHandle.repairCommandExecutor;
}
- private final IFailureDetector failureDetector;
- private final Gossiper gossiper;
private final Cache<Integer, Pair<ParentRepairStatus, List<String>>> repairStatusByCmd;
+ public final ExecutorPlus snapshotExecutor;
- public final ExecutorPlus snapshotExecutor = executorFactory().configurePooled("RepairSnapshotExecutor", 1)
- .withKeepAlive(1, TimeUnit.HOURS)
- .build();
+ public ActiveRepairService()
+ {
+ this(SharedContext.Global.instance);
+ }
- public ActiveRepairService(IFailureDetector failureDetector, Gossiper gossiper)
+ @VisibleForTesting
+ public ActiveRepairService(SharedContext ctx)
{
- this.failureDetector = failureDetector;
- this.gossiper = gossiper;
+ this.ctx = ctx;
+ consistent = new ConsistentSessions(ctx);
+ this.snapshotExecutor = ctx.executorFactory().configurePooled("RepairSnapshotExecutor", 1)
+ .withKeepAlive(1, TimeUnit.HOURS)
+ .build();
this.repairStatusByCmd = CacheBuilder.newBuilder()
.expireAfterWrite(PARENT_REPAIR_STATUS_EXPIRY_SECONDS.getLong(), TimeUnit.SECONDS)
// using weight wouldn't work so well, since it doesn't reflect mutation of cached data
@@ -245,15 +256,15 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
.maximumSize(numElements)
.build();
- MBeanWrapper.instance.registerMBean(this, MBEAN_NAME);
+ ctx.mbean().registerMBean(this, MBEAN_NAME);
}
public void start()
{
consistent.local.start();
- this.irCleanup = ScheduledExecutors.optionalTasks.scheduleAtFixedRate(consistent.local::cleanup, 0,
- LocalSessions.CLEANUP_INTERVAL,
- TimeUnit.SECONDS);
+ this.irCleanup = ctx.optionalTasks().scheduleAtFixedRate(consistent.local::cleanup, 0,
+ LocalSessions.CLEANUP_INTERVAL,
+ TimeUnit.SECONDS);
}
@VisibleForTesting
@@ -428,7 +439,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
if (cfnames.length == 0)
return null;
- final RepairSession session = new RepairSession(parentRepairSession, range, keyspace,
+ final RepairSession session = new RepairSession(ctx, parentRepairSession, range, keyspace,
parallelismDegree, isIncremental, pullRepair,
previewKind, optimiseStreams, repairPaxos, paxosOnly, cfnames);
repairs.getIfPresent(parentRepairSession).register(session.state);
@@ -463,8 +474,8 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
IEndpointStateChangeSubscriber &
IFailureDetectionEventListener> void registerOnFdAndGossip(final T task)
{
- gossiper.register(task);
- failureDetector.registerFailureDetectionEventListener(task);
+ ctx.gossiper().register(task);
+ ctx.failureDetector().registerFailureDetectionEventListener(task);
// unregister listeners at completion
task.addListener(new Runnable()
@@ -474,8 +485,8 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
*/
public void run()
{
- failureDetector.unregisterFailureDetectionEventListener(task);
- gossiper.unregister(task);
+ ctx.failureDetector().unregisterFailureDetectionEventListener(task);
+ ctx.gossiper().unregister(task);
}
});
}
@@ -511,9 +522,9 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
* @param dataCenters the data centers to involve in the repair
* @return neighbors with whom we share the provided range
*/
- public static EndpointsForRange getNeighbors(String keyspaceName, Iterable<Range<Token>> keyspaceLocalRanges,
- Range<Token> toRepair, Collection<String> dataCenters,
- Collection<String> hosts)
+ public EndpointsForRange getNeighbors(String keyspaceName, Iterable<Range<Token>> keyspaceLocalRanges,
+ Range<Token> toRepair, Collection<String> dataCenters,
+ Collection<String> hosts)
{
StorageService ss = StorageService.instance;
EndpointsByRange replicaSets = ss.getRangeToAddressMap(keyspaceName);
@@ -536,7 +547,8 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
if (rangeSuperSet == null || !replicaSets.containsKey(rangeSuperSet))
return EndpointsForRange.empty(toRepair);
- EndpointsForRange neighbors = replicaSets.get(rangeSuperSet).withoutSelf();
+ // same as withoutSelf(), but done this way for testing
+ EndpointsForRange neighbors = replicaSets.get(rangeSuperSet).filter(r -> !ctx.broadcastAddressAndPort().equals(r.endpoint()));
if (dataCenters != null && !dataCenters.isEmpty())
{
@@ -553,7 +565,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
try
{
final InetAddressAndPort endpoint = InetAddressAndPort.getByName(host.trim());
- if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()) || neighbors.endpoints().contains(endpoint))
+ if (endpoint.equals(ctx.broadcastAddressAndPort()) || neighbors.endpoints().contains(endpoint))
specifiedHost.add(endpoint);
}
catch (UnknownHostException e)
@@ -562,7 +574,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
}
}
- if (!specifiedHost.contains(FBUtilities.getBroadcastAddressAndPort()))
+ if (!specifiedHost.contains(ctx.broadcastAddressAndPort()))
throw new IllegalArgumentException("The current host must be part of the repair");
if (specifiedHost.size() <= 1)
@@ -573,7 +585,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
throw new IllegalArgumentException(String.format(msg, hosts, toRepair, neighbors));
}
- specifiedHost.remove(FBUtilities.getBroadcastAddressAndPort());
+ specifiedHost.remove(ctx.broadcastAddressAndPort());
return neighbors.keep(specifiedHost);
}
@@ -585,14 +597,14 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
* incremental repairs, forced incremental repairs, and full repairs, the UNREPAIRED_SSTABLE value will prevent
* sstables from being promoted to repaired or preserve the repairedAt/pendingRepair values, respectively.
*/
- static long getRepairedAt(RepairOption options, boolean force)
+ long getRepairedAt(RepairOption options, boolean force)
{
// we only want to set repairedAt for incremental repairs including all replicas for a token range. For non-global incremental repairs, full repairs, the UNREPAIRED_SSTABLE value will prevent
// sstables from being promoted to repaired or preserve the repairedAt/pendingRepair values, respectively. For forced repairs, repairedAt time is only set to UNREPAIRED_SSTABLE if we actually
// end up skipping replicas
if (options.isIncremental() && options.isGlobal() && !force)
{
- return currentTimeMillis();
+ return ctx.clock().currentTimeMillis();
}
else
{
@@ -600,11 +612,11 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
}
}
- public static boolean verifyCompactionsPendingThreshold(TimeUUID parentRepairSession, PreviewKind previewKind)
+ public boolean verifyCompactionsPendingThreshold(TimeUUID parentRepairSession, PreviewKind previewKind)
{
// Snapshot values so failure message is consistent with decision
- int pendingCompactions = CompactionManager.instance.getPendingTasks();
- int pendingThreshold = ActiveRepairService.instance.getRepairPendingCompactionRejectThreshold();
+ int pendingCompactions = ctx.compactionManager().getPendingTasks();
+ int pendingThreshold = getRepairPendingCompactionRejectThreshold();
if (pendingCompactions > pendingThreshold)
{
logger.error("[{}] Rejecting incoming repair, pending compactions ({}) above threshold ({})",
@@ -614,54 +626,28 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
return true;
}
- public TimeUUID prepareForRepair(TimeUUID parentRepairSession, InetAddressAndPort coordinator, Set<InetAddressAndPort> endpoints, RepairOption options, boolean isForcedRepair, List<ColumnFamilyStore> columnFamilyStores)
+ public Future<?> prepareForRepair(TimeUUID parentRepairSession, InetAddressAndPort coordinator, Set<InetAddressAndPort> endpoints, RepairOption options, boolean isForcedRepair, List<ColumnFamilyStore> columnFamilyStores)
{
if (!verifyCompactionsPendingThreshold(parentRepairSession, options.getPreviewKind()))
failRepair(parentRepairSession, "Rejecting incoming repair, pending compactions above threshold"); // failRepair throws exception
long repairedAt = getRepairedAt(options, isForcedRepair);
registerParentRepairSession(parentRepairSession, coordinator, columnFamilyStores, options.getRanges(), options.isIncremental(), repairedAt, options.isGlobal(), options.getPreviewKind());
- final CountDownLatch prepareLatch = newCountDownLatch(endpoints.size());
- final AtomicBoolean status = new AtomicBoolean(true);
- final Set<String> failedNodes = synchronizedSet(new HashSet<String>());
- final AtomicInteger timeouts = new AtomicInteger(0);
- RequestCallback callback = new RequestCallback()
- {
- @Override
- public void onResponse(Message msg)
- {
- prepareLatch.decrement();
- }
-
- @Override
- public void onFailure(InetAddressAndPort from, RequestFailureReason failureReason)
- {
- status.set(false);
- failedNodes.add(from.toString());
- if (failureReason == RequestFailureReason.TIMEOUT)
- timeouts.incrementAndGet();
- prepareLatch.decrement();
- }
-
- @Override
- public boolean invokeOnFailure()
- {
- return true;
- }
- };
+ AtomicInteger pending = new AtomicInteger(endpoints.size());
+ Set<String> failedNodes = synchronizedSet(new HashSet<>());
+ AsyncPromise<Void> promise = new AsyncPromise<>();
List<TableId> tableIds = new ArrayList<>(columnFamilyStores.size());
for (ColumnFamilyStore cfs : columnFamilyStores)
tableIds.add(cfs.metadata.id);
PrepareMessage message = new PrepareMessage(parentRepairSession, tableIds, options.getRanges(), options.isIncremental(), repairedAt, options.isGlobal(), options.getPreviewKind());
- register(new ParticipateState(FBUtilities.getBroadcastAddressAndPort(), message));
+ register(new ParticipateState(ctx.clock(), ctx.broadcastAddressAndPort(), message));
for (InetAddressAndPort neighbour : endpoints)
{
- if (FailureDetector.instance.isAlive(neighbour))
+ if (ctx.failureDetector().isAlive(neighbour))
{
- Message<RepairMessage> msg = out(PREPARE_MSG, message);
- MessagingService.instance().sendWithCallback(msg, neighbour, callback);
+ sendPrepareWithRetries(parentRepairSession, pending, failedNodes, promise, neighbour, message);
}
else
{
@@ -669,7 +655,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
// remaining ones go down, we still want to fail so we don't create repair sessions that can't complete
if (isForcedRepair && !options.isIncremental())
{
- prepareLatch.decrement();
+ pending.decrementAndGet();
}
else
{
@@ -678,22 +664,66 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
}
}
}
- try
+ // implement timeout to bound the runtime of the future
+ long timeoutMillis = getRepairRetrySpec().isEnabled() ? getRepairRpcTimeout(MILLISECONDS)
+ : getRpcTimeout(MILLISECONDS);
+ ctx.optionalTasks().schedule(() -> {
+ if (promise.isDone())
+ return;
+ String errorMsg = "Did not get replies from all endpoints.";
+ if (promise.tryFailure(new RuntimeException(errorMsg)))
+ participateFailed(parentRepairSession, errorMsg);
+ }, timeoutMillis, MILLISECONDS);
+
+ return promise;
+ }
+
+ private void sendPrepareWithRetries(TimeUUID parentRepairSession,
+ AtomicInteger pending,
+ Set<String> failedNodes,
+ AsyncPromise<Void> promise,
+ InetAddressAndPort to,
+ RepairMessage msg)
+ {
+ RepairMessage.sendMessageWithRetries(ctx, notDone(promise), msg, PREPARE_MSG, to, new RequestCallback<>()
{
- if (!prepareLatch.await(getRpcTimeout(MILLISECONDS), MILLISECONDS) || timeouts.get() > 0)
- failRepair(parentRepairSession, "Did not get replies from all endpoints.");
- }
- catch (InterruptedException e)
- {
- failRepair(parentRepairSession, "Interrupted while waiting for prepare repair response.");
- }
+ @Override
+ public void onResponse(Message<Object> msg)
+ {
+ ack();
+ }
- if (!status.get())
- {
- failRepair(parentRepairSession, "Got negative replies from endpoints " + failedNodes);
- }
+ @Override
+ public void onFailure(InetAddressAndPort from, RequestFailureReason failureReason)
+ {
+ failedNodes.add(from.toString());
+ if (failureReason == RequestFailureReason.TIMEOUT)
+ {
+ pending.set(-1);
+ promise.setFailure(failRepairException(parentRepairSession, "Did not get replies from all endpoints."));
+ }
+ else
+ {
+ ack();
+ }
+ }
+
+ private void ack()
+ {
+ if (pending.decrementAndGet() == 0)
+ {
+ if (failedNodes.isEmpty())
+ {
+ promise.setSuccess(null);
+ }
+ else
+ {
+ promise.setFailure(failRepairException(parentRepairSession, "Got negative replies from endpoints " + failedNodes));
+ }
+ }
+ }
+ });
- return parentRepairSession;
}
/**
@@ -707,10 +737,9 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
{
try
{
- if (FailureDetector.instance.isAlive(endpoint))
+ if (ctx.failureDetector().isAlive(endpoint))
{
CleanupMessage message = new CleanupMessage(parentRepairSession);
- Message<CleanupMessage> msg = Message.out(Verb.CLEANUP_MSG, message);
RequestCallback loggingCallback = new RequestCallback()
{
@@ -728,8 +757,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
"of messages like this.", parentRepairSession, endpoint);
}
};
-
- MessagingService.instance().sendWithCallback(msg, endpoint, loggingCallback);
+ RepairMessage.sendMessageWithRetries(ctx, message, Verb.CLEANUP_MSG, endpoint, loggingCallback);
}
}
catch (Exception exc)
@@ -743,12 +771,22 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
}
private void failRepair(TimeUUID parentRepairSession, String errorMsg)
+ {
+ throw failRepairException(parentRepairSession, errorMsg);
+ }
+
+ private RuntimeException failRepairException(TimeUUID parentRepairSession, String errorMsg)
+ {
+ participateFailed(parentRepairSession, errorMsg);
+ removeParentRepairSession(parentRepairSession);
+ return new RuntimeException(errorMsg);
+ }
+
+ private void participateFailed(TimeUUID parentRepairSession, String errorMsg)
{
ParticipateState state = participate(parentRepairSession);
if (state != null)
state.phase.fail(errorMsg);
- removeParentRepairSession(parentRepairSession);
- throw new RuntimeException(errorMsg);
}
public synchronized void registerParentRepairSession(TimeUUID parentRepairSession, InetAddressAndPort coordinator, List<ColumnFamilyStore> columnFamilyStores, Collection<Range<Token>> ranges, boolean isIncremental, long repairedAt, boolean isGlobal, PreviewKind previewKind)
@@ -756,8 +794,8 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
assert isIncremental || repairedAt == ActiveRepairService.UNREPAIRED_SSTABLE;
if (!registeredForEndpointChanges)
{
- Gossiper.instance.register(this);
- FailureDetector.instance.registerFailureDetectionEventListener(this);
+ ctx.gossiper().register(this);
+ ctx.failureDetector().registerFailureDetectionEventListener(this);
registeredForEndpointChanges = true;
}
@@ -796,25 +834,25 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
*/
public synchronized ParentRepairSession removeParentRepairSession(TimeUUID parentSessionId)
{
- String snapshotName = parentSessionId.toString();
ParentRepairSession session = parentRepairSessions.remove(parentSessionId);
if (session == null)
return null;
- if (session.hasSnapshots)
+ String snapshotName = parentSessionId.toString();
+ if (session.hasSnapshots.get())
{
snapshotExecutor.submit(() -> {
logger.info("[repair #{}] Clearing snapshots for {}", parentSessionId,
session.columnFamilyStores.values()
.stream()
.map(cfs -> cfs.metadata().toString()).collect(Collectors.joining(", ")));
- long startNanos = nanoTime();
+ long startNanos = ctx.clock().nanoTime();
for (ColumnFamilyStore cfs : session.columnFamilyStores.values())
{
if (cfs.snapshotExists(snapshotName))
cfs.clearSnapshot(snapshotName);
}
- logger.info("[repair #{}] Cleared snapshots in {}ms", parentSessionId, TimeUnit.NANOSECONDS.toMillis(nanoTime() - startNanos));
+ logger.info("[repair #{}] Cleared snapshots in {}ms", parentSessionId, TimeUnit.NANOSECONDS.toMillis(ctx.clock().nanoTime() - startNanos));
});
}
return session;
@@ -828,6 +866,13 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
if (session == null)
{
+ switch (message.verb())
+ {
+ case VALIDATION_RSP:
+ case SYNC_RSP:
+ ctx.messaging().send(message.emptyResponse(), message.from());
+ break;
+ }
if (payload instanceof ValidationResponse)
{
// The trees may be off-heap, and will therefore need to be released.
@@ -845,13 +890,10 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
switch (message.verb())
{
case VALIDATION_RSP:
- ValidationResponse validation = (ValidationResponse) payload;
- session.validationComplete(desc, message.from(), validation.trees);
+ session.validationComplete(desc, (Message<ValidationResponse>) message);
break;
case SYNC_RSP:
- // one of replica is synced.
- SyncResponse sync = (SyncResponse) payload;
- session.syncComplete(desc, sync.nodes, sync.success, sync.summaries);
+ session.syncComplete(desc, (Message<SyncResponse>) message);
break;
default:
break;
@@ -873,7 +915,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
public final long repairedAt;
public final InetAddressAndPort coordinator;
public final PreviewKind previewKind;
- public volatile boolean hasSnapshots = false;
+ public final AtomicBoolean hasSnapshots = new AtomicBoolean(false);
public ParentRepairSession(InetAddressAndPort coordinator, List<ColumnFamilyStore> columnFamilyStores, Collection<Range<Token>> ranges, boolean isIncremental, long repairedAt, boolean isGlobal, PreviewKind previewKind)
{
@@ -930,9 +972,9 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
'}';
}
- public void setHasSnapshots()
+ public boolean setHasSnapshots()
{
- hasSnapshots = true;
+ return hasSnapshots.compareAndSet(false, true);
}
}
diff --git a/src/java/org/apache/cassandra/service/CassandraDaemon.java b/src/java/org/apache/cassandra/service/CassandraDaemon.java
index 2407dea8d0..332f2a714f 100644
--- a/src/java/org/apache/cassandra/service/CassandraDaemon.java
+++ b/src/java/org/apache/cassandra/service/CassandraDaemon.java
@@ -367,7 +367,7 @@ public class CassandraDaemon
if (sizeRecorderInterval > 0)
ScheduledExecutors.optionalTasks.scheduleWithFixedDelay(SizeEstimatesRecorder.instance, 30, sizeRecorderInterval, TimeUnit.SECONDS);
- ActiveRepairService.instance.start();
+ ActiveRepairService.instance().start();
StreamManager.instance.start();
// Prepared statements
diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java
index c119f2d82b..1a79cdeeed 100644
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@ -179,7 +179,7 @@ import org.apache.cassandra.metrics.StorageMetrics;
import org.apache.cassandra.net.AsyncOneResponse;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.repair.RepairRunnable;
+import org.apache.cassandra.repair.RepairCoordinator;
import org.apache.cassandra.repair.messages.RepairOption;
import org.apache.cassandra.schema.CompactionParams.TombstoneOption;
import org.apache.cassandra.schema.KeyspaceMetadata;
@@ -361,8 +361,12 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
public RangesAtEndpoint getLocalReplicas(String keyspaceName)
{
- return Keyspace.open(keyspaceName).getReplicationStrategy()
- .getAddressReplicas(FBUtilities.getBroadcastAddressAndPort());
+ return getReplicas(keyspaceName, FBUtilities.getBroadcastAddressAndPort());
+ }
+
+ public RangesAtEndpoint getReplicas(String keyspaceName, InetAddressAndPort endpoint)
+ {
+ return Keyspace.open(keyspaceName).getReplicationStrategy().getAddressReplicas(endpoint);
}
public List<Range<Token>> getLocalRanges(String ks)
@@ -4593,7 +4597,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
* @param cfNames CFs
* @throws java.lang.IllegalArgumentException when given CF name does not exist
*/
- public Iterable<ColumnFamilyStore> getValidColumnFamilies(boolean allowIndexes, boolean autoAddIndexes, String keyspaceName, String... cfNames) throws IOException
+ public Iterable<ColumnFamilyStore> getValidColumnFamilies(boolean allowIndexes, boolean autoAddIndexes, String keyspaceName, String... cfNames)
{
Keyspace keyspace = getValidKeyspace(keyspaceName);
return keyspace.getValidColumnFamilies(allowIndexes, autoAddIndexes, cfNames);
@@ -4714,7 +4718,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
{
if (!options.getDataCenters().isEmpty() && !options.getDataCenters().contains(DatabaseDescriptor.getLocalDataCenter()))
{
- throw new IllegalArgumentException("the local data center must be part of the repair");
+ throw new IllegalArgumentException("the local data center must be part of the repair; requested " + options.getDataCenters() + " but DC is " + DatabaseDescriptor.getLocalDataCenter());
}
Set<String> existingDatacenters = tokenMetadata.cloneOnlyTokenMap().getTopology().getDatacenterEndpoints().keys().elementSet();
List<String> datacenters = new ArrayList<>(options.getDataCenters());
@@ -4724,7 +4728,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
throw new IllegalArgumentException("data center(s) " + datacenters.toString() + " not found");
}
- RepairRunnable task = new RepairRunnable(this, cmd, options, keyspace);
+ RepairCoordinator task = new RepairCoordinator(this, cmd, options, keyspace);
task.addProgressListener(progressSupport);
for (ProgressListener listener : listeners)
task.addProgressListener(listener);
@@ -4807,7 +4811,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
continue;
List<Range<Token>> ranges = getLocalAndPendingRanges(ksName);
- futures.add(ActiveRepairService.instance.repairPaxosForTopologyChange(ksName, ranges, reason));
+ futures.add(ActiveRepairService.instance().repairPaxosForTopologyChange(ksName, ranges, reason));
}
return FutureCombiner.allOf(futures);
@@ -4827,13 +4831,13 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
public void forceTerminateAllRepairSessions()
{
- ActiveRepairService.instance.terminateSessions();
+ ActiveRepairService.instance().terminateSessions();
}
@Nullable
public List<String> getParentRepairStatus(int cmd)
{
- Pair<ParentRepairStatus, List<String>> pair = ActiveRepairService.instance.getRepairStatus(cmd);
+ Pair<ParentRepairStatus, List<String>> pair = ActiveRepairService.instance().getRepairStatus(cmd);
return pair == null ? null :
ImmutableList.<String>builder().add(pair.left.name()).addAll(pair.right).build();
}
@@ -5729,7 +5733,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
shutdownClientServers();
ScheduledExecutors.optionalTasks.shutdown();
Gossiper.instance.stop();
- ActiveRepairService.instance.stop();
+ ActiveRepairService.instance().stop();
if (!isFinalShutdown)
setMode(Mode.DRAINING, "shutting down MessageService", false);
diff --git a/src/java/org/apache/cassandra/service/StorageServiceMBean.java b/src/java/org/apache/cassandra/service/StorageServiceMBean.java
index 0ed8e655cf..dc76845cfc 100644
--- a/src/java/org/apache/cassandra/service/StorageServiceMBean.java
+++ b/src/java/org/apache/cassandra/service/StorageServiceMBean.java
@@ -477,7 +477,7 @@ public interface StorageServiceMBean extends NotificationEmitter
/**
* Get the status of a given parent repair session.
* @param cmd the int reference returned when issuing the repair
- * @return status of parent repair from enum {@link org.apache.cassandra.repair.RepairRunnable.Status}
+ * @return status of parent repair from enum {@link ActiveRepairService.ParentRepairStatus}
* followed by final message or messages of the session
*/
@Nullable
diff --git a/src/java/org/apache/cassandra/streaming/PreviewKind.java b/src/java/org/apache/cassandra/streaming/PreviewKind.java
index 8d7ad8ab1b..23b010e971 100644
--- a/src/java/org/apache/cassandra/streaming/PreviewKind.java
+++ b/src/java/org/apache/cassandra/streaming/PreviewKind.java
@@ -86,7 +86,7 @@ public enum PreviewKind
StatsMetadata sstableMetadata = sstable.getSSTableMetadata();
if (sstableMetadata.pendingRepair != null)
{
- LocalSession session = ActiveRepairService.instance.consistent.local.getSession(sstableMetadata.pendingRepair);
+ LocalSession session = ActiveRepairService.instance().consistent.local.getSession(sstableMetadata.pendingRepair);
if (session == null)
return false;
else if (session.getState() == ConsistentSession.State.FINALIZED)
diff --git a/src/java/org/apache/cassandra/streaming/StreamPlan.java b/src/java/org/apache/cassandra/streaming/StreamPlan.java
index 9e79a5db4b..47fa9e1463 100644
--- a/src/java/org/apache/cassandra/streaming/StreamPlan.java
+++ b/src/java/org/apache/cassandra/streaming/StreamPlan.java
@@ -151,6 +151,22 @@ public class StreamPlan
return this;
}
+ public TimeUUID planId()
+ {
+ return planId;
+ }
+
+ public StreamOperation streamOperation()
+ {
+ return streamOperation;
+ }
+
+ @VisibleForTesting
+ public List<StreamEventHandler> handlers()
+ {
+ return handlers;
+ }
+
/**
* Set custom StreamConnectionFactory to be used for establishing connection
*
diff --git a/src/java/org/apache/cassandra/streaming/StreamSession.java b/src/java/org/apache/cassandra/streaming/StreamSession.java
index c63e69cd23..d76f1f6d25 100644
--- a/src/java/org/apache/cassandra/streaming/StreamSession.java
+++ b/src/java/org/apache/cassandra/streaming/StreamSession.java
@@ -542,7 +542,9 @@ public class StreamSession
this.failureReason = failureReason;
sink.onClose(peer);
- streamResult.handleSessionComplete(this);
+ // closed before init?
+ if (streamResult != null)
+ streamResult.handleSessionComplete(this);
}}).flatMap(ignore -> {
List<Future<?>> futures = new ArrayList<>();
// ensure aborting the tasks do not happen on the network IO thread (read: netty event loop)
@@ -959,7 +961,7 @@ public class StreamSession
}
}
Collections.sort(tables);
- int pendingThreshold = ActiveRepairService.instance.getRepairPendingCompactionRejectThreshold();
+ int pendingThreshold = ActiveRepairService.instance().getRepairPendingCompactionRejectThreshold();
if (pendingCompactionsAfterStreaming > pendingThreshold)
{
logger.error("[Stream #{}] Rejecting incoming files based on pending compactions calculation " +
diff --git a/src/java/org/apache/cassandra/tools/nodetool/Repair.java b/src/java/org/apache/cassandra/tools/nodetool/Repair.java
index 8b077628d8..3583240830 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/Repair.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/Repair.java
@@ -20,6 +20,7 @@ package org.apache.cassandra.tools.nodetool;
import static com.google.common.collect.Lists.newArrayList;
import static org.apache.commons.lang3.StringUtils.EMPTY;
import io.airlift.airline.Arguments;
+import io.airlift.airline.Cli;
import io.airlift.airline.Command;
import io.airlift.airline.Option;
@@ -28,6 +29,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.function.Supplier;
import com.google.common.collect.Sets;
@@ -141,39 +143,7 @@ public class Repair extends NodeToolCmd
if ((args == null || args.isEmpty()) && ONLY_EXPLICITLY_REPAIRED.contains(keyspace))
continue;
- Map<String, String> options = new HashMap<>();
- RepairParallelism parallelismDegree = RepairParallelism.PARALLEL;
- if (sequential)
- parallelismDegree = RepairParallelism.SEQUENTIAL;
- else if (dcParallel)
- parallelismDegree = RepairParallelism.DATACENTER_AWARE;
- options.put(RepairOption.PARALLELISM_KEY, parallelismDegree.getName());
- options.put(RepairOption.PRIMARY_RANGE_KEY, Boolean.toString(primaryRange));
- options.put(RepairOption.INCREMENTAL_KEY, Boolean.toString(!fullRepair && !(paxosOnly && getPreviewKind() == PreviewKind.NONE)));
- options.put(RepairOption.JOB_THREADS_KEY, Integer.toString(numJobThreads));
- options.put(RepairOption.TRACE_KEY, Boolean.toString(trace));
- options.put(RepairOption.COLUMNFAMILIES_KEY, StringUtils.join(cfnames, ","));
- options.put(RepairOption.PULL_REPAIR_KEY, Boolean.toString(pullRepair));
- options.put(RepairOption.FORCE_REPAIR_KEY, Boolean.toString(force));
- options.put(RepairOption.PREVIEW, getPreviewKind().toString());
- options.put(RepairOption.OPTIMISE_STREAMS_KEY, Boolean.toString(optimiseStreams));
- options.put(RepairOption.IGNORE_UNREPLICATED_KS, Boolean.toString(ignoreUnreplicatedKeyspaces));
- options.put(RepairOption.REPAIR_PAXOS_KEY, Boolean.toString(!skipPaxos && getPreviewKind() == PreviewKind.NONE));
- options.put(RepairOption.PAXOS_ONLY_KEY, Boolean.toString(paxosOnly && getPreviewKind() == PreviewKind.NONE));
-
- if (!startToken.isEmpty() || !endToken.isEmpty())
- {
- options.put(RepairOption.RANGES_KEY, startToken + ":" + endToken);
- }
- if (localDC)
- {
- options.put(RepairOption.DATACENTERS_KEY, StringUtils.join(newArrayList(probe.getDataCenter()), ","));
- }
- else
- {
- options.put(RepairOption.DATACENTERS_KEY, StringUtils.join(specificDataCenters, ","));
- }
- options.put(RepairOption.HOSTS_KEY, StringUtils.join(specificHosts, ","));
+ Map<String, String> options = createOptions(probe::getDataCenter, cfnames);
try
{
probe.repairAsync(probe.output().out, keyspace, options);
@@ -183,4 +153,53 @@ public class Repair extends NodeToolCmd
}
}
}
+
+ public static Map<String, String> parseOptionMap(Supplier<String> localDCOption, List<String> args)
+ {
+ List<String> realArgs = new ArrayList<>(args.size() + 1);
+ realArgs.add("repair");
+ realArgs.addAll(args);
+ Cli<Object> parser = Cli.builder("fortesting").withCommand(Repair.class).build();
+ Repair repair = (Repair) parser.parse(realArgs);
+ String[] cfnames = repair.parseOptionalTables(repair.args);
+ return repair.createOptions(localDCOption, cfnames);
+ }
+
+ private Map<String, String> createOptions(Supplier<String> localDCOption, String[] cfnames)
+ {
+ Map<String, String> options = new HashMap<>();
+ RepairParallelism parallelismDegree = RepairParallelism.PARALLEL;
+ if (sequential)
+ parallelismDegree = RepairParallelism.SEQUENTIAL;
+ else if (dcParallel)
+ parallelismDegree = RepairParallelism.DATACENTER_AWARE;
+ options.put(RepairOption.PARALLELISM_KEY, parallelismDegree.getName());
+ options.put(RepairOption.PRIMARY_RANGE_KEY, Boolean.toString(primaryRange));
+ options.put(RepairOption.INCREMENTAL_KEY, Boolean.toString(!fullRepair && !(paxosOnly && getPreviewKind() == PreviewKind.NONE)));
+ options.put(RepairOption.JOB_THREADS_KEY, Integer.toString(numJobThreads));
+ options.put(RepairOption.TRACE_KEY, Boolean.toString(trace));
+ options.put(RepairOption.COLUMNFAMILIES_KEY, StringUtils.join(cfnames, ","));
+ options.put(RepairOption.PULL_REPAIR_KEY, Boolean.toString(pullRepair));
+ options.put(RepairOption.FORCE_REPAIR_KEY, Boolean.toString(force));
+ options.put(RepairOption.PREVIEW, getPreviewKind().toString());
+ options.put(RepairOption.OPTIMISE_STREAMS_KEY, Boolean.toString(optimiseStreams));
+ options.put(RepairOption.IGNORE_UNREPLICATED_KS, Boolean.toString(ignoreUnreplicatedKeyspaces));
+ options.put(RepairOption.REPAIR_PAXOS_KEY, Boolean.toString(!skipPaxos && getPreviewKind() == PreviewKind.NONE));
+ options.put(RepairOption.PAXOS_ONLY_KEY, Boolean.toString(paxosOnly && getPreviewKind() == PreviewKind.NONE));
+
+ if (!startToken.isEmpty() || !endToken.isEmpty())
+ {
+ options.put(RepairOption.RANGES_KEY, startToken + ":" + endToken);
+ }
+ if (localDC)
+ {
+ options.put(RepairOption.DATACENTERS_KEY, StringUtils.join(newArrayList(localDCOption.get()), ","));
+ }
+ else
+ {
+ options.put(RepairOption.DATACENTERS_KEY, StringUtils.join(specificDataCenters, ","));
+ }
+ options.put(RepairOption.HOSTS_KEY, StringUtils.join(specificHosts, ","));
+ return options;
+ }
}
diff --git a/src/java/org/apache/cassandra/utils/Backoff.java b/src/java/org/apache/cassandra/utils/Backoff.java
new file mode 100644
index 0000000000..2f0b7e2c9c
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/Backoff.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import java.util.concurrent.TimeUnit;
+import java.util.function.DoubleSupplier;
+
+public interface Backoff
+{
+ /**
+ * @return max attempts allowed, {@code == 0} implies no retries are allowed
+ */
+ int maxAttempts();
+ long computeWaitTime(int retryCount);
+ TimeUnit unit();
+
+ enum None implements Backoff
+ {
+ INSTANCE;
+
+ @Override
+ public int maxAttempts()
+ {
+ return 0;
+ }
+
+ @Override
+ public long computeWaitTime(int retryCount)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public TimeUnit unit()
+ {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ class ExponentialBackoff implements Backoff
+ {
+ private final int maxAttempts;
+ private final long baseSleepTimeMillis;
+ private final long maxSleepMillis;
+ private final DoubleSupplier randomSource;
+
+ public ExponentialBackoff(int maxAttempts, long baseSleepTimeMillis, long maxSleepMillis, DoubleSupplier randomSource)
+ {
+ this.maxAttempts = maxAttempts;
+ this.baseSleepTimeMillis = baseSleepTimeMillis;
+ this.maxSleepMillis = maxSleepMillis;
+ this.randomSource = randomSource;
+ }
+
+ @Override
+ public int maxAttempts()
+ {
+ return maxAttempts;
+ }
+
+ @Override
+ public long computeWaitTime(int retryCount)
+ {
+ long baseTimeMillis = baseSleepTimeMillis * (1L << retryCount);
+ // it's possible that this overflows, so fall back to max;
+ if (baseTimeMillis <= 0)
+ baseTimeMillis = maxSleepMillis;
+ // now make sure this is capped to target max
+ baseTimeMillis = Math.min(baseTimeMillis, maxSleepMillis);
+
+ return (long) (baseTimeMillis * (randomSource.getAsDouble() + 0.5));
+ }
+
+ @Override
+ public TimeUnit unit()
+ {
+ return TimeUnit.MILLISECONDS;
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/utils/Clock.java b/src/java/org/apache/cassandra/utils/Clock.java
index 7fffefb587..c8ba785cab 100644
--- a/src/java/org/apache/cassandra/utils/Clock.java
+++ b/src/java/org/apache/cassandra/utils/Clock.java
@@ -87,6 +87,11 @@ public interface Clock
INITIALIZE_MESSAGE = null;
}
+ public static Clock clock()
+ {
+ return instance;
+ }
+
/**
* Semantically equivalent to {@link System#nanoTime()}
*/
@@ -133,6 +138,11 @@ public interface Clock
*/
public long currentTimeMillis();
+ public default long nowInSeconds()
+ {
+ return currentTimeMillis() / 1000L;
+ }
+
@Intercept
public static void waitUntil(long deadlineNanos) throws InterruptedException
{
diff --git a/src/java/org/apache/cassandra/utils/FailingBiConsumer.java b/src/java/org/apache/cassandra/utils/FailingBiConsumer.java
new file mode 100644
index 0000000000..d7dcbfb294
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/FailingBiConsumer.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import java.util.function.BiConsumer;
+
+public interface FailingBiConsumer<A, B> extends BiConsumer<A, B>
+{
+ void acceptOrFail(A a, B b) throws Throwable;
+
+ @Override
+ default void accept(A a, B b)
+ {
+ try
+ {
+ acceptOrFail(a, b);
+ }
+ catch (Throwable e)
+ {
+ throw Throwables.throwAsUncheckedException(e);
+ }
+ }
+
+ static <A, B> FailingBiConsumer<A, B> orFail(FailingBiConsumer<A, B> fn)
+ {
+ return fn;
+ }
+}
diff --git a/src/java/org/apache/cassandra/utils/MerkleTree.java b/src/java/org/apache/cassandra/utils/MerkleTree.java
index 0cdb74e742..d84eedfced 100644
--- a/src/java/org/apache/cassandra/utils/MerkleTree.java
+++ b/src/java/org/apache/cassandra/utils/MerkleTree.java
@@ -1015,7 +1015,7 @@ public class MerkleTree
default void serialize(DataOutputPlus out, int version) throws IOException
{
byte[] hash = hash();
- assert hash.length == HASH_SIZE;
+ assert hash.length == HASH_SIZE: String.format("Expected hash length to be %d, but given %d", HASH_SIZE, hash.length);
out.writeByte(Leaf.IDENT);
diff --git a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java
index a6d838e3fe..7c09d90aa7 100644
--- a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java
+++ b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java
@@ -747,7 +747,7 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance
FBUtilities.getBroadcastAddressAndPort().getPort() != broadcastAddress().getPort())
throw new IllegalStateException(String.format("%s != %s", FBUtilities.getBroadcastAddressAndPort(), broadcastAddress()));
- ActiveRepairService.instance.start();
+ ActiveRepairService.instance().start();
StreamManager.instance.start();
PaxosState.startAutoRepairs();
@@ -898,7 +898,7 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance
() -> DiagnosticSnapshotService.instance.shutdownAndWait(1L, MINUTES),
() -> SSTableReader.shutdownBlocking(1L, MINUTES),
() -> shutdownAndWait(Collections.singletonList(ActiveRepairService.repairCommandExecutor())),
- () -> ActiveRepairService.instance.shutdownNowAndWait(1L, MINUTES),
+ () -> ActiveRepairService.instance().shutdownNowAndWait(1L, MINUTES),
() -> SnapshotManager.shutdownAndWait(1L, MINUTES)
);
diff --git a/test/distributed/org/apache/cassandra/distributed/mock/nodetool/InternalNodeProbe.java b/test/distributed/org/apache/cassandra/distributed/mock/nodetool/InternalNodeProbe.java
index 381bb09ec4..4de180636c 100644
--- a/test/distributed/org/apache/cassandra/distributed/mock/nodetool/InternalNodeProbe.java
+++ b/test/distributed/org/apache/cassandra/distributed/mock/nodetool/InternalNodeProbe.java
@@ -79,7 +79,7 @@ public class InternalNodeProbe extends NodeProbe
gcProxy = new GCInspector();
gossProxy = Gossiper.instance;
bmProxy = BatchlogManager.instance;
- arsProxy = ActiveRepairService.instance;
+ arsProxy = ActiveRepairService.instance();
memProxy = ManagementFactory.getMemoryMXBean();
runtimeProxy = ManagementFactory.getRuntimeMXBean();
}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/ClearSnapshotTest.java b/test/distributed/org/apache/cassandra/distributed/test/ClearSnapshotTest.java
index 3a1ba4edd5..384e79d468 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/ClearSnapshotTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/ClearSnapshotTest.java
@@ -100,7 +100,7 @@ public class ClearSnapshotTest extends TestBaseImpl
long activeRepairs;
do
{
- activeRepairs = cluster.get(1).callOnInstance(() -> ActiveRepairService.instance.parentRepairSessionCount());
+ activeRepairs = cluster.get(1).callOnInstance(() -> ActiveRepairService.instance().parentRepairSessionCount());
Thread.sleep(50);
}
while (activeRepairs < 35);
diff --git a/test/distributed/org/apache/cassandra/distributed/test/IncRepairAdminTest.java b/test/distributed/org/apache/cassandra/distributed/test/IncRepairAdminTest.java
index b902632674..f376c4b4d2 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/IncRepairAdminTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/IncRepairAdminTest.java
@@ -149,14 +149,14 @@ public class IncRepairAdminTest extends TestBaseImpl
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore("tbl");
Range<Token> range = new Range<>(cfs.metadata().partitioner.getMinimumToken(),
cfs.metadata().partitioner.getRandomToken());
- ActiveRepairService.instance.registerParentRepairSession(sessionId,
- InetAddressAndPort.getByAddress(coordinator.getAddress()),
- Lists.newArrayList(cfs),
- Sets.newHashSet(range),
- true,
- currentTimeMillis(),
- true,
- PreviewKind.NONE);
+ ActiveRepairService.instance().registerParentRepairSession(sessionId,
+ InetAddressAndPort.getByAddress(coordinator.getAddress()),
+ Lists.newArrayList(cfs),
+ Sets.newHashSet(range),
+ true,
+ currentTimeMillis(),
+ true,
+ PreviewKind.NONE);
LocalSessionAccessor.prepareUnsafe(sessionId,
InetAddressAndPort.getByAddress(coordinator.getAddress()),
participants.stream().map(participant -> InetAddressAndPort.getByAddress(participant.getAddress())).collect(Collectors.toSet()));
diff --git a/test/distributed/org/apache/cassandra/distributed/test/IncRepairCoordinatorErrorTest.java b/test/distributed/org/apache/cassandra/distributed/test/IncRepairCoordinatorErrorTest.java
index 447cd822e2..06ac200be5 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/IncRepairCoordinatorErrorTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/IncRepairCoordinatorErrorTest.java
@@ -48,7 +48,7 @@ public class IncRepairCoordinatorErrorTest extends TestBaseImpl
cluster.get(1).nodetoolResult("repair", KEYSPACE).asserts().success();
TimeUUID result = (TimeUUID) cluster.get(1).executeInternal("select parent_id from system_distributed.repair_history")[0][0];
cluster.get(3).runOnInstance(() -> {
- ActiveRepairService.instance.failSession(result.toString(), true);
+ ActiveRepairService.instance().failSession(result.toString(), true);
});
assertThat(cluster.get(1).logs().watchFor("Can't transition endpoints .* to FAILED").getResult()).isNotEmpty();
}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/OptimiseStreamsRepairTest.java b/test/distributed/org/apache/cassandra/distributed/test/OptimiseStreamsRepairTest.java
index 14d49dc0ca..1febbdbff7 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/OptimiseStreamsRepairTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/OptimiseStreamsRepairTest.java
@@ -39,6 +39,7 @@ import net.bytebuddy.ByteBuddy;
import net.bytebuddy.dynamic.loading.ClassLoadingStrategy;
import net.bytebuddy.implementation.MethodDelegation;
import net.bytebuddy.implementation.bind.annotation.SuperCall;
+import org.apache.cassandra.repair.SharedContext;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.distributed.Cluster;
@@ -114,7 +115,8 @@ public class OptimiseStreamsRepairTest extends TestBaseImpl
.load(cl, ClassLoadingStrategy.Default.INJECTION);
}
- public static List<SyncTask> createOptimisedSyncingSyncTasks(RepairJobDesc desc,
+ public static List<SyncTask> createOptimisedSyncingSyncTasks(SharedContext ctx,
+ RepairJobDesc desc,
List<TreeResponse> trees,
InetAddressAndPort local,
Predicate<InetAddressAndPort> isTransient,
diff --git a/test/distributed/org/apache/cassandra/distributed/test/PaxosRepair2Test.java b/test/distributed/org/apache/cassandra/distributed/test/PaxosRepair2Test.java
index 47932b3c73..a150a33a95 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/PaxosRepair2Test.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/PaxosRepair2Test.java
@@ -161,7 +161,7 @@ public class PaxosRepair2Test extends TestBaseImpl
{
throw new AssertionError(e);
}
- Pair<ActiveRepairService.ParentRepairStatus, List<String>> status = ActiveRepairService.instance.getRepairStatus(cmd);
+ Pair<ActiveRepairService.ParentRepairStatus, List<String>> status = ActiveRepairService.instance().getRepairStatus(cmd);
if (status == null)
continue;
diff --git a/test/distributed/org/apache/cassandra/distributed/test/PaxosRepairTest.java b/test/distributed/org/apache/cassandra/distributed/test/PaxosRepairTest.java
index 3251f84ee5..dd1aa07e20 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/PaxosRepairTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/PaxosRepairTest.java
@@ -175,7 +175,7 @@ public class PaxosRepairTest extends TestBaseImpl
{
throw new AssertionError(e);
}
- Pair<ActiveRepairService.ParentRepairStatus, List<String>> status = ActiveRepairService.instance.getRepairStatus(cmd);
+ Pair<ActiveRepairService.ParentRepairStatus, List<String>> status = ActiveRepairService.instance().getRepairStatus(cmd);
if (status == null)
continue;
diff --git a/test/distributed/org/apache/cassandra/distributed/test/RepairErrorsTest.java b/test/distributed/org/apache/cassandra/distributed/test/RepairErrorsTest.java
index f34bb2a79f..537599c297 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/RepairErrorsTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/RepairErrorsTest.java
@@ -201,7 +201,7 @@ public class RepairErrorsTest extends TestBaseImpl
private void assertNoActiveRepairSessions(IInvokableInstance instance)
{
// Make sure we've cleaned up local sessions:
- Integer sessions = instance.callOnInstance(() -> ActiveRepairService.instance.sessionCount());
+ Integer sessions = instance.callOnInstance(() -> ActiveRepairService.instance().sessionCount());
assertEquals(0, sessions.intValue());
}
diff --git a/test/unit/accord/utils/DefaultRandom.java b/test/unit/accord/utils/DefaultRandom.java
new file mode 100644
index 0000000000..b16f1f8bbf
--- /dev/null
+++ b/test/unit/accord/utils/DefaultRandom.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package accord.utils;
+
+import java.util.Random;
+
+public class DefaultRandom implements RandomSource
+{
+ private final Random delegate;
+ public DefaultRandom()
+ {
+ this.delegate = new Random();
+ }
+
+ public DefaultRandom(long seed)
+ {
+ this.delegate = new Random(seed);
+ }
+
+ @Override
+ public void nextBytes(byte[] bytes)
+ {
+ delegate.nextBytes(bytes);
+ }
+
+ @Override
+ public boolean nextBoolean()
+ {
+ return delegate.nextBoolean();
+ }
+
+ @Override
+ public int nextInt()
+ {
+ return delegate.nextInt();
+ }
+
+ @Override
+ public long nextLong()
+ {
+ return delegate.nextLong();
+ }
+
+ @Override
+ public float nextFloat()
+ {
+ return delegate.nextFloat();
+ }
+
+ @Override
+ public double nextDouble()
+ {
+ return delegate.nextDouble();
+ }
+
+ @Override
+ public double nextGaussian()
+ {
+ return delegate.nextGaussian();
+ }
+
+ @Override
+ public void setSeed(long seed)
+ {
+ delegate.setSeed(seed);
+ }
+
+ @Override
+ public DefaultRandom fork() {
+ return new DefaultRandom(nextLong());
+ }
+
+ @Override
+ public Random asJdkRandom()
+ {
+ return delegate;
+ }
+}
diff --git a/test/unit/accord/utils/Gen.java b/test/unit/accord/utils/Gen.java
new file mode 100644
index 0000000000..7563e8c29b
--- /dev/null
+++ b/test/unit/accord/utils/Gen.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package accord.utils;
+
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import java.util.function.IntPredicate;
+import java.util.function.IntSupplier;
+import java.util.function.LongPredicate;
+import java.util.function.LongSupplier;
+import java.util.function.Predicate;
+import java.util.function.Supplier;
+import java.util.function.ToIntFunction;
+import java.util.function.ToLongFunction;
+import java.util.stream.IntStream;
+import java.util.stream.LongStream;
+import java.util.stream.Stream;
+
+public interface Gen<A> {
+ /**
+ * For cases where method handles isn't able to detect the proper type, this method acts as a cast
+ * to inform the compiler of the desired type.
+ */
+ static <A> Gen<A> of(Gen<A> fn)
+ {
+ return fn;
+ }
+
+ A next(RandomSource random);
+
+ default <B> Gen<B> map(Function<? super A, ? extends B> fn)
+ {
+ return r -> fn.apply(this.next(r));
+ }
+
+ default <B> Gen<B> map(BiFunction<RandomSource, ? super A, ? extends B> fn)
+ {
+ return r -> fn.apply(r, this.next(r));
+ }
+
+ default IntGen mapToInt(ToIntFunction<A> fn)
+ {
+ return r -> fn.applyAsInt(next(r));
+ }
+
+ default LongGen mapToLong(ToLongFunction<A> fn)
+ {
+ return r -> fn.applyAsLong(next(r));
+ }
+
+ default <B> Gen<B> flatMap(Function<? super A, Gen<? extends B>> mapper)
+ {
+ return rs -> mapper.apply(this.next(rs)).next(rs);
+ }
+
+ default <B> Gen<B> flatMap(BiFunction<RandomSource, ? super A, Gen<? extends B>> mapper)
+ {
+ return rs -> mapper.apply(rs, this.next(rs)).next(rs);
+ }
+
+ default Gen<A> filter(Predicate<A> fn)
+ {
+ Gen<A> self = this;
+ return r -> {
+ A value;
+ do {
+ value = self.next(r);
+ }
+ while (!fn.test(value));
+ return value;
+ };
+ }
+
+ default Supplier<A> asSupplier(RandomSource rs)
+ {
+ return () -> next(rs);
+ }
+
+ default Stream<A> asStream(RandomSource rs)
+ {
+ return Stream.generate(() -> next(rs));
+ }
+
+ interface IntGen extends Gen<Integer>
+ {
+ int nextInt(RandomSource random);
+
+ @Override
+ default Integer next(RandomSource random)
+ {
+ return nextInt(random);
+ }
+
+ default Gen.IntGen filterInt(IntPredicate fn)
+ {
+ return rs -> {
+ int value;
+ do
+ {
+ value = nextInt(rs);
+ }
+ while (!fn.test(value));
+ return value;
+ };
+ }
+
+ @Override
+ default Gen.IntGen filter(Predicate<Integer> fn)
+ {
+ return filterInt(i -> fn.test(i));
+ }
+
+ default IntSupplier asIntSupplier(RandomSource rs)
+ {
+ return () -> nextInt(rs);
+ }
+
+ default IntStream asIntStream(RandomSource rs)
+ {
+ return IntStream.generate(() -> nextInt(rs));
+ }
+ }
+
+ interface LongGen extends Gen<Long>
+ {
+ long nextLong(RandomSource random);
+
+ @Override
+ default Long next(RandomSource random)
+ {
+ return nextLong(random);
+ }
+
+ default Gen.LongGen filterLong(LongPredicate fn)
+ {
+ return rs -> {
+ long value;
+ do
+ {
+ value = nextLong(rs);
+ }
+ while (!fn.test(value));
+ return value;
+ };
+ }
+
+ @Override
+ default Gen.LongGen filter(Predicate<Long> fn)
+ {
+ return filterLong(i -> fn.test(i));
+ }
+
+ default LongSupplier asLongSupplier(RandomSource rs)
+ {
+ return () -> nextLong(rs);
+ }
+
+ default LongStream asLongStream(RandomSource rs)
+ {
+ return LongStream.generate(() -> nextLong(rs));
+ }
+ }
+}
diff --git a/test/unit/accord/utils/Gens.java b/test/unit/accord/utils/Gens.java
new file mode 100644
index 0000000000..4a80d724af
--- /dev/null
+++ b/test/unit/accord/utils/Gens.java
@@ -0,0 +1,576 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package accord.utils;
+
+import java.lang.reflect.Array;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.EnumMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableSet;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
+
+public class Gens {
+ private Gens() {
+ }
+
+ public static <T> Gen<T> constant(T constant)
+ {
+ return ignore -> constant;
+ }
+
+ public static <T> Gen<T> constant(Supplier<T> constant)
+ {
+ return ignore -> constant.get();
+ }
+
+ public static <T> Gen<T> pick(T... ts)
+ {
+ return pick(Arrays.asList(ts));
+ }
+
+ public static <T> Gen<T> pick(List<T> ts)
+ {
+ Gen.IntGen offset = ints().between(0, ts.size() - 1);
+ return rs -> ts.get(offset.nextInt(rs));
+ }
+
+ public static <T extends Comparable<T>> Gen<T> pick(Set<T> set)
+ {
+ List<T> list = new ArrayList<>(set);
+ // Non-ordered sets may have different iteration order on different environments, which would make a seed produce different histories!
+ // To avoid such a problem, make sure to apply a deterministic function (sort).
+ if (!(set instanceof NavigableSet))
+ list.sort(Comparator.naturalOrder());
+ return pick(list);
+ }
+
+ public static <T> Gen<T> pick(Map<T, Integer> values)
+ {
+ if (values == null || values.isEmpty())
+ throw new IllegalArgumentException("values is empty");
+ double totalWeight = values.values().stream().mapToDouble(Integer::intValue).sum();
+ List<Weight<T>> list = values.entrySet().stream().map(e -> new Weight<>(e.getKey(), e.getValue())).collect(Collectors.toList());
+ Collections.sort(list);
+ return rs -> {
+ double value = rs.nextDouble() * totalWeight;
+ for (Weight<T> w : list)
+ {
+ value -= w.weight;
+ if (value <= 0)
+ return w.value;
+ }
+ return list.get(list.size() - 1).value;
+ };
+ }
+
+ public static Gen<char[]> charArray(Gen.IntGen sizes, char[] domain)
+ {
+ return charArray(sizes, domain, (a, b) -> true);
+ }
+
+ public interface IntCharBiPredicate
+ {
+ boolean test(int a, char b);
+ }
+
+ public static Gen<char[]> charArray(Gen.IntGen sizes, char[] domain, IntCharBiPredicate fn)
+ {
+ Gen.IntGen indexGen = ints().between(0, domain.length - 1);
+ return rs -> {
+ int size = sizes.nextInt(rs);
+ char[] is = new char[size];
+ for (int i = 0; i != size; i++)
+ {
+ char c;
+ do
+ {
+ c = domain[indexGen.nextInt(rs)];
+ }
+ while (!fn.test(i, c));
+ is[i] = c;
+ }
+ return is;
+ };
+ }
+
+ public static Gen<RandomSource> random() {
+ return r -> r;
+ }
+
+ public static BooleanDSL bools()
+ {
+ return new BooleanDSL();
+ }
+
+ public static IntDSL ints()
+ {
+ return new IntDSL();
+ }
+
+ public static LongDSL longs() {
+ return new LongDSL();
+ }
+
+ public static <T> ListDSL<T> lists(Gen<T> fn) {
+ return new ListDSL<>(fn);
+ }
+
+ public static <T> ArrayDSL<T> arrays(Class<T> type, Gen<T> fn) {
+ return new ArrayDSL<>(type, fn);
+ }
+
+ public static IntArrayDSL arrays(Gen.IntGen fn) {
+ return new IntArrayDSL(fn);
+ }
+
+ public static LongArrayDSL arrays(Gen.LongGen fn) {
+ return new LongArrayDSL(fn);
+ }
+
+ public static EnumDSL enums()
+ {
+ return new EnumDSL();
+ }
+
+ public static StringDSL strings()
+ {
+ return new StringDSL();
+ }
+
+ public static class BooleanDSL
+ {
+ public Gen<Boolean> all()
+ {
+ return RandomSource::nextBoolean;
+ }
+
+ public Gen<Boolean> runs(double ratio, int maxRuns)
+ {
+ Invariants.checkArgument(ratio > 0 && ratio <= 1, "Expected %d to be larger than 0 and <= 1", ratio);
+ double lower = ratio * .8;
+ double upper = ratio * 1.2;
+ return new Gen<>() {
+ // run represents how many consecutaive true values should be returned; -1 implies no active "run" exists
+ private int run = -1;
+ private long falseCount = 0, trueCount = 0;
+ @Override
+ public Boolean next(RandomSource rs)
+ {
+ if (run != -1)
+ {
+ run--;
+ trueCount++;
+ return true;
+ }
+ double currentRatio = trueCount / (double) (falseCount + trueCount);
+ if (currentRatio < lower)
+ {
+ // not enough true
+ trueCount++;
+ return true;
+ }
+ if (currentRatio > upper)
+ {
+ // not enough false
+ falseCount++;
+ return false;
+ }
+ if (rs.decide(ratio))
+ {
+ run = rs.nextInt(maxRuns);
+ run--;
+ trueCount++;
+ return true;
+ }
+ falseCount++;
+ return false;
+ }
+ };
+ }
+ }
+
+ public static class IntDSL
+ {
+ public Gen.IntGen of(int value)
+ {
+ return r -> value;
+ }
+
+ public Gen.IntGen all()
+ {
+ return RandomSource::nextInt;
+ }
+
+ public Gen.IntGen between(int min, int max)
+ {
+ Invariants.checkArgument(max >= min, "max (%d) < min (%d)", max, min);
+ if (min == max)
+ return of(min);
+ // since bounds is exclusive, if max == max_value unable to do +1 to include... so will return a gen
+ // that does not include
+ if (max == Integer.MAX_VALUE)
+ return r -> r.nextInt(min, max);
+ return r -> r.nextInt(min, max + 1);
+ }
+ }
+
+ public static class LongDSL {
+ public Gen.LongGen of(long value)
+ {
+ return r -> value;
+ }
+
+ public Gen.LongGen all() {
+ return RandomSource::nextLong;
+ }
+
+ public Gen.LongGen between(long min, long max) {
+ Invariants.checkArgument(max >= min);
+ if (min == max)
+ return of(min);
+ // since bounds is exclusive, if max == max_value unable to do +1 to include... so will return a gen
+ // that does not include
+ if (max == Long.MAX_VALUE)
+ return r -> r.nextLong(min, max);
+ return r -> r.nextLong(min, max + 1);
+ }
+ }
+
+ public static class EnumDSL
+ {
+ public <T extends Enum<T>> Gen<T> all(Class<T> klass)
+ {
+ return pick(klass.getEnumConstants());
+ }
+
+ public <T extends Enum<T>> Gen<T> allWithWeights(Class<T> klass, int... weights)
+ {
+ T[] constants = klass.getEnumConstants();
+ if (constants.length != weights.length)
+ throw new IllegalArgumentException(String.format("Total number of weights (%s) does not match the enum (%s)", Arrays.toString(weights), Arrays.toString(constants)));
+ Map<T, Integer> values = new EnumMap<>(klass);
+ for (int i = 0; i < constants.length; i++)
+ values.put(constants[i], weights[i]);
+ return pick(values);
+ }
+ }
+
+ public static class StringDSL
+ {
+ public Gen<String> of(Gen.IntGen sizes, char[] domain)
+ {
+ // note, map is overloaded so String::new is ambugious to javac, so need a lambda here
+ return charArray(sizes, domain).map(c -> new String(c));
+ }
+
+ public SizeBuilder<String> of(char[] domain)
+ {
+ return new SizeBuilder<>(sizes -> of(sizes, domain));
+ }
+
+ public Gen<String> of(Gen.IntGen sizes, char[] domain, IntCharBiPredicate fn)
+ {
+ // note, map is overloaded so String::new is ambugious to javac, so need a lambda here
+ return charArray(sizes, domain, fn).map(c -> new String(c));
+ }
+
+ public SizeBuilder<String> of(char[] domain, IntCharBiPredicate fn)
+ {
+ return new SizeBuilder<>(sizes -> of(sizes, domain, fn));
+ }
+
+ public Gen<String> all(Gen.IntGen sizes)
+ {
+ return betweenCodePoints(sizes, Character.MIN_CODE_POINT, Character.MAX_CODE_POINT);
+ }
+
+ public SizeBuilder<String> all()
+ {
+ return new SizeBuilder<>(this::all);
+ }
+
+ public Gen<String> ascii(Gen.IntGen sizes)
+ {
+ return betweenCodePoints(sizes, 0, 127);
+ }
+
+ public SizeBuilder<String> ascii()
+ {
+ return new SizeBuilder<>(this::ascii);
+ }
+
+ public Gen<String> betweenCodePoints(Gen.IntGen sizes, int min, int max)
+ {
+ Gen.IntGen codePointGen = ints().between(min, max).filter(Character::isDefined);
+ return rs -> {
+ int[] array = new int[sizes.nextInt(rs)];
+ for (int i = 0; i < array.length; i++)
+ array[i] = codePointGen.nextInt(rs);
+ return new String(array, 0, array.length);
+ };
+ }
+
+ public SizeBuilder<String> betweenCodePoints(int min, int max)
+ {
+ return new SizeBuilder<>(sizes -> betweenCodePoints(sizes, min, max));
+ }
+ }
+
+ public static class SizeBuilder<T>
+ {
+ private final Function<Gen.IntGen, Gen<T>> fn;
+
+ public SizeBuilder(Function<Gen.IntGen, Gen<T>> fn)
+ {
+ this.fn = fn;
+ }
+
+ public Gen<T> ofLength(int fixed)
+ {
+ return ofLengthBetween(fixed, fixed);
+ }
+
+ public Gen<T> ofLengthBetween(int min, int max)
+ {
+ return fn.apply(ints().between(min, max));
+ }
+ }
+
+ public static class ListDSL<T> implements BaseSequenceDSL<ListDSL<T>, List<T>> {
+ private final Gen<T> fn;
+
+ public ListDSL(Gen<T> fn) {
+ this.fn = Objects.requireNonNull(fn);
+ }
+
+ @Override
+ public ListDSL<T> unique()
+ {
+ return new ListDSL<>(new GenReset<>(fn));
+ }
+
+ @Override
+ public Gen<List<T>> ofSizeBetween(int minSize, int maxSize) {
+ Gen.IntGen sizeGen = ints().between(minSize, maxSize);
+ return r ->
+ {
+ Reset.tryReset(fn);
+ int size = sizeGen.nextInt(r);
+ List<T> list = new ArrayList<>(size);
+ for (int i = 0; i < size; i++)
+ list.add(fn.next(r));
+ return list;
+ };
+ }
+ }
+
+ public static class ArrayDSL<T> implements BaseSequenceDSL<ArrayDSL<T>, T[]> {
+ private final Class<T> type;
+ private final Gen<T> fn;
+
+ public ArrayDSL(Class<T> type, Gen<T> fn) {
+ this.type = Objects.requireNonNull(type);
+ this.fn = Objects.requireNonNull(fn);
+ }
+
+ @Override
+ public ArrayDSL<T> unique()
+ {
+ return new ArrayDSL<>(type, new GenReset<>(fn));
+ }
+
+ @Override
+ public Gen<T[]> ofSizeBetween(int minSize, int maxSize) {
+ Gen.IntGen sizeGen = ints().between(minSize, maxSize);
+ return r ->
+ {
+ Reset.tryReset(fn);
+ int size = sizeGen.nextInt(r);
+ T[] list = (T[]) Array.newInstance(type, size);
+ for (int i = 0; i < size; i++)
+ list[i] = fn.next(r);
+ return list;
+ };
+ }
+ }
+
+ public static class IntArrayDSL implements BaseSequenceDSL<IntArrayDSL, int[]> {
+ private final Gen.IntGen fn;
+
+ public IntArrayDSL(Gen.IntGen fn) {
+ this.fn = Objects.requireNonNull(fn);
+ }
+
+ @Override
+ public IntArrayDSL unique()
+ {
+ return new IntArrayDSL(new IntGenReset(fn));
+ }
+
+ @Override
+ public Gen<int[]> ofSizeBetween(int minSize, int maxSize) {
+ Gen.IntGen sizeGen = ints().between(minSize, maxSize);
+ return r ->
+ {
+ int size = sizeGen.nextInt(r);
+ int[] list = new int[size];
+ for (int i = 0; i < size; i++)
+ list[i] = fn.nextInt(r);
+ return list;
+ };
+ }
+ }
+
+ public static class LongArrayDSL implements BaseSequenceDSL<LongArrayDSL, long[]> {
+ private final Gen.LongGen fn;
+
+ public LongArrayDSL(Gen.LongGen fn) {
+ this.fn = Objects.requireNonNull(fn);
+ }
+
+ @Override
+ public LongArrayDSL unique()
+ {
+ return new LongArrayDSL(new LongGenReset(fn));
+ }
+
+ @Override
+ public Gen<long[]> ofSizeBetween(int minSize, int maxSize) {
+ Gen.IntGen sizeGen = ints().between(minSize, maxSize);
+ return r ->
+ {
+ int size = sizeGen.nextInt(r);
+ long[] list = new long[size];
+ for (int i = 0; i < size; i++)
+ list[i] = fn.nextLong(r);
+ return list;
+ };
+ }
+ }
+
+ public interface BaseSequenceDSL<A extends BaseSequenceDSL<A, B>, B>
+ {
+ A unique();
+
+ Gen<B> ofSizeBetween(int min, int max);
+
+ default Gen<B> ofSize(int size) {
+ return ofSizeBetween(size, size);
+ }
+ }
+
+ private interface Reset {
+ static void tryReset(Object o)
+ {
+ if (o instanceof Reset)
+ ((Reset) o).reset();
+ }
+
+ void reset();
+ }
+
+ private static class GenReset<T> implements Gen<T>, Reset
+ {
+ private final Set<T> seen = new HashSet<>();
+ private final Gen<T> fn;
+
+ private GenReset(Gen<T> fn)
+ {
+ this.fn = fn;
+ }
+
+ @Override
+ public T next(RandomSource random)
+ {
+ T value;
+ while (!seen.add((value = fn.next(random)))) {}
+ return value;
+ }
+
+ @Override
+ public void reset()
+ {
+ seen.clear();
+ }
+ }
+
+ private static class IntGenReset implements Gen.IntGen, Reset
+ {
+ private final GenReset<Integer> base;
+
+ private IntGenReset(Gen.IntGen fn)
+ {
+ this.base = new GenReset<>(fn);
+ }
+ @Override
+ public int nextInt(RandomSource random) {
+ return base.next(random);
+ }
+
+ @Override
+ public void reset() {
+ base.reset();
+ }
+ }
+
+ private static class LongGenReset implements Gen.LongGen, Reset
+ {
+ private final GenReset<Long> base;
+
+ private LongGenReset(Gen.LongGen fn)
+ {
+ this.base = new GenReset<>(fn);
+ }
+ @Override
+ public long nextLong(RandomSource random) {
+ return base.next(random);
+ }
+
+ @Override
+ public void reset() {
+ base.reset();
+ }
+ }
+
+ private static class Weight<T> implements Comparable<Weight<T>>
+ {
+ private final T value;
+ private final double weight;
+
+ private Weight(T value, double weight) {
+ this.value = value;
+ this.weight = weight;
+ }
+
+ @Override
+ public int compareTo(Weight<T> o) {
+ return Double.compare(weight, o.weight);
+ }
+ }
+}
diff --git a/test/unit/accord/utils/Invariants.java b/test/unit/accord/utils/Invariants.java
new file mode 100644
index 0000000000..b960d8b268
--- /dev/null
+++ b/test/unit/accord/utils/Invariants.java
@@ -0,0 +1,327 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package accord.utils;
+
+import net.nicoulaj.compilecommand.annotations.Inline;
+
+import javax.annotation.Nullable;
+import java.util.function.Predicate;
+
+import static java.lang.String.format;
+
+public class Invariants
+{
+ private static final boolean PARANOID = true;
+ private static final boolean DEBUG = true;
+
+ public static boolean isParanoid()
+ {
+ return PARANOID;
+ }
+ public static boolean debug()
+ {
+ return DEBUG;
+ }
+
+ private static void illegalState(String msg)
+ {
+ throw new IllegalStateException(msg);
+ }
+
+ private static void illegalState()
+ {
+ illegalState(null);
+ }
+
+ private static void illegalArgument(String msg)
+ {
+ throw new IllegalArgumentException(msg);
+ }
+
+
+ private static void illegalArgument()
+ {
+ illegalArgument(null);
+ }
+
+ public static <T1, T2 extends T1> T2 checkType(T1 cast)
+ {
+ return (T2)cast;
+ }
+
+ public static <T1, T2 extends T1> T2 checkType(Class<T2> to, T1 cast)
+ {
+ if (cast != null && !to.isInstance(cast))
+ illegalState();
+ return (T2)cast;
+ }
+
+ public static <T1, T2 extends T1> T2 checkType(Class<T2> to, T1 cast, String msg)
+ {
+ if (cast != null && !to.isInstance(cast))
+ illegalState(msg);
+ return (T2)cast;
+ }
+
+ public static void paranoid(boolean condition)
+ {
+ if (PARANOID && !condition)
+ illegalState();
+ }
+
+ public static void checkState(boolean condition)
+ {
+ if (!condition)
+ illegalState();
+ }
+
+ public static void checkState(boolean condition, String msg)
+ {
+ if (!condition)
+ illegalState(msg);
+ }
+
+ public static void checkState(boolean condition, String fmt, int p1)
+ {
+ if (!condition)
+ illegalState(format(fmt, p1));
+ }
+
+ public static void checkState(boolean condition, String fmt, int p1, int p2)
+ {
+ if (!condition)
+ illegalState(format(fmt, p1, p2));
+ }
+
+ public static void checkState(boolean condition, String fmt, long p1)
+ {
+ if (!condition)
+ illegalState(format(fmt, p1));
+ }
+
+ public static void checkState(boolean condition, String fmt, long p1, long p2)
+ {
+ if (!condition)
+ illegalState(format(fmt, p1, p2));
+ }
+
+ public static void checkState(boolean condition, String fmt, @Nullable Object p1)
+ {
+ if (!condition)
+ illegalState(format(fmt, p1));
+ }
+
+ public static void checkState(boolean condition, String fmt, @Nullable Object p1, @Nullable Object p2)
+ {
+ if (!condition)
+ illegalState(format(fmt, p1, p2));
+ }
+
+ public static void checkState(boolean condition, String fmt, Object... args)
+ {
+ if (!condition)
+ illegalState(format(fmt, args));
+ }
+
+ public static <T> T nonNull(T param)
+ {
+ if (param == null)
+ throw new NullPointerException();
+ return param;
+ }
+
+ public static <T> T nonNull(T param, String fmt, Object... args)
+ {
+ if (param == null)
+ throw new NullPointerException(format(fmt, args));
+ return param;
+ }
+
+ public static int isNatural(int input)
+ {
+ if (input < 0)
+ illegalState();
+ return input;
+ }
+
+ public static long isNatural(long input)
+ {
+ if (input < 0)
+ illegalState();
+ return input;
+ }
+
+ public static void checkArgument(boolean condition)
+ {
+ if (!condition)
+ illegalArgument();
+ }
+
+ public static void checkArgument(boolean condition, String msg)
+ {
+ if (!condition)
+ illegalArgument(msg);
+ }
+
+ public static void checkArgument(boolean condition, String fmt, int p1)
+ {
+ if (!condition)
+ illegalArgument(format(fmt, p1));
+ }
+
+ public static void checkArgument(boolean condition, String fmt, int p1, int p2)
+ {
+ if (!condition)
+ illegalArgument(format(fmt, p1, p2));
+ }
+
+ public static void checkArgument(boolean condition, String fmt, long p1)
+ {
+ if (!condition)
+ illegalArgument(format(fmt, p1));
+ }
+
+ public static void checkArgument(boolean condition, String fmt, long p1, long p2)
+ {
+ if (!condition)
+ illegalArgument(format(fmt, p1, p2));
+ }
+
+ public static void checkArgument(boolean condition, String fmt, @Nullable Object p1)
+ {
+ if (!condition)
+ illegalArgument(format(fmt, p1));
+ }
+
+ public static void checkArgument(boolean condition, String fmt, @Nullable Object p1, @Nullable Object p2)
+ {
+ if (!condition)
+ illegalArgument(format(fmt, p1, p2));
+ }
+
+ public static void checkArgument(boolean condition, String fmt, Object... args)
+ {
+ if (!condition)
+ illegalArgument(format(fmt, args));
+ }
+
+ public static <T> T checkArgument(T param, boolean condition)
+ {
+ if (!condition)
+ illegalArgument();
+ return param;
+ }
+
+ public static <T> T checkArgument(T param, boolean condition, String msg)
+ {
+ if (!condition)
+ illegalArgument(msg);
+ return param;
+ }
+
+ public static <T> T checkArgument(T param, boolean condition, String fmt, int p1)
+ {
+ if (!condition)
+ illegalArgument(format(fmt, p1));
+ return param;
+ }
+
+ public static <T> T checkArgument(T param, boolean condition, String fmt, int p1, int p2)
+ {
+ if (!condition)
+ illegalArgument(format(fmt, p1, p2));
+ return param;
+ }
+
+ public static <T> T checkArgument(T param, boolean condition, String fmt, long p1)
+ {
+ if (!condition)
+ illegalArgument(format(fmt, p1));
+ return param;
+ }
+
+ public static <T> T checkArgument(T param, boolean condition, String fmt, long p1, long p2)
+ {
+ if (!condition)
+ illegalArgument(format(fmt, p1, p2));
+ return param;
+ }
+
+ public static <T> T checkArgument(T param, boolean condition, String fmt, @Nullable Object p1)
+ {
+ if (!condition)
+ illegalArgument(format(fmt, p1));
+ return param;
+ }
+
+ public static <T> T checkArgument(T param, boolean condition, String fmt, @Nullable Object p1, @Nullable Object p2)
+ {
+ if (!condition)
+ illegalArgument(format(fmt, p1, p2));
+ return param;
+ }
+
+ public static <T> T checkArgument(T param, boolean condition, String fmt, Object... args)
+ {
+ if (!condition)
+ illegalArgument(format(fmt, args));
+ return param;
+ }
+
+ @Inline
+ public static <T> T checkArgument(T param, Predicate<T> condition)
+ {
+ if (!condition.test(param))
+ illegalArgument();
+ return param;
+ }
+
+ @Inline
+ public static <T> T checkArgument(T param, Predicate<T> condition, String msg)
+ {
+ if (!condition.test(param))
+ illegalArgument(msg);
+ return param;
+ }
+
+ public static <O> O cast(Object o, Class<O> klass)
+ {
+ try
+ {
+ return klass.cast(o);
+ }
+ catch (ClassCastException e)
+ {
+ throw new IllegalArgumentException(format("Unable to cast %s to %s", o, klass.getName()));
+ }
+ }
+
+ public static void checkIndexInBounds(int realLength, int offset, int length)
+ {
+ if (realLength == 0 || length == 0)
+ throw new IndexOutOfBoundsException("Unable to access offset " + offset + "; empty");
+ if (offset < 0)
+ throw new IndexOutOfBoundsException("Offset " + offset + " must not be negative");
+ if (length < 0)
+ throw new IndexOutOfBoundsException("Length " + length + " must not be negative");
+ int endOffset = offset + length;
+ if (endOffset > realLength)
+ throw new IndexOutOfBoundsException(String.format("Offset %d, length = %d; real length was %d", offset, length, realLength));
+ }
+}
diff --git a/test/unit/accord/utils/Property.java b/test/unit/accord/utils/Property.java
new file mode 100644
index 0000000000..b90472c862
--- /dev/null
+++ b/test/unit/accord/utils/Property.java
@@ -0,0 +1,382 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package accord.utils;
+
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.Objects;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.transport.ProtocolException;
+import org.apache.cassandra.utils.concurrent.AsyncPromise;
+
+public class Property
+{
+ public static abstract class Common<T extends Common<T>>
+ {
+ protected long seed = ThreadLocalRandom.current().nextLong();
+ protected int examples = 1000;
+
+ protected boolean pure = true;
+ @Nullable
+ protected Duration timeout = null;
+
+ protected Common() {
+ }
+
+ protected Common(Common<?> other) {
+ this.seed = other.seed;
+ this.examples = other.examples;
+ this.pure = other.pure;
+ this.timeout = other.timeout;
+ }
+
+ public T withSeed(long seed)
+ {
+ this.seed = seed;
+ return (T) this;
+ }
+
+ public T withExamples(int examples)
+ {
+ if (examples <= 0)
+ throw new IllegalArgumentException("Examples must be positive");
+ this.examples = examples;
+ return (T) this;
+ }
+
+ public T withPure(boolean pure)
+ {
+ this.pure = pure;
+ return (T) this;
+ }
+
+ public T withTimeout(Duration timeout)
+ {
+ this.timeout = timeout;
+ return (T) this;
+ }
+
+ protected void checkWithTimeout(Runnable fn)
+ {
+ AsyncPromise<?> promise = new AsyncPromise<>();
+ Thread t = new Thread(() -> {
+ try
+ {
+ fn.run();
+ promise.setSuccess(null);
+ }
+ catch (Throwable e)
+ {
+ promise.setFailure(e);
+ }
+ });
+ t.setName("property with timeout");
+ t.setDaemon(true);
+ try
+ {
+ t.start();
+ promise.get(timeout.toNanos(), TimeUnit.NANOSECONDS);
+ }
+ catch (ExecutionException e)
+ {
+ throw new ProtocolException(propertyError(this, e.getCause()));
+ }
+ catch (InterruptedException e)
+ {
+ t.interrupt();
+ throw new PropertyError(propertyError(this, e));
+ }
+ catch (TimeoutException e)
+ {
+ t.interrupt();
+ TimeoutException override = new TimeoutException("property test did not complete within " + this.timeout);
+ override.setStackTrace(new StackTraceElement[0]);
+ throw new PropertyError(propertyError(this, override));
+ }
+ }
+ }
+
+ public static class ForBuilder extends Common<ForBuilder>
+ {
+ public void check(FailingConsumer<RandomSource> fn)
+ {
+ forAll(Gens.random()).check(fn);
+ }
+
+ public <T> SingleBuilder<T> forAll(Gen<T> gen)
+ {
+ return new SingleBuilder<>(gen, this);
+ }
+
+ public <A, B> DoubleBuilder<A, B> forAll(Gen<A> a, Gen<B> b)
+ {
+ return new DoubleBuilder<>(a, b, this);
+ }
+
+ public <A, B, C> TrippleBuilder<A, B, C> forAll(Gen<A> a, Gen<B> b, Gen<C> c)
+ {
+ return new TrippleBuilder<>(a, b, c, this);
+ }
+ }
+
+ private static Object normalizeValue(Object value)
+ {
+ if (value == null)
+ return null;
+ // one day java arrays will have a useful toString... one day...
+ if (value.getClass().isArray())
+ {
+ Class<?> subType = value.getClass().getComponentType();
+ if (!subType.isPrimitive())
+ return Arrays.asList((Object[]) value);
+ if (Byte.TYPE == subType)
+ return Arrays.toString((byte[]) value);
+ if (Character.TYPE == subType)
+ return Arrays.toString((char[]) value);
+ if (Short.TYPE == subType)
+ return Arrays.toString((short[]) value);
+ if (Integer.TYPE == subType)
+ return Arrays.toString((int[]) value);
+ if (Long.TYPE == subType)
+ return Arrays.toString((long[]) value);
+ if (Float.TYPE == subType)
+ return Arrays.toString((float[]) value);
+ if (Double.TYPE == subType)
+ return Arrays.toString((double[]) value);
+ }
+ try
+ {
+ return value.toString();
+ }
+ catch (Throwable t)
+ {
+ return "Object.toString failed: " + t.getClass().getCanonicalName() + ": " + t.getMessage();
+ }
+ }
+
+ private static String propertyError(Common<?> input, Throwable cause, Object... values)
+ {
+ StringBuilder sb = new StringBuilder();
+ // return "Seed=" + seed + "\nExamples=" + examples;
+ sb.append("Property error detected:\nSeed = ").append(input.seed).append('\n');
+ sb.append("Examples = ").append(input.examples).append('\n');
+ sb.append("Pure = ").append(input.pure).append('\n');
+ if (cause != null)
+ {
+ String msg = cause.getMessage();
+ sb.append("Error: ");
+ // to improve readability, if a newline is detected move the error msg to the next line
+ if (msg != null && msg.contains("\n"))
+ msg = "\n\t" + msg.replace("\n", "\n\t");
+ if (msg == null)
+ msg = cause.getClass().getCanonicalName();
+ sb.append(msg).append('\n');
+ }
+ if (values != null)
+ {
+ sb.append("Values:\n");
+ for (int i = 0; i < values.length; i++)
+ sb.append('\t').append(i).append(" = ").append(normalizeValue(values[i])).append('\n');
+ }
+ return sb.toString();
+ }
+
+ public interface FailingConsumer<A>
+ {
+ void accept(A value) throws Exception;
+ }
+
+ public static class SingleBuilder<T> extends Common<SingleBuilder<T>>
+ {
+ private final Gen<T> gen;
+
+ private SingleBuilder(Gen<T> gen, Common<?> other) {
+ super(other);
+ this.gen = Objects.requireNonNull(gen);
+ }
+
+ public void check(FailingConsumer<T> fn)
+ {
+ if (timeout != null)
+ {
+ checkWithTimeout(() -> checkInternal(fn));
+ return;
+ }
+ checkInternal(fn);
+ }
+
+ private void checkInternal(FailingConsumer<T> fn)
+ {
+ RandomSource random = new DefaultRandom(seed);
+ for (int i = 0; i < examples; i++)
+ {
+ T value = null;
+ try
+ {
+ checkInterrupted();
+ fn.accept(value = gen.next(random));
+ }
+ catch (Throwable t)
+ {
+ throw new PropertyError(propertyError(this, t, value), t);
+ }
+ if (pure)
+ {
+ seed = random.nextLong();
+ random.setSeed(seed);
+ }
+ }
+ }
+ }
+
+ public interface FailingBiConsumer<A, B>
+ {
+ void accept(A a, B b) throws Exception;
+ }
+
+ public static class DoubleBuilder<A, B> extends Common<DoubleBuilder<A, B>>
+ {
+ private final Gen<A> aGen;
+ private final Gen<B> bGen;
+
+ private DoubleBuilder(Gen<A> aGen, Gen<B> bGen, Common<?> other) {
+ super(other);
+ this.aGen = Objects.requireNonNull(aGen);
+ this.bGen = Objects.requireNonNull(bGen);
+ }
+
+ public void check(FailingBiConsumer<A, B> fn)
+ {
+ if (timeout != null)
+ {
+ checkWithTimeout(() -> checkInternal(fn));
+ return;
+ }
+ checkInternal(fn);
+ }
+
+ private void checkInternal(FailingBiConsumer<A, B> fn)
+ {
+ RandomSource random = new DefaultRandom(seed);
+ for (int i = 0; i < examples; i++)
+ {
+ A a = null;
+ B b = null;
+ try
+ {
+ checkInterrupted();
+ fn.accept(a = aGen.next(random), b = bGen.next(random));
+ }
+ catch (Throwable t)
+ {
+ throw new PropertyError(propertyError(this, t, a, b), t);
+ }
+ if (pure)
+ {
+ seed = random.nextLong();
+ random.setSeed(seed);
+ }
+ }
+ }
+ }
+
+ public interface FailingTriConsumer<A, B, C>
+ {
+ void accept(A a, B b, C c) throws Exception;
+ }
+
+ public static class TrippleBuilder<A, B, C> extends Common<TrippleBuilder<A, B, C>>
+ {
+ private final Gen<A> as;
+ private final Gen<B> bs;
+ private final Gen<C> cs;
+
+ public TrippleBuilder(Gen<A> as, Gen<B> bs, Gen<C> cs, Common<?> other)
+ {
+ super(other);
+ this.as = as;
+ this.bs = bs;
+ this.cs = cs;
+ }
+
+ public void check(FailingTriConsumer<A, B, C> fn)
+ {
+ if (timeout != null)
+ {
+ checkWithTimeout(() -> checkInternal(fn));
+ return;
+ }
+ checkInternal(fn);
+ }
+
+ private void checkInternal(FailingTriConsumer<A, B, C> fn)
+ {
+ RandomSource random = new DefaultRandom(seed);
+ for (int i = 0; i < examples; i++)
+ {
+ A a = null;
+ B b = null;
+ C c = null;
+ try
+ {
+ checkInterrupted();
+ fn.accept(a = as.next(random), b = bs.next(random), c = cs.next(random));
+ }
+ catch (Throwable t)
+ {
+ throw new PropertyError(propertyError(this, t, a, b, c), t);
+ }
+ if (pure)
+ {
+ seed = random.nextLong();
+ random.setSeed(seed);
+ }
+ }
+ }
+ }
+
+ private static void checkInterrupted() throws InterruptedException
+ {
+ if (Thread.currentThread().isInterrupted())
+ throw new InterruptedException();
+ }
+
+ public static class PropertyError extends AssertionError
+ {
+ public PropertyError(String message, Throwable cause)
+ {
+ super(message, cause);
+ }
+
+ public PropertyError(String message)
+ {
+ super(message);
+ }
+ }
+
+ public static ForBuilder qt()
+ {
+ return new ForBuilder();
+ }
+}
diff --git a/test/unit/accord/utils/RandomSource.java b/test/unit/accord/utils/RandomSource.java
new file mode 100644
index 0000000000..3d4861e5e4
--- /dev/null
+++ b/test/unit/accord/utils/RandomSource.java
@@ -0,0 +1,361 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package accord.utils;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.NavigableSet;
+import java.util.Random;
+import java.util.Set;
+import java.util.stream.DoubleStream;
+import java.util.stream.IntStream;
+import java.util.stream.LongStream;
+
+public interface RandomSource
+{
+ static RandomSource wrap(Random random)
+ {
+ return new WrappedRandomSource(random);
+ }
+
+ void nextBytes(byte[] bytes);
+
+ boolean nextBoolean();
+
+ int nextInt();
+
+ default int nextInt(int maxExclusive)
+ {
+ return nextInt(0, maxExclusive);
+ }
+
+ default int nextInt(int minInclusive, int maxExclusive)
+ {
+ // this is diff behavior than ThreadLocalRandom, which returns nextInt
+ if (minInclusive >= maxExclusive)
+ throw new IllegalArgumentException(String.format("Min (%s) should be less than max (%d).", minInclusive, maxExclusive));
+
+ int result = nextInt();
+ int delta = maxExclusive - minInclusive;
+ int mask = delta - 1;
+ if ((delta & mask) == 0) // power of two
+ result = (result & mask) + minInclusive;
+ else if (delta > 0)
+ {
+ // reject over-represented candidates
+ for (int u = result >>> 1; // ensure nonnegative
+ u + mask - (result = u % delta) < 0; // rejection check
+ u = nextInt() >>> 1) // retry
+ ;
+ result += minInclusive;
+ }
+ else
+ {
+ // range not representable as int
+ while (result < minInclusive || result >= maxExclusive)
+ result = nextInt();
+ }
+ return result;
+ }
+
+ default IntStream ints()
+ {
+ return IntStream.generate(this::nextInt);
+ }
+
+ default IntStream ints(int maxExclusive)
+ {
+ return IntStream.generate(() -> nextInt(maxExclusive));
+ }
+
+ default IntStream ints(int minInclusive, int maxExclusive)
+ {
+ return IntStream.generate(() -> nextInt(minInclusive, maxExclusive));
+ }
+
+ long nextLong();
+
+ default long nextLong(long maxExclusive)
+ {
+ return nextLong(0, maxExclusive);
+ }
+
+ default long nextLong(long minInclusive, long maxExclusive)
+ {
+ // this is diff behavior than ThreadLocalRandom, which returns nextLong
+ if (minInclusive >= maxExclusive)
+ throw new IllegalArgumentException(String.format("Min (%s) should be less than max (%d).", minInclusive, maxExclusive));
+
+ long result = nextLong();
+ long delta = maxExclusive - minInclusive;
+ long mask = delta - 1;
+ if ((delta & mask) == 0L) // power of two
+ result = (result & mask) + minInclusive;
+ else if (delta > 0L)
+ {
+ // reject over-represented candidates
+ for (long u = result >>> 1; // ensure nonnegative
+ u + mask - (result = u % delta) < 0L; // rejection check
+ u = nextLong() >>> 1) // retry
+ ;
+ result += minInclusive;
+ }
+ else
+ {
+ // range not representable as long
+ while (result < minInclusive || result >= maxExclusive)
+ result = nextLong();
+ }
+ return result;
+ }
+
+ default LongStream longs()
+ {
+ return LongStream.generate(this::nextLong);
+ }
+
+ default LongStream longs(long maxExclusive)
+ {
+ return LongStream.generate(() -> nextLong(maxExclusive));
+ }
+
+ default LongStream longs(long minInclusive, long maxExclusive)
+ {
+ return LongStream.generate(() -> nextLong(minInclusive, maxExclusive));
+ }
+
+ float nextFloat();
+
+ double nextDouble();
+
+ default double nextDouble(double maxExclusive)
+ {
+ return nextDouble(0, maxExclusive);
+ }
+
+ default double nextDouble(double minInclusive, double maxExclusive)
+ {
+ if (minInclusive >= maxExclusive)
+ throw new IllegalArgumentException(String.format("Min (%s) should be less than max (%d).", minInclusive, maxExclusive));
+
+ double result = nextDouble();
+ result = result * (maxExclusive - minInclusive) + minInclusive;
+ if (result >= maxExclusive) // correct for rounding
+ result = Double.longBitsToDouble(Double.doubleToLongBits(maxExclusive) - 1);
+ return result;
+ }
+
+ default DoubleStream doubles()
+ {
+ return DoubleStream.generate(this::nextDouble);
+ }
+
+ default DoubleStream doubles(double maxExclusive)
+ {
+ return DoubleStream.generate(() -> nextDouble(maxExclusive));
+ }
+
+ default DoubleStream doubles(double minInclusive, double maxExclusive)
+ {
+ return DoubleStream.generate(() -> nextDouble(minInclusive, maxExclusive));
+ }
+
+ double nextGaussian();
+
+ default int pickInt(int first, int second, int... rest)
+ {
+ int offset = nextInt(0, rest.length + 2);
+ switch (offset)
+ {
+ case 0: return first;
+ case 1: return second;
+ default: return rest[offset - 2];
+ }
+ }
+
+ default int pickInt(int[] array)
+ {
+ return pickInt(array, 0, array.length);
+ }
+
+ default int pickInt(int[] array, int offset, int length)
+ {
+ Invariants.checkIndexInBounds(array.length, offset, length);
+ if (length == 1)
+ return array[offset];
+ return array[nextInt(offset, offset + length)];
+ }
+
+ default long pickLong(long first, long second, long... rest)
+ {
+ int offset = nextInt(0, rest.length + 2);
+ switch (offset)
+ {
+ case 0: return first;
+ case 1: return second;
+ default: return rest[offset - 2];
+ }
+ }
+
+ default long pickLong(long[] array)
+ {
+ return pickLong(array, 0, array.length);
+ }
+
+ default long pickLong(long[] array, int offset, int length)
+ {
+ Invariants.checkIndexInBounds(array.length, offset, length);
+ if (length == 1)
+ return array[offset];
+ return array[nextInt(offset, offset + length)];
+ }
+
+ default <T extends Comparable<T>> T pick(Set<T> set)
+ {
+ List<T> values = new ArrayList<>(set);
+ // Non-ordered sets may have different iteration order on different environments, which would make a seed produce different histories!
+ // To avoid such a problem, make sure to apply a deterministic function (sort).
+ if (!(set instanceof NavigableSet))
+ values.sort(Comparator.naturalOrder());
+ return pick(values);
+ }
+
+ default <T> T pick(T first, T second, T... rest)
+ {
+ int offset = nextInt(0, rest.length + 2);
+ switch (offset)
+ {
+ case 0: return first;
+ case 1: return second;
+ default: return rest[offset - 2];
+ }
+ }
+
+ default <T> T pick(T[] array)
+ {
+ return array[nextInt(array.length)];
+ }
+
+ default <T> T pick(List<T> values)
+ {
+ return pick(values, 0, values.size());
+ }
+
+ default <T> T pick(List<T> values, int offset, int length)
+ {
+ Invariants.checkIndexInBounds(values.size(), offset, length);
+ if (length == 1)
+ return values.get(offset);
+ return values.get(nextInt(offset, offset + length));
+ }
+
+ void setSeed(long seed);
+
+ RandomSource fork();
+
+ /**
+ * Returns true with a probability of {@code chance}. This logic is logically the same as
+ * <pre>{@code nextFloat() < chance}</pre>
+ *
+ * @param chance cumulative probability in range [0..1]
+ */
+ default boolean decide(float chance)
+ {
+ return nextFloat() < chance;
+ }
+
+ /**
+ * Returns true with a probability of {@code chance}. This logic is logically the same as
+ * <pre>{@code nextDouble() < chance}</pre>
+ *
+ * @param chance cumulative probability in range [0..1]
+ */
+ default boolean decide(double chance)
+ {
+ return nextDouble() < chance;
+ }
+
+ default long reset()
+ {
+ long seed = nextLong();
+ setSeed(seed);
+ return seed;
+ }
+
+ default Random asJdkRandom()
+ {
+ return new Random()
+ {
+ @Override
+ public void setSeed(long seed)
+ {
+ RandomSource.this.setSeed(seed);
+ }
+
+ @Override
+ public void nextBytes(byte[] bytes)
+ {
+ RandomSource.this.nextBytes(bytes);
+ }
+
+ @Override
+ public int nextInt()
+ {
+ return RandomSource.this.nextInt();
+ }
+
+ @Override
+ public int nextInt(int bound)
+ {
+ return RandomSource.this.nextInt(bound);
+ }
+
+ @Override
+ public long nextLong()
+ {
+ return RandomSource.this.nextLong();
+ }
+
+ @Override
+ public boolean nextBoolean()
+ {
+ return RandomSource.this.nextBoolean();
+ }
+
+ @Override
+ public float nextFloat()
+ {
+ return RandomSource.this.nextFloat();
+ }
+
+ @Override
+ public double nextDouble()
+ {
+ return RandomSource.this.nextDouble();
+ }
+
+ @Override
+ public double nextGaussian()
+ {
+ return RandomSource.this.nextGaussian();
+ }
+ };
+ }
+}
diff --git a/test/unit/accord/utils/WrappedRandomSource.java b/test/unit/accord/utils/WrappedRandomSource.java
new file mode 100644
index 0000000000..3d02c101fb
--- /dev/null
+++ b/test/unit/accord/utils/WrappedRandomSource.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package accord.utils;
+
+import java.util.Random;
+
+class WrappedRandomSource implements RandomSource
+{
+ private final Random random;
+
+ WrappedRandomSource(Random random)
+ {
+ this.random = random;
+ }
+
+ @Override
+ public Random asJdkRandom()
+ {
+ return random;
+ }
+
+ @Override
+ public void nextBytes(byte[] bytes)
+ {
+ random.nextBytes(bytes);
+ }
+
+ @Override
+ public boolean nextBoolean()
+ {
+ return random.nextBoolean();
+ }
+
+ @Override
+ public int nextInt()
+ {
+ return random.nextInt();
+ }
+
+ @Override
+ public int nextInt(int maxExclusive)
+ {
+ return random.nextInt(maxExclusive);
+ }
+
+ @Override
+ public long nextLong()
+ {
+ return random.nextLong();
+ }
+
+ @Override
+ public float nextFloat()
+ {
+ return random.nextFloat();
+ }
+
+ @Override
+ public double nextDouble()
+ {
+ return random.nextDouble();
+ }
+
+ @Override
+ public double nextGaussian()
+ {
+ return random.nextGaussian();
+ }
+
+ @Override
+ public void setSeed(long seed)
+ {
+ random.setSeed(seed);
+ }
+
+ @Override
+ public RandomSource fork()
+ {
+ return new WrappedRandomSource(new Random(nextLong()));
+ }
+}
diff --git a/test/unit/org/apache/cassandra/concurrent/ForwardingExecutorPlus.java b/test/unit/org/apache/cassandra/concurrent/ForwardingExecutorPlus.java
new file mode 100644
index 0000000000..71c88a947e
--- /dev/null
+++ b/test/unit/org/apache/cassandra/concurrent/ForwardingExecutorPlus.java
@@ -0,0 +1,219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.concurrent;
+
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.MoreExecutors;
+
+import org.apache.cassandra.utils.WithResources;
+import org.apache.cassandra.utils.concurrent.AsyncPromise;
+import org.apache.cassandra.utils.concurrent.Future;
+
+public class ForwardingExecutorPlus implements ExecutorPlus
+{
+ private final ExecutorService delegate;
+
+ public ForwardingExecutorPlus(ExecutorService delegate)
+ {
+ this.delegate = delegate;
+ }
+
+ protected ExecutorService delegate()
+ {
+ return delegate;
+ }
+
+ @Override
+ public void shutdown()
+ {
+ delegate().shutdown();
+ }
+
+ @Override
+ public List<Runnable> shutdownNow()
+ {
+ return delegate().shutdownNow();
+ }
+
+ @Override
+ public boolean isShutdown()
+ {
+ return delegate().isShutdown();
+ }
+
+ @Override
+ public boolean isTerminated()
+ {
+ return delegate().isTerminated();
+ }
+
+ @Override
+ public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException
+ {
+ return delegate().awaitTermination(timeout, unit);
+ }
+
+ @Override
+ public <T> Future<T> submit(Callable<T> task)
+ {
+ return wrap(delegate().submit(task));
+ }
+
+ @Override
+ public <T> Future<T> submit(Runnable task, T result)
+ {
+ return wrap(delegate().submit(task, result));
+ }
+
+ @Override
+ public Future<?> submit(Runnable task)
+ {
+ return wrap(delegate().submit(task));
+ }
+
+ @Override
+ public void execute(WithResources withResources, Runnable task)
+ {
+ execute(TaskFactory.standard().toExecute(withResources, task));
+ }
+
+ @Override
+ public <T> Future<T> submit(WithResources withResources, Callable<T> task)
+ {
+ class Catch
+ {
+ T value;
+ }
+ Catch c = new Catch();
+ Runnable exec = TaskFactory.standard().toExecute(withResources, () -> {
+ try
+ {
+ c.value = task.call();
+ }
+ catch (Exception e)
+ {
+ throw new RuntimeException(e);
+ }
+ });
+ return submit(() -> {
+ exec.run();
+ return c.value;
+ });
+ }
+
+ @Override
+ public Future<?> submit(WithResources withResources, Runnable task)
+ {
+ return submit(TaskFactory.standard().toExecute(withResources, task));
+ }
+
+ @Override
+ public <T> Future<T> submit(WithResources withResources, Runnable task, T result)
+ {
+ return submit(Executors.callable(TaskFactory.standard().toSubmit(withResources, task), result));
+ }
+
+ @Override
+ public boolean inExecutor()
+ {
+ return false;
+ }
+
+ @Override
+ public void execute(Runnable command)
+ {
+ delegate().execute(command);
+ }
+
+ @Override
+ public int getCorePoolSize()
+ {
+ return 0;
+ }
+
+ @Override
+ public void setCorePoolSize(int newCorePoolSize)
+ {
+
+ }
+
+ @Override
+ public int getMaximumPoolSize()
+ {
+ return 0;
+ }
+
+ @Override
+ public void setMaximumPoolSize(int newMaximumPoolSize)
+ {
+
+ }
+
+ @Override
+ public int getActiveTaskCount()
+ {
+ return 0;
+ }
+
+ @Override
+ public long getCompletedTaskCount()
+ {
+ return 0;
+ }
+
+ @Override
+ public int getPendingTaskCount()
+ {
+ return 0;
+ }
+
+ private <T> Future<T> wrap(java.util.concurrent.Future<T> submit)
+ {
+ if (submit instanceof Future)
+ return (Future<T>) submit;
+ if (submit instanceof ListenableFuture)
+ {
+ AsyncPromise<T> promise = new AsyncPromise<>();
+ Futures.addCallback((ListenableFuture<T>) submit, new FutureCallback<T>()
+ {
+ @Override
+ public void onSuccess(T result)
+ {
+ promise.setSuccess(result);
+ }
+
+ @Override
+ public void onFailure(Throwable t)
+ {
+ promise.setFailure(t);
+ }
+ }, MoreExecutors.directExecutor());
+ return promise;
+ }
+ throw new IllegalStateException("Unexpected future type: " + submit.getClass());
+ }
+}
diff --git a/test/unit/org/apache/cassandra/concurrent/ForwardingLocalAwareExecutorPlus.java b/test/unit/org/apache/cassandra/concurrent/ForwardingLocalAwareExecutorPlus.java
new file mode 100644
index 0000000000..e0bde7646d
--- /dev/null
+++ b/test/unit/org/apache/cassandra/concurrent/ForwardingLocalAwareExecutorPlus.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.concurrent;
+
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.utils.WithResources;
+import org.apache.cassandra.utils.concurrent.Future;
+
+public class ForwardingLocalAwareExecutorPlus implements LocalAwareExecutorPlus
+{
+ private final ExecutorPlus delegate;
+
+ public ForwardingLocalAwareExecutorPlus(ExecutorPlus delegate)
+ {
+ this.delegate = delegate;
+ }
+
+ protected ExecutorPlus delegate()
+ {
+ return delegate;
+ }
+
+ @Override
+ public void shutdown()
+ {
+ delegate().shutdown();
+ }
+
+ @Override
+ public List<Runnable> shutdownNow()
+ {
+ return delegate().shutdownNow();
+ }
+
+ @Override
+ public boolean isShutdown()
+ {
+ return delegate().isShutdown();
+ }
+
+ @Override
+ public boolean isTerminated()
+ {
+ return delegate().isTerminated();
+ }
+
+ @Override
+ public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException
+ {
+ return delegate().awaitTermination(timeout, unit);
+ }
+
+ @Override
+ public <T> Future<T> submit(Callable<T> task)
+ {
+ return delegate().submit(task);
+ }
+
+ @Override
+ public <T> Future<T> submit(Runnable task, T result)
+ {
+ return delegate().submit(task, result);
+ }
+
+ @Override
+ public Future<?> submit(Runnable task)
+ {
+ return delegate().submit(task);
+ }
+
+ @Override
+ public void execute(WithResources withResources, Runnable task)
+ {
+ delegate().execute(withResources, task);
+ }
+
+ @Override
+ public <T> Future<T> submit(WithResources withResources, Callable<T> task)
+ {
+ return delegate().submit(withResources, task);
+ }
+
+ @Override
+ public Future<?> submit(WithResources withResources, Runnable task)
+ {
+ return delegate().submit(withResources, task);
+ }
+
+ @Override
+ public <T> Future<T> submit(WithResources withResources, Runnable task, T result)
+ {
+ return delegate().submit(withResources, task, result);
+ }
+
+ @Override
+ public boolean inExecutor()
+ {
+ return delegate().inExecutor();
+ }
+
+ @Override
+ public void execute(Runnable command)
+ {
+ delegate().execute(command);
+ }
+
+ @Override
+ public int getCorePoolSize()
+ {
+ return delegate().getCorePoolSize();
+ }
+
+ @Override
+ public void setCorePoolSize(int newCorePoolSize)
+ {
+ delegate().setCorePoolSize(newCorePoolSize);
+ }
+
+ @Override
+ public int getMaximumPoolSize()
+ {
+ return delegate().getMaximumPoolSize();
+ }
+
+ @Override
+ public void setMaximumPoolSize(int newMaximumPoolSize)
+ {
+ delegate().setMaximumPoolSize(newMaximumPoolSize);
+ }
+
+ @Override
+ public int getActiveTaskCount()
+ {
+ return delegate().getActiveTaskCount();
+ }
+
+ @Override
+ public long getCompletedTaskCount()
+ {
+ return delegate().getCompletedTaskCount();
+ }
+
+ @Override
+ public int getPendingTaskCount()
+ {
+ return delegate().getPendingTaskCount();
+ }
+}
diff --git a/test/unit/org/apache/cassandra/concurrent/ForwardingScheduledExecutorPlus.java b/test/unit/org/apache/cassandra/concurrent/ForwardingScheduledExecutorPlus.java
new file mode 100644
index 0000000000..b6f0bb6067
--- /dev/null
+++ b/test/unit/org/apache/cassandra/concurrent/ForwardingScheduledExecutorPlus.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.concurrent;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+
+import static com.google.common.primitives.Longs.max;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static org.apache.cassandra.utils.Clock.Global.nanoTime;
+
+public class ForwardingScheduledExecutorPlus extends ForwardingExecutorPlus implements ScheduledExecutorPlus
+{
+ private final ScheduledExecutorService delegate;
+
+ public ForwardingScheduledExecutorPlus(ScheduledExecutorService delegate)
+ {
+ super(delegate);
+ this.delegate = delegate;
+ }
+
+ protected ScheduledExecutorService delegate()
+ {
+ return delegate;
+ }
+
+ @Override
+ public ScheduledFuture<?> scheduleSelfRecurring(Runnable run, long delay, TimeUnit units)
+ {
+ return schedule(run, delay, units);
+ }
+
+ @Override
+ public ScheduledFuture<?> scheduleAt(Runnable run, long deadline)
+ {
+ return schedule(run, max(0, deadline - nanoTime()), NANOSECONDS);
+ }
+
+ @Override
+ public ScheduledFuture<?> scheduleTimeoutAt(Runnable run, long deadline)
+ {
+ return scheduleTimeoutWithDelay(run, max(0, deadline - nanoTime()), NANOSECONDS);
+ }
+
+ @Override
+ public ScheduledFuture<?> scheduleTimeoutWithDelay(Runnable run, long delay, TimeUnit units)
+ {
+ return schedule(run, delay, units);
+ }
+
+ @Override
+ public ScheduledFuture<?> schedule(Runnable command, long delay, TimeUnit unit)
+ {
+ return delegate().schedule(command, delay, unit);
+ }
+
+ @Override
+ public <V> ScheduledFuture<V> schedule(Callable<V> callable, long delay, TimeUnit unit)
+ {
+ return delegate().schedule(callable, delay, unit);
+ }
+
+ @Override
+ public ScheduledFuture<?> scheduleAtFixedRate(Runnable command, long initialDelay, long period, TimeUnit unit)
+ {
+ return delegate().scheduleAtFixedRate(command, initialDelay, period, unit);
+ }
+
+ @Override
+ public ScheduledFuture<?> scheduleWithFixedDelay(Runnable command, long initialDelay, long delay, TimeUnit unit)
+ {
+ return delegate().scheduleWithFixedDelay(command, initialDelay, delay, unit);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/concurrent/SimulatedExecutorFactory.java b/test/unit/org/apache/cassandra/concurrent/SimulatedExecutorFactory.java
new file mode 100644
index 0000000000..ffb1a0fabd
--- /dev/null
+++ b/test/unit/org/apache/cassandra/concurrent/SimulatedExecutorFactory.java
@@ -0,0 +1,506 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.concurrent;
+
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.PriorityQueue;
+import java.util.Queue;
+import java.util.concurrent.AbstractExecutorService;
+import java.util.concurrent.Callable;
+import java.util.concurrent.Delayed;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.RejectedExecutionHandler;
+import java.util.concurrent.RunnableFuture;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.function.LongSupplier;
+
+import accord.utils.Gens;
+import accord.utils.RandomSource;
+import org.apache.cassandra.utils.Clock;
+
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+
+public class SimulatedExecutorFactory implements ExecutorFactory, Clock
+{
+ private static class Item implements Comparable<Item>
+ {
+ private final long runAtNanos;
+ private final long seq;
+ private final org.apache.cassandra.utils.concurrent.RunnableFuture<?> action;
+
+ private Item(long runAtNanos, long seq, org.apache.cassandra.utils.concurrent.RunnableFuture<?> action)
+ {
+ if (runAtNanos < 0)
+ throw new IllegalArgumentException("Time went backwards! Given " + runAtNanos);
+ this.runAtNanos = runAtNanos;
+ this.seq = seq;
+ this.action = action;
+ }
+
+ @Override
+ public int compareTo(Item o)
+ {
+ int rc = Long.compare(runAtNanos, o.runAtNanos);
+ if (rc != 0)
+ return rc;
+ return Long.compare(seq, o.seq);
+ }
+
+ @Override
+ public String toString()
+ {
+ return "Item{" +
+ "runAtNanos=" + runAtNanos +
+ ", seq=" + seq +
+ '}';
+ }
+ }
+
+ private final RandomSource rs;
+ private final long startTimeNanos;
+ private final PriorityQueue<Item> queue = new PriorityQueue<>();
+ private long seq = 0;
+ private long nowNanos;
+ private int repeatedTasks = 0;
+
+ public SimulatedExecutorFactory(RandomSource rs, long startTimeNanos)
+ {
+ this.rs = rs;
+ this.startTimeNanos = startTimeNanos;
+ }
+
+ public boolean processOne()
+ {
+ // if we count the repeated tasks, then processAll will never complete
+ if (queue.size() == repeatedTasks)
+ return false;
+ Item item = queue.poll();
+ if (item == null)
+ return false;
+ nowNanos = Math.max(nowNanos + 1, item.runAtNanos);
+ item.action.run();
+ return true;
+ }
+
+ @Override
+ public long nanoTime()
+ {
+ return nowNanos++;
+ }
+
+ @Override
+ public long currentTimeMillis()
+ {
+ return TimeUnit.NANOSECONDS.toMillis(startTimeNanos + nanoTime());
+ }
+
+
+ @Override
+ public ExecutorBuilder<? extends SequentialExecutorPlus> configureSequential(String name)
+ {
+ return new SimulatedExecutorBuilder<>().configureSequential(name);
+ }
+
+ @Override
+ public ExecutorBuilder<? extends ExecutorPlus> configurePooled(String name, int threads)
+ {
+ return new SimulatedExecutorBuilder<>().configurePooled(name, threads);
+ }
+
+ @Override
+ public ExecutorBuilderFactory<ExecutorPlus, SequentialExecutorPlus> withJmx(String jmxPath)
+ {
+ return this;
+ }
+
+ @Override
+ public LocalAwareSubFactory localAware()
+ {
+ return new SimulatedExecutorBuilder<>();
+ }
+
+ @Override
+ public ScheduledExecutorPlus scheduled(boolean executeOnShutdown, String name, int priority, SimulatorSemantics simulatorSemantics)
+ {
+ return new ForwardingScheduledExecutorPlus(new UnorderedScheduledExecutorService());
+ }
+
+ @Override
+ public Thread startThread(String name, Runnable runnable, InfiniteLoopExecutor.Daemon daemon)
+ {
+ throw new UnsupportedOperationException("Thread can't be simualted");
+ }
+
+ @Override
+ public Interruptible infiniteLoop(String name, Interruptible.Task task, InfiniteLoopExecutor.SimulatorSafe simulatorSafe, InfiniteLoopExecutor.Daemon daemon, InfiniteLoopExecutor.Interrupts interrupts)
+ {
+ throw new UnsupportedOperationException("TODO");
+ }
+
+ @Override
+ public ThreadGroup newThreadGroup(String name)
+ {
+ throw new UnsupportedOperationException("Thread can't be simualted");
+ }
+
+ private class SimulatedExecutorBuilder<E extends ExecutorService> implements ExecutorBuilder<E>, LocalAwareSubFactory, LocalAwareSubFactoryWithJMX
+ {
+ private int threads = -1;
+
+ @Override
+ public ExecutorBuilder<E> withKeepAlive(long keepAlive, TimeUnit keepAliveUnits)
+ {
+ return this;
+ }
+
+ @Override
+ public ExecutorBuilder<E> withKeepAlive()
+ {
+ return this;
+ }
+
+ @Override
+ public ExecutorBuilder<E> withThreadPriority(int threadPriority)
+ {
+ return this;
+ }
+
+ @Override
+ public ExecutorBuilder<E> withThreadGroup(ThreadGroup threadGroup)
+ {
+ return this;
+ }
+
+ @Override
+ public ExecutorBuilder<E> withDefaultThreadGroup()
+ {
+ return this;
+ }
+
+ @Override
+ public ExecutorBuilder<E> withQueueLimit(int queueLimit)
+ {
+ throw new UnsupportedOperationException("TODO");
+ }
+
+ @Override
+ public ExecutorBuilder<E> withRejectedExecutionHandler(RejectedExecutionHandler rejectedExecutionHandler)
+ {
+ throw new UnsupportedOperationException("TODO");
+ }
+
+ @Override
+ public ExecutorBuilder<E> withUncaughtExceptionHandler(Thread.UncaughtExceptionHandler uncaughtExceptionHandler)
+ {
+ throw new UnsupportedOperationException("TODO");
+ }
+
+ @Override
+ public ExecutorBuilder<? extends LocalAwareSequentialExecutorPlus> configureSequential(String name)
+ {
+ threads = 1;
+ return (ExecutorBuilder<? extends LocalAwareSequentialExecutorPlus>) this;
+ }
+
+ @Override
+ public ExecutorBuilder<? extends LocalAwareExecutorPlus> configurePooled(String name, int threads)
+ {
+ this.threads = threads;
+ return (ExecutorBuilder<? extends LocalAwareExecutorPlus>) this;
+ }
+
+ @Override
+ public LocalAwareSubFactoryWithJMX withJmx(String jmxPath)
+ {
+ return this;
+ }
+
+ @Override
+ public LocalAwareSubFactoryWithJMX withJmxInternal()
+ {
+ return this;
+ }
+
+ @Override
+ public LocalAwareExecutorPlus shared(String name, int threads, ExecutorPlus.MaximumPoolSizeListener onSetMaxSize)
+ {
+ return new ForwardingLocalAwareExecutorPlus(build0());
+ }
+
+ @Override
+ public E build()
+ {
+ return (E) build0();
+ }
+
+ private ForwardingExecutorPlus build0()
+ {
+ return new ForwardingExecutorPlus(threads == 1 ?
+ new OrderedExecutorService() :
+ new UnorderedExecutorService());
+ }
+ }
+
+ private class UnorderedExecutorService extends AbstractExecutorService
+ {
+ private final LongSupplier jitterNanos;
+ private boolean shutdown = false;
+
+ UnorderedExecutorService()
+ {
+ long maxSmall = TimeUnit.MICROSECONDS.toNanos(50);
+ long max = TimeUnit.MILLISECONDS.toNanos(5);
+ LongSupplier small = () -> rs.nextLong(0, maxSmall);
+ LongSupplier large = () -> rs.nextLong(maxSmall, max);
+ this.jitterNanos = Gens.bools().runs(rs.nextInt(1, 11) / 100.0D, rs.nextInt(3, 15)).mapToLong(b -> b ? large.getAsLong() : small.getAsLong()).asLongSupplier(rs);
+ }
+
+ @Override
+ protected <T> RunnableFuture<T> newTaskFor(Runnable runnable, T value)
+ {
+ return new FutureTask<>(Executors.callable(runnable, value));
+ }
+
+ @Override
+ protected <T> RunnableFuture<T> newTaskFor(Callable<T> callable)
+ {
+ return new FutureTask<>(callable);
+ }
+
+ protected org.apache.cassandra.utils.concurrent.RunnableFuture<?> taskFor(Runnable command)
+ {
+ if (command instanceof org.apache.cassandra.utils.concurrent.RunnableFuture<?>)
+ return (org.apache.cassandra.utils.concurrent.RunnableFuture<?>) command;
+ return new FutureTask<>(Executors.callable(command));
+ }
+
+ @Override
+ public void shutdown()
+ {
+ shutdown = true;
+ }
+
+ @Override
+ public List<Runnable> shutdownNow()
+ {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public boolean isShutdown()
+ {
+ return shutdown;
+ }
+
+ @Override
+ public boolean isTerminated()
+ {
+ return shutdown;
+ }
+
+ @Override
+ public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException
+ {
+ return shutdown;
+ }
+
+ @Override
+ public void execute(Runnable command)
+ {
+ checkNotShutdown();
+ queue.add(new Item(nowWithJitter(), SimulatedExecutorFactory.this.seq++, taskFor(command)));
+ }
+
+ protected void checkNotShutdown()
+ {
+ if (isShutdown())
+ throw new RejectedExecutionException("Shutdown");
+ }
+
+ protected long nowWithJitter()
+ {
+ return nanoTime() + jitterNanos.getAsLong();
+ }
+ }
+
+ private class OrderedExecutorService extends UnorderedExecutorService
+ {
+ private final Queue<Item> pending = new LinkedList<>();
+
+ @Override
+ public void execute(Runnable command)
+ {
+ checkNotShutdown();
+ boolean wasEmpty = pending.isEmpty();
+ Item task = new Item(nowWithJitter(), SimulatedExecutorFactory.this.seq++, taskFor(command));
+ pending.add(task);
+ if (wasEmpty)
+ runNextTask();
+ }
+
+ private void runNextTask()
+ {
+ Item next = pending.peek();
+ if (next == null)
+ return;
+
+ next.action.addCallback((s, f) -> afterExecution());
+ queue.add(next);
+ }
+
+ private void afterExecution()
+ {
+ pending.poll();
+ runNextTask();
+ }
+ }
+
+ private class UnorderedScheduledExecutorService extends UnorderedExecutorService implements ScheduledExecutorService
+ {
+ private class ScheduledFuture<T> extends FutureTask<T> implements java.util.concurrent.ScheduledFuture<T>
+ {
+ private final long sequenceNumber;
+ private final long periodNanos;
+ private long nextExecuteAtNanos;
+
+ ScheduledFuture(long sequenceNumber, long initialDelay, long value, TimeUnit unit, Callable<? extends T> call)
+ {
+ super(call);
+ this.sequenceNumber = sequenceNumber;
+ periodNanos = unit.toNanos(value);
+ nextExecuteAtNanos = triggerTimeNanos(initialDelay, unit);
+ }
+
+ private long triggerTimeNanos(long delay, TimeUnit unit)
+ {
+ long delayNanos = unit.toNanos(delay < 0 ? 0 : delay);
+ return nanoTime() + delayNanos;
+ }
+
+ @Override
+ public long getDelay(TimeUnit unit)
+ {
+ return unit.convert(nextExecuteAtNanos - nanoTime(), TimeUnit.NANOSECONDS);
+ }
+
+ @Override
+ public int compareTo(Delayed other)
+ {
+ if (other == this) // compare zero if same object
+ return 0;
+ if (other instanceof ScheduledFuture)
+ {
+ ScheduledFuture<?> x = (ScheduledFuture<?>) other;
+ long diff = nextExecuteAtNanos - x.nextExecuteAtNanos;
+ if (diff < 0)
+ return -1;
+ else if (diff > 0)
+ return 1;
+ else if (sequenceNumber < x.sequenceNumber)
+ return -1;
+ else
+ return 1;
+ }
+ long diff = getDelay(NANOSECONDS) - other.getDelay(NANOSECONDS);
+ return (diff < 0) ? -1 : (diff > 0) ? 1 : 0;
+ }
+
+ @Override
+ public void run()
+ {
+ boolean periodic = periodNanos != 0;
+ if (!periodic)
+ {
+ super.run();
+ }
+ else
+ {
+ if (isCancelled())
+ return;
+ // run without setting the result
+ try
+ {
+ call();
+ long nowNanos = nanoTime();
+ if (periodNanos > 0)
+ {
+ // scheduleAtFixedRate
+ nextExecuteAtNanos += periodNanos;
+ }
+ else
+ {
+ // scheduleWithFixedDelay
+ nextExecuteAtNanos = nowNanos + (-periodNanos);
+ }
+ long delayNanos = nextExecuteAtNanos - nowNanos;
+ if (delayNanos < 0)
+ delayNanos = 0;
+ schedule(this, delayNanos, NANOSECONDS);
+ }
+ catch (Throwable t)
+ {
+ tryFailure(t);
+ }
+ }
+ }
+ }
+
+ @Override
+ public ScheduledFuture<?> schedule(Runnable command, long delay, TimeUnit unit)
+ {
+ return schedule(Executors.callable(command), delay, unit);
+ }
+
+ @Override
+ public <V> ScheduledFuture<V> schedule(Callable<V> callable, long delay, TimeUnit unit)
+ {
+ checkNotShutdown();
+ ScheduledFuture<V> task = new ScheduledFuture<>(seq++, delay, 0, NANOSECONDS, callable);
+ queue.add(new Item(nowWithJitter() + unit.toNanos(delay), task.sequenceNumber, task));
+ return task;
+ }
+
+ @Override
+ public ScheduledFuture<?> scheduleAtFixedRate(Runnable command, long initialDelay, long period, TimeUnit unit)
+ {
+ checkNotShutdown();
+ ScheduledFuture<?> task = new ScheduledFuture<>(seq++, initialDelay, period, unit, Executors.callable(command));
+ repeatedTasks++;
+ task.addCallback((s, f) -> repeatedTasks--);
+ queue.add(new Item(nowWithJitter() + unit.toNanos(initialDelay), task.sequenceNumber, task));
+ return task;
+ }
+
+ @Override
+ public ScheduledFuture<?> scheduleWithFixedDelay(Runnable command, long initialDelay, long delay, TimeUnit unit)
+ {
+ checkNotShutdown();
+ ScheduledFuture<?> task = new ScheduledFuture<>(seq++, initialDelay, -delay, unit, Executors.callable(command));
+ repeatedTasks++;
+ task.addCallback((s, f) -> repeatedTasks--);
+ queue.add(new Item(nowWithJitter() + unit.toNanos(initialDelay), task.sequenceNumber, task));
+ return task;
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java b/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java
index b1e9c1ec0d..b4da790360 100644
--- a/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java
+++ b/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java
@@ -140,6 +140,11 @@ public class DatabaseDescriptorRefTest
"org.apache.cassandra.config.GuardrailsOptions$ConsistencyLevels",
"org.apache.cassandra.config.GuardrailsOptions$TableProperties",
"org.apache.cassandra.config.ParameterizedClass",
+ "org.apache.cassandra.config.RepairConfig",
+ "org.apache.cassandra.config.RepairRetrySpec",
+ "org.apache.cassandra.config.RetrySpec",
+ "org.apache.cassandra.config.RetrySpec$MaxAttempt",
+ "org.apache.cassandra.config.RetrySpec$Type",
"org.apache.cassandra.config.ReplicaFilteringProtectionOptions",
"org.apache.cassandra.config.StartupChecksOptions",
"org.apache.cassandra.config.SubnetGroups",
diff --git a/test/unit/org/apache/cassandra/config/UnitConfigOverride.java b/test/unit/org/apache/cassandra/config/UnitConfigOverride.java
new file mode 100644
index 0000000000..bd6294f0bd
--- /dev/null
+++ b/test/unit/org/apache/cassandra/config/UnitConfigOverride.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.config;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.util.Arrays;
+import java.util.List;
+import javax.annotation.Nullable;
+
+public class UnitConfigOverride
+{
+ private enum ConfigType
+ {
+ CDC("cassandra.yaml", "cdc.yaml"),
+ COMPRESSION("cassandra.yaml", "commitlog_compression_LZ4.yaml"), // not 100% correct, LZ4 could be changed at build time
+ OA("cassandra.yaml", "storage_compatibility_mode_none.yaml"),
+ SYSTEM_KEYSPACE_DIRECTORY("cassandra.yaml", "system_keyspaces_directory.yaml"),
+ TRIE("cassandra.yaml", "trie_memtable.yaml", "storage_compatibility_mode_none.yaml");
+
+ public final List<String> configs;
+
+ ConfigType(String... configs)
+ {
+ this.configs = Arrays.asList(configs);
+ }
+ }
+
+ @Nullable
+ private static final ConfigType CONFIG_TYPE = null;
+
+ @SuppressWarnings("ConstantConditions")
+ public static void maybeOverrideConfig()
+ {
+ // CONFIG_TYPE is meant to be changed by users while debugging, so this condition won't be false on every machine
+ if (CONFIG_TYPE != null)
+ setConfigType(CONFIG_TYPE);
+ }
+
+ @SuppressWarnings("SameParameterValue")
+ private static void setConfigType(ConfigType type)
+ {
+ File confDir = new File("test/conf");
+ try
+ {
+ File tmp = Files.createTempFile("cassandra-conf", ".yaml").toFile();
+ tmp.deleteOnExit();
+ for (String name : type.configs)
+ {
+ try (FileOutputStream out = new FileOutputStream(tmp, true);
+ InputStream in = new FileInputStream(new File(confDir, name)))
+ {
+ in.transferTo(out);
+ }
+ }
+ CassandraRelevantProperties.CASSANDRA_CONFIG.setString(tmp.toURI().toString());
+ } catch (IOException e)
+ {
+ throw new UncheckedIOException(e);
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java b/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java
index efa47b05d7..11833783a3 100644
--- a/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java
+++ b/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java
@@ -49,6 +49,51 @@ import static org.junit.Assert.assertTrue;
public class YamlConfigurationLoaderTest
{
+ @Test
+ public void repairRetryEmpty()
+ {
+ RepairRetrySpec repair_retries = loadRepairRetry(ImmutableMap.of());
+ // repair is empty
+ assertThat(repair_retries.isEnabled()).isFalse();
+ assertThat(repair_retries.isMerkleTreeRetriesEnabled()).isFalse();
+ }
+
+ @Test
+ public void repairRetryInheritance()
+ {
+ RepairRetrySpec repair_retries = loadRepairRetry(ImmutableMap.of("max_attempts", "3"));
+ assertThat(repair_retries.isEnabled()).isTrue();
+ assertThat(repair_retries.getMaxAttempts()).isEqualTo(3);
+ RetrySpec spec = repair_retries.getMerkleTreeResponseSpec();
+ assertThat(spec.isEnabled()).isTrue();
+ assertThat(spec.getMaxAttempts()).isEqualTo(3);
+ }
+
+ @Test
+ public void repairRetryOverride()
+ {
+ RepairRetrySpec repair_retries = loadRepairRetry(ImmutableMap.of(
+ "merkle_tree_response", ImmutableMap.of("max_attempts", 10,
+ "base_sleep_time", "1s",
+ "max_sleep_time", "10s")
+ ));
+ assertThat(repair_retries.isEnabled()).isFalse();
+ assertThat(repair_retries.getMaxAttempts()).isNull();
+ assertThat(repair_retries.baseSleepTime).isEqualTo(RetrySpec.DEFAULT_BASE_SLEEP);
+ assertThat(repair_retries.maxSleepTime).isEqualTo(RetrySpec.DEFAULT_MAX_SLEEP);
+
+ RetrySpec spec = repair_retries.getMerkleTreeResponseSpec();
+ assertThat(spec.isEnabled()).isTrue();
+ assertThat(spec.maxAttempts).isEqualTo(10);
+ assertThat(spec.baseSleepTime).isEqualTo(RetrySpec.DEFAULT_MAX_SLEEP);
+ assertThat(spec.maxSleepTime).isEqualTo(new DurationSpec.LongMillisecondsBound("10s"));
+ }
+
+ private static RepairRetrySpec loadRepairRetry(Map<String, Object> map)
+ {
+ return YamlConfigurationLoader.fromMap(ImmutableMap.of("repair", ImmutableMap.of("retries", map)), true, Config.class).repair.retries;
+ }
+
@Test
public void validateTypes()
{
diff --git a/test/unit/org/apache/cassandra/db/ReadCommandTest.java b/test/unit/org/apache/cassandra/db/ReadCommandTest.java
index f1efe97722..46eb5e7078 100644
--- a/test/unit/org/apache/cassandra/db/ReadCommandTest.java
+++ b/test/unit/org/apache/cassandra/db/ReadCommandTest.java
@@ -1332,14 +1332,14 @@ public class ReadCommandTest
// check for sessions which have exceeded timeout and been purged
Range<Token> range = new Range<>(cfs.metadata().partitioner.getMinimumToken(),
cfs.metadata().partitioner.getRandomToken());
- ActiveRepairService.instance.registerParentRepairSession(pendingSession,
- REPAIR_COORDINATOR,
- Lists.newArrayList(cfs),
- Sets.newHashSet(range),
- true,
- repairedAt,
- true,
- PreviewKind.NONE);
+ ActiveRepairService.instance().registerParentRepairSession(pendingSession,
+ REPAIR_COORDINATOR,
+ Lists.newArrayList(cfs),
+ Sets.newHashSet(range),
+ true,
+ repairedAt,
+ true,
+ PreviewKind.NONE);
LocalSessionAccessor.prepareUnsafe(pendingSession, null, Sets.newHashSet(REPAIR_COORDINATOR));
}
diff --git a/test/unit/org/apache/cassandra/db/compaction/AbstractPendingRepairTest.java b/test/unit/org/apache/cassandra/db/compaction/AbstractPendingRepairTest.java
index b0bb23cc64..ff6fd203de 100644
--- a/test/unit/org/apache/cassandra/db/compaction/AbstractPendingRepairTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/AbstractPendingRepairTest.java
@@ -57,7 +57,7 @@ public class AbstractPendingRepairTest extends AbstractRepairTest
public static void setupClass()
{
SchemaLoader.prepareServer();
- ARS = ActiveRepairService.instance;
+ ARS = ActiveRepairService.instance();
LocalSessionAccessor.startup();
// cutoff messaging service
diff --git a/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java b/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java
index d3112d8ad4..97da2a4076 100644
--- a/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java
@@ -111,11 +111,11 @@ public class AntiCompactionTest
private void registerParentRepairSession(TimeUUID sessionID, Iterable<Range<Token>> ranges, long repairedAt, TimeUUID pendingRepair) throws IOException
{
- ActiveRepairService.instance.registerParentRepairSession(sessionID,
- InetAddressAndPort.getByName("10.0.0.1"),
- Lists.newArrayList(cfs), ImmutableSet.copyOf(ranges),
- pendingRepair != null || repairedAt != UNREPAIRED_SSTABLE,
- repairedAt, true, PreviewKind.NONE);
+ ActiveRepairService.instance().registerParentRepairSession(sessionID,
+ InetAddressAndPort.getByName("10.0.0.1"),
+ Lists.newArrayList(cfs), ImmutableSet.copyOf(ranges),
+ pendingRepair != null || repairedAt != UNREPAIRED_SSTABLE,
+ repairedAt, true, PreviewKind.NONE);
}
private static RangesAtEndpoint atEndpoint(Collection<Range<Token>> full, Collection<Range<Token>> trans)
diff --git a/test/unit/org/apache/cassandra/db/compaction/CancelCompactionsTest.java b/test/unit/org/apache/cassandra/db/compaction/CancelCompactionsTest.java
index ebddac7bf3..ba054eac4c 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CancelCompactionsTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CancelCompactionsTest.java
@@ -261,7 +261,7 @@ public class CancelCompactionsTest extends CQLTester
Range<Token> range = new Range<>(token(-1), token(49));
TimeUUID prsid = nextTimeUUID();
- ActiveRepairService.instance.registerParentRepairSession(prsid, InetAddressAndPort.getLocalHost(), Collections.singletonList(cfs), Collections.singleton(range), true, 1, true, PreviewKind.NONE);
+ ActiveRepairService.instance().registerParentRepairSession(prsid, InetAddressAndPort.getLocalHost(), Collections.singletonList(cfs), Collections.singleton(range), true, 1, true, PreviewKind.NONE);
InetAddressAndPort local = FBUtilities.getBroadcastAddressAndPort();
RangesAtEndpoint rae = RangesAtEndpoint.builder(local).add(new Replica(local, range, true)).build();
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerPendingRepairTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerPendingRepairTest.java
index 76bbd88e3f..62a1258043 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerPendingRepairTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerPendingRepairTest.java
@@ -262,7 +262,7 @@ public class CompactionStrategyManagerPendingRepairTest extends AbstractPendingR
Assert.assertFalse(hasTransientStrategiesFor(repairID));
// sstable should have pendingRepair cleared, and repairedAt set correctly
- long expectedRepairedAt = ActiveRepairService.instance.getParentRepairSession(repairID).repairedAt;
+ long expectedRepairedAt = ActiveRepairService.instance().getParentRepairSession(repairID).repairedAt;
Assert.assertFalse(sstable.isPendingRepair());
Assert.assertTrue(sstable.isRepaired());
Assert.assertEquals(expectedRepairedAt, sstable.getSSTableMetadata().repairedAt);
diff --git a/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java
index e9e2d42ab4..d56003ae2a 100644
--- a/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java
@@ -66,6 +66,7 @@ import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.schema.MockSchema;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.utils.Clock;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.TimeUUID;
@@ -208,16 +209,16 @@ public class LeveledCompactionStrategyTest
Range<Token> range = new Range<>(Util.token(""), Util.token(""));
long gcBefore = keyspace.getColumnFamilyStore(CF_STANDARDDLEVELED).gcBefore(FBUtilities.nowInSeconds());
TimeUUID parentRepSession = nextTimeUUID();
- ActiveRepairService.instance.registerParentRepairSession(parentRepSession,
- FBUtilities.getBroadcastAddressAndPort(),
- Arrays.asList(cfs),
- Arrays.asList(range),
- false,
- ActiveRepairService.UNREPAIRED_SSTABLE,
- true,
- PreviewKind.NONE);
+ ActiveRepairService.instance().registerParentRepairSession(parentRepSession,
+ FBUtilities.getBroadcastAddressAndPort(),
+ Arrays.asList(cfs),
+ Arrays.asList(range),
+ false,
+ ActiveRepairService.UNREPAIRED_SSTABLE,
+ true,
+ PreviewKind.NONE);
RepairJobDesc desc = new RepairJobDesc(parentRepSession, nextTimeUUID(), KEYSPACE1, CF_STANDARDDLEVELED, Arrays.asList(range));
- Validator validator = new Validator(new ValidationState(desc, FBUtilities.getBroadcastAddressAndPort()), gcBefore, PreviewKind.NONE);
+ Validator validator = new Validator(new ValidationState(Clock.Global.clock(), desc, FBUtilities.getBroadcastAddressAndPort()), gcBefore, PreviewKind.NONE);
ValidationManager.instance.submitValidation(cfs, validator).get();
}
diff --git a/test/unit/org/apache/cassandra/db/repair/AbstractPendingAntiCompactionTest.java b/test/unit/org/apache/cassandra/db/repair/AbstractPendingAntiCompactionTest.java
index 79be4c8643..c96b85ff77 100644
--- a/test/unit/org/apache/cassandra/db/repair/AbstractPendingAntiCompactionTest.java
+++ b/test/unit/org/apache/cassandra/db/repair/AbstractPendingAntiCompactionTest.java
@@ -76,7 +76,7 @@ public abstract class AbstractPendingAntiCompactionTest
{
SchemaLoader.prepareServer();
local = InetAddressAndPort.getByName("127.0.0.1");
- ActiveRepairService.instance.consistent.local.start();
+ ActiveRepairService.instance().consistent.local.start();
}
@Before
diff --git a/test/unit/org/apache/cassandra/db/repair/CompactionManagerGetSSTablesForValidationTest.java b/test/unit/org/apache/cassandra/db/repair/CompactionManagerGetSSTablesForValidationTest.java
index bcf75827b0..011db44d09 100644
--- a/test/unit/org/apache/cassandra/db/repair/CompactionManagerGetSSTablesForValidationTest.java
+++ b/test/unit/org/apache/cassandra/db/repair/CompactionManagerGetSSTablesForValidationTest.java
@@ -30,6 +30,7 @@ import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
+import org.apache.cassandra.repair.SharedContext;
import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
import org.apache.cassandra.repair.state.ValidationState;
import org.apache.cassandra.schema.TableMetadata;
@@ -46,6 +47,7 @@ import org.apache.cassandra.repair.RepairJobDesc;
import org.apache.cassandra.repair.Validator;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.utils.Clock;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.TimeUUID;
@@ -105,14 +107,14 @@ public class CompactionManagerGetSSTablesForValidationTest
{
sessionID = nextTimeUUID();
Range<Token> range = new Range<>(MT, MT);
- ActiveRepairService.instance.registerParentRepairSession(sessionID,
- coordinator,
- Lists.newArrayList(cfs),
- Sets.newHashSet(range),
- incremental,
- incremental ? System.currentTimeMillis() : ActiveRepairService.UNREPAIRED_SSTABLE,
- true,
- PreviewKind.NONE);
+ ActiveRepairService.instance().registerParentRepairSession(sessionID,
+ coordinator,
+ Lists.newArrayList(cfs),
+ Sets.newHashSet(range),
+ incremental,
+ incremental ? System.currentTimeMillis() : ActiveRepairService.UNREPAIRED_SSTABLE,
+ true,
+ PreviewKind.NONE);
desc = new RepairJobDesc(sessionID, nextTimeUUID(), ks, tbl, singleton(range));
}
@@ -141,8 +143,8 @@ public class CompactionManagerGetSSTablesForValidationTest
modifySSTables();
// get sstables for repair
- Validator validator = new Validator(new ValidationState(desc, coordinator), FBUtilities.nowInSeconds(), true, PreviewKind.NONE);
- Set<SSTableReader> sstables = Sets.newHashSet(getSSTablesToValidate(cfs, validator.desc.ranges, validator.desc.parentSessionId, validator.isIncremental));
+ Validator validator = new Validator(new ValidationState(Clock.Global.clock(), desc, coordinator), FBUtilities.nowInSeconds(), true, PreviewKind.NONE);
+ Set<SSTableReader> sstables = Sets.newHashSet(getSSTablesToValidate(cfs, SharedContext.Global.instance, validator.desc.ranges, validator.desc.parentSessionId, validator.isIncremental));
Assert.assertNotNull(sstables);
Assert.assertEquals(1, sstables.size());
Assert.assertTrue(sstables.contains(pendingRepair));
@@ -156,8 +158,8 @@ public class CompactionManagerGetSSTablesForValidationTest
modifySSTables();
// get sstables for repair
- Validator validator = new Validator(new ValidationState(desc, coordinator), FBUtilities.nowInSeconds(), false, PreviewKind.NONE);
- Set<SSTableReader> sstables = Sets.newHashSet(getSSTablesToValidate(cfs, validator.desc.ranges, validator.desc.parentSessionId, validator.isIncremental));
+ Validator validator = new Validator(new ValidationState(Clock.Global.clock(), desc, coordinator), FBUtilities.nowInSeconds(), false, PreviewKind.NONE);
+ Set<SSTableReader> sstables = Sets.newHashSet(getSSTablesToValidate(cfs, SharedContext.Global.instance, validator.desc.ranges, validator.desc.parentSessionId, validator.isIncremental));
Assert.assertNotNull(sstables);
Assert.assertEquals(2, sstables.size());
Assert.assertTrue(sstables.contains(pendingRepair));
@@ -172,8 +174,8 @@ public class CompactionManagerGetSSTablesForValidationTest
modifySSTables();
// get sstables for repair
- Validator validator = new Validator(new ValidationState(desc, coordinator), FBUtilities.nowInSeconds(), false, PreviewKind.NONE);
- Set<SSTableReader> sstables = Sets.newHashSet(getSSTablesToValidate(cfs, validator.desc.ranges, validator.desc.parentSessionId, validator.isIncremental));
+ Validator validator = new Validator(new ValidationState(Clock.Global.clock(), desc, coordinator), FBUtilities.nowInSeconds(), false, PreviewKind.NONE);
+ Set<SSTableReader> sstables = Sets.newHashSet(getSSTablesToValidate(cfs, SharedContext.Global.instance, validator.desc.ranges, validator.desc.parentSessionId, validator.isIncremental));
Assert.assertNotNull(sstables);
Assert.assertEquals(3, sstables.size());
Assert.assertTrue(sstables.contains(pendingRepair));
diff --git a/test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionTest.java b/test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionTest.java
index d4a2b991d4..4960bfd188 100644
--- a/test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionTest.java
+++ b/test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionTest.java
@@ -142,7 +142,7 @@ public class PendingAntiCompactionTest extends AbstractPendingAntiCompactionTest
// create a session so the anti compaction can fine it
TimeUUID sessionID = nextTimeUUID();
- ActiveRepairService.instance.registerParentRepairSession(sessionID, InetAddressAndPort.getLocalHost(), tables, ranges, true, 1, true, PreviewKind.NONE);
+ ActiveRepairService.instance().registerParentRepairSession(sessionID, InetAddressAndPort.getLocalHost(), tables, ranges, true, 1, true, PreviewKind.NONE);
PendingAntiCompaction pac;
ExecutorService executor = Executors.newSingleThreadExecutor();
@@ -370,13 +370,13 @@ public class PendingAntiCompactionTest extends AbstractPendingAntiCompactionTest
PendingAntiCompaction.AcquisitionCallable acquisitionCallable = new PendingAntiCompaction.AcquisitionCallable(cfs, FULL_RANGE, nextTimeUUID(), 0, 0);
PendingAntiCompaction.AcquireResult result = acquisitionCallable.call();
TimeUUID sessionID = nextTimeUUID();
- ActiveRepairService.instance.registerParentRepairSession(sessionID,
- InetAddressAndPort.getByName("127.0.0.1"),
- Lists.newArrayList(cfs),
- FULL_RANGE,
- true,0,
- true,
- PreviewKind.NONE);
+ ActiveRepairService.instance().registerParentRepairSession(sessionID,
+ InetAddressAndPort.getByName("127.0.0.1"),
+ Lists.newArrayList(cfs),
+ FULL_RANGE,
+ true, 0,
+ true,
+ PreviewKind.NONE);
CompactionManager.instance.performAnticompaction(result.cfs, atEndpoint(FULL_RANGE, NO_RANGES), result.refs, result.txn, sessionID, () -> false);
}
@@ -390,13 +390,13 @@ public class PendingAntiCompactionTest extends AbstractPendingAntiCompactionTest
PendingAntiCompaction.AcquisitionCallable acquisitionCallable = new PendingAntiCompaction.AcquisitionCallable(cfs, FULL_RANGE, nextTimeUUID(), 0, 0);
PendingAntiCompaction.AcquireResult result = acquisitionCallable.call();
TimeUUID sessionID = nextTimeUUID();
- ActiveRepairService.instance.registerParentRepairSession(sessionID,
- InetAddressAndPort.getByName("127.0.0.1"),
- Lists.newArrayList(cfs),
- FULL_RANGE,
- true,0,
- true,
- PreviewKind.NONE);
+ ActiveRepairService.instance().registerParentRepairSession(sessionID,
+ InetAddressAndPort.getByName("127.0.0.1"),
+ Lists.newArrayList(cfs),
+ FULL_RANGE,
+ true, 0,
+ true,
+ PreviewKind.NONE);
// attempt to anti-compact the sstable in half
SSTableReader sstable = Iterables.getOnlyElement(cfs.getLiveSSTables());
diff --git a/test/unit/org/apache/cassandra/db/streaming/CassandraStreamManagerTest.java b/test/unit/org/apache/cassandra/db/streaming/CassandraStreamManagerTest.java
index eba574ebaa..41dcdcb66b 100644
--- a/test/unit/org/apache/cassandra/db/streaming/CassandraStreamManagerTest.java
+++ b/test/unit/org/apache/cassandra/db/streaming/CassandraStreamManagerTest.java
@@ -261,10 +261,10 @@ public class CassandraStreamManagerTest
@Test
public void checkAvailableDiskSpaceAndCompactionsFailing()
{
- int threshold = ActiveRepairService.instance.getRepairPendingCompactionRejectThreshold();
- ActiveRepairService.instance.setRepairPendingCompactionRejectThreshold(1);
+ int threshold = ActiveRepairService.instance().getRepairPendingCompactionRejectThreshold();
+ ActiveRepairService.instance().setRepairPendingCompactionRejectThreshold(1);
assertFalse(StreamSession.checkAvailableDiskSpaceAndCompactions(createSummaries(), nextTimeUUID(), null, false));
- ActiveRepairService.instance.setRepairPendingCompactionRejectThreshold(threshold);
+ ActiveRepairService.instance().setRepairPendingCompactionRejectThreshold(threshold);
}
private Collection<StreamSummary> createSummaries()
diff --git a/test/unit/org/apache/cassandra/db/virtual/LocalRepairTablesTest.java b/test/unit/org/apache/cassandra/db/virtual/LocalRepairTablesTest.java
index 2a6da1ac69..aec6c1574e 100644
--- a/test/unit/org/apache/cassandra/db/virtual/LocalRepairTablesTest.java
+++ b/test/unit/org/apache/cassandra/db/virtual/LocalRepairTablesTest.java
@@ -39,7 +39,7 @@ import org.apache.cassandra.dht.Token;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.repair.CommonRange;
import org.apache.cassandra.repair.RepairJobDesc;
-import org.apache.cassandra.repair.RepairRunnable;
+import org.apache.cassandra.repair.RepairCoordinator;
import org.apache.cassandra.repair.messages.PrepareMessage;
import org.apache.cassandra.repair.messages.RepairOption;
import org.apache.cassandra.repair.state.Completable;
@@ -52,6 +52,7 @@ import org.apache.cassandra.repair.state.ValidationState;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.utils.Clock;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.TimeUUID;
@@ -75,7 +76,7 @@ public class LocalRepairTablesTest extends CQLTester
@Before
public void cleanupRepairs()
{
- ActiveRepairService.instance.clearLocalRepairState();
+ ActiveRepairService.instance().clearLocalRepairState();
}
@Test
@@ -90,7 +91,7 @@ public class LocalRepairTablesTest extends CQLTester
assertState("repairs", state, CoordinatorState.State.SETUP);
List<ColumnFamilyStore> tables = Collections.singletonList(table());
- RepairRunnable.NeighborsAndRanges neighbors = neighbors();
+ RepairCoordinator.NeighborsAndRanges neighbors = neighbors();
state.phase.start(tables, neighbors);
assertState("repairs", state, CoordinatorState.State.START);
List<List<String>> expectedRanges = neighbors.commonRanges.stream().map(a -> a.ranges.stream().map(Object::toString).collect(Collectors.toList())).collect(Collectors.toList());
@@ -265,9 +266,9 @@ public class LocalRepairTablesTest extends CQLTester
return Schema.instance.getColumnFamilyStoreInstance(Schema.instance.getTableMetadata(ks, name).id);
}
- private static RepairRunnable.NeighborsAndRanges neighbors()
+ private static RepairCoordinator.NeighborsAndRanges neighbors()
{
- return new RepairRunnable.NeighborsAndRanges(false, ADDRESSES, ImmutableList.of(COMMON_RANGE));
+ return new RepairCoordinator.NeighborsAndRanges(false, ADDRESSES, ImmutableList.of(COMMON_RANGE));
}
private static Range<Token> range(long a, long b)
@@ -290,15 +291,15 @@ public class LocalRepairTablesTest extends CQLTester
private static CoordinatorState coordinator()
{
RepairOption options = RepairOption.parse(Collections.emptyMap(), DatabaseDescriptor.getPartitioner());
- CoordinatorState state = new CoordinatorState(0, "test", options);
- ActiveRepairService.instance.register(state);
+ CoordinatorState state = new CoordinatorState(Clock.Global.clock(), 0, "test", options);
+ ActiveRepairService.instance().register(state);
return state;
}
private static SessionState session()
{
CoordinatorState parent = coordinator();
- SessionState state = new SessionState(parent.id, REPAIR_KS, new String[]{ REPAIR_TABLE }, COMMON_RANGE);
+ SessionState state = new SessionState(Clock.Global.clock(), parent.id, REPAIR_KS, new String[]{ REPAIR_TABLE }, COMMON_RANGE);
parent.register(state);
return state;
}
@@ -306,7 +307,7 @@ public class LocalRepairTablesTest extends CQLTester
private static JobState job()
{
SessionState session = session();
- JobState state = new JobState(new RepairJobDesc(session.parentRepairSession, session.id, session.keyspace, session.cfnames[0], session.commonRange.ranges), session.commonRange.endpoints);
+ JobState state = new JobState(Clock.Global.clock(), new RepairJobDesc(session.parentRepairSession, session.id, session.keyspace, session.cfnames[0], session.commonRange.ranges), session.commonRange.endpoints);
session.register(state);
return state;
}
@@ -315,7 +316,7 @@ public class LocalRepairTablesTest extends CQLTester
{
JobState job = job(); // job isn't needed but makes getting the descriptor easier
ParticipateState participate = participate();
- ValidationState state = new ValidationState(job.desc, ADDRESSES.stream().findFirst().get());
+ ValidationState state = new ValidationState(Clock.Global.clock(), job.desc, ADDRESSES.stream().findFirst().get());
participate.register(state);
return state;
}
@@ -323,8 +324,8 @@ public class LocalRepairTablesTest extends CQLTester
private ParticipateState participate()
{
List<Range<Token>> ranges = Arrays.asList(new Range<>(new Murmur3Partitioner.LongToken(0), new Murmur3Partitioner.LongToken(42)));
- ParticipateState state = new ParticipateState(FBUtilities.getBroadcastAddressAndPort(), new PrepareMessage(TimeUUID.Generator.nextTimeUUID(), Collections.emptyList(), ranges, true, 42, true, PreviewKind.ALL));
- ActiveRepairService.instance.register(state);
+ ParticipateState state = new ParticipateState(Clock.Global.clock(), FBUtilities.getBroadcastAddressAndPort(), new PrepareMessage(TimeUUID.Generator.nextTimeUUID(), Collections.emptyList(), ranges, true, 42, true, PreviewKind.ALL));
+ ActiveRepairService.instance().register(state);
return state;
}
diff --git a/test/unit/org/apache/cassandra/index/sai/functional/CompactionTest.java b/test/unit/org/apache/cassandra/index/sai/functional/CompactionTest.java
index e13cca5108..da9a051f4f 100644
--- a/test/unit/org/apache/cassandra/index/sai/functional/CompactionTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/functional/CompactionTest.java
@@ -96,14 +96,14 @@ public class CompactionTest extends SAITester
{
InetAddressAndPort endpoint = InetAddressAndPort.getByName("10.0.0.1");
TimeUUID parentRepairSession = TimeUUID.Generator.nextTimeUUID();
- ActiveRepairService.instance.registerParentRepairSession(parentRepairSession,
- endpoint,
- Lists.newArrayList(cfs),
- Collections.singleton(range),
- true,
- 1000,
- false,
- PreviewKind.NONE);
+ ActiveRepairService.instance().registerParentRepairSession(parentRepairSession,
+ endpoint,
+ Lists.newArrayList(cfs),
+ Collections.singleton(range),
+ true,
+ 1000,
+ false,
+ PreviewKind.NONE);
RangesAtEndpoint replicas = RangesAtEndpoint.builder(endpoint).add(Replica.fullReplica(endpoint, range)).build();
CompactionManager.instance.performAnticompaction(cfs, replicas, refs, txn, parentRepairSession, () -> false);
}
diff --git a/test/unit/org/apache/cassandra/repair/AbstractRepairTest.java b/test/unit/org/apache/cassandra/repair/AbstractRepairTest.java
index eb4ca9f445..a4c1304bf3 100644
--- a/test/unit/org/apache/cassandra/repair/AbstractRepairTest.java
+++ b/test/unit/org/apache/cassandra/repair/AbstractRepairTest.java
@@ -88,14 +88,14 @@ public abstract class AbstractRepairTest
... 3106 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org