You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by pa...@apache.org on 2026/04/08 15:20:32 UTC
(cassandra) branch cassandra-5.0 updated: Backport Automated Repair Inside Cassandra (CEP-37)
This is an automated email from the ASF dual-hosted git repository.
paulo pushed a commit to branch cassandra-5.0
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/cassandra-5.0 by this push:
new 9500eb129b Backport Automated Repair Inside Cassandra (CEP-37)
9500eb129b is described below
commit 9500eb129bd61b2eaec78df3b9a7a5ebfca91c92
Author: Paulo Motta <pa...@apache.org>
AuthorDate: Mon Mar 23 15:49:38 2026 -0400
Backport Automated Repair Inside Cassandra (CEP-37)
Includes bug fixes and features:
- Improved observability in AutoRepair (CASSANDRA-20581)
- Stop repair scheduler if two major versions detected (CASSANDRA-20048)
- Safeguard Full repair against disk protection (CASSANDRA-20045)
- Stop AutoRepair monitoring thread upon shutdown (CASSANDRA-20623)
- Fix race condition in auto-repair scheduler (CASSANDRA-20265)
- Minimum repair task duration setting (CASSANDRA-20160)
- Preview_repaired auto-repair type (CASSANDRA-20046)
- Gate auto-repair behind cassandra.autorepair.enable JVM property
- Add cassandra.autorepair.check_min_version to gate minimum version enforcement
- Prevent auto-repair from running if any node is below 5.0.7
- Make system_distributed auto-repair schema conditional on feature being enabled
- Add user-friendly errors for disabled auto-repair and schema incompatibility
patch by Paulo Motta; reviewed by Andy Tolbert, Jaydeepkumar Chovatia for CASSANDRA-21138
Co-Authored-By: Andy Tolbert <an...@apple.com>
Co-Authored-By: Chris Lohfink <cl...@netflix.com>
Co-Authored-By: Francisco Guerrero <fr...@apache.org>
Co-Authored-By: Himanshu Jindal <hi...@amazon.com>
Co-Authored-By: Jaydeepkumar Chovatia <jc...@uber.com>
Co-Authored-By: Kristijonas Zalys <kz...@uber.com>
Co-Authored-By: jaydeepkumar1984 <ch...@gmail.com>
---
.build/run-tests.sh | 8 +-
CHANGES.txt | 10 +
NEWS.txt | 39 +
conf/cassandra.yaml | 173 +++
conf/cassandra_latest.yaml | 164 +++
doc/modules/cassandra/nav.adoc | 3 +-
.../pages/managing/operating/auto_repair.adoc | 473 +++++++
.../config/CassandraRelevantProperties.java | 4 +
src/java/org/apache/cassandra/config/Config.java | 9 +
.../cassandra/config/DatabaseDescriptor.java | 30 +
.../apache/cassandra/cql3/UntypedResultSet.java | 8 +
.../cql3/statements/ModificationStatement.java | 2 +-
.../cql3/statements/schema/TableAttributes.java | 9 +
.../cassandra/db/compaction/CompactionManager.java | 4 +-
.../db/streaming/CassandraStreamReceiver.java | 8 +-
src/java/org/apache/cassandra/gms/Gossiper.java | 2 +-
.../cassandra/io/sstable/format/SSTableReader.java | 13 +
.../cassandra/locator/InetAddressAndPort.java | 56 +-
.../cassandra/metrics/AutoRepairMetrics.java | 254 ++++
.../metrics/AutoRepairMetricsManager.java | 37 +
.../metrics/CassandraMetricsRegistry.java | 67 +-
.../apache/cassandra/metrics/KeyspaceMetrics.java | 11 +
.../org/apache/cassandra/metrics/TableMetrics.java | 40 +-
.../apache/cassandra/repair/PreviewRepairTask.java | 18 +-
.../apache/cassandra/repair/RepairCoordinator.java | 8 +-
.../cassandra/repair/RepairMessageVerbHandler.java | 7 +
.../apache/cassandra/repair/ValidationManager.java | 5 +
.../cassandra/repair/autorepair/AutoRepair.java | 629 +++++++++
.../repair/autorepair/AutoRepairConfig.java | 616 +++++++++
.../repair/autorepair/AutoRepairState.java | 386 ++++++
.../repair/autorepair/AutoRepairUtils.java | 1409 ++++++++++++++++++++
.../autorepair/FixedSplitTokenRangeSplitter.java | 154 +++
.../autorepair/IAutoRepairTokenRangeSplitter.java | 71 +
.../autorepair/KeyspaceRepairAssignments.java | 53 +
.../repair/autorepair/KeyspaceRepairPlan.java | 105 ++
.../repair/autorepair/PrioritizedRepairPlan.java | 148 ++
.../repair/autorepair/RepairAssignment.java | 94 ++
.../autorepair/RepairAssignmentIterator.java | 84 ++
.../autorepair/RepairTokenRangeSplitter.java | 810 +++++++++++
.../repair/consistent/SyncStatSummary.java | 22 +-
.../apache/cassandra/schema/AutoRepairParams.java | 187 +++
.../apache/cassandra/schema/SchemaConstants.java | 2 +-
.../apache/cassandra/schema/SchemaKeyspace.java | 32 +-
.../schema/SystemDistributedKeyspace.java | 63 +-
.../org/apache/cassandra/schema/TableParams.java | 36 +-
.../cassandra/service/ActiveRepairService.java | 34 +-
.../service/ActiveRepairServiceMBean.java | 4 +
.../cassandra/service/AutoRepairService.java | 349 +++++
.../cassandra/service/AutoRepairServiceMBean.java | 79 ++
.../apache/cassandra/service/StorageService.java | 92 +-
.../cassandra/service/StorageServiceMBean.java | 6 +
src/java/org/apache/cassandra/tools/NodeProbe.java | 145 ++
src/java/org/apache/cassandra/tools/NodeTool.java | 4 +
.../cassandra/tools/nodetool/AutoRepairStatus.java | 79 ++
.../tools/nodetool/GetAutoRepairConfig.java | 45 +
.../tools/nodetool/SSTableRepairedSet.java | 108 ++
.../tools/nodetool/SetAutoRepairConfig.java | 201 +++
.../cassandra/distributed/impl/Instance.java | 2 +
.../test/repair/AutoRepairFlagToggleTest.java | 232 ++++
...owParallelReplicaRepairAcrossSchedulesTest.java | 134 ++
.../repair/AutoRepairSchedulerStatsHelper.java | 262 ++++
.../AutoRepairSchedulerStatsNoVNodesTest.java | 54 +
.../repair/AutoRepairSchedulerStatsVNodesTest.java | 54 +
.../test/repair/AutoRepairSchedulerTest.java | 206 +++
.../test/repair/AutoRepairTablePropertyTest.java | 168 +++
.../AutoRepairDisabledSchemaUpgradeTest.java | 120 ++
test/unit/org/apache/cassandra/Util.java | 16 +
.../config/DatabaseDescriptorRefTest.java | 11 +
.../config/YamlConfigurationLoaderTest.java | 30 +-
.../cql3/SystemKeyspaceTablesNamesTest.java | 4 +-
.../db/streaming/CassandraStreamReceiverTest.java | 153 +++
.../cassandra/repair/RepairCoordinatorTest.java | 61 +
.../autorepair/AutoRepairConfigRepairTypeTest.java | 60 +
.../repair/autorepair/AutoRepairConfigTest.java | 518 +++++++
.../repair/autorepair/AutoRepairKeyspaceTest.java | 71 +
.../repair/autorepair/AutoRepairMetricsTest.java | 92 ++
.../autorepair/AutoRepairParameterizedTest.java | 934 +++++++++++++
.../repair/autorepair/AutoRepairShutdownTest.java | 75 ++
.../autorepair/AutoRepairStateFactoryTest.java | 68 +
.../repair/autorepair/AutoRepairStateTest.java | 323 +++++
.../repair/autorepair/AutoRepairTest.java | 162 +++
.../repair/autorepair/AutoRepairUtilsTest.java | 712 ++++++++++
.../FixedSplitTokenRangeSplitterHelper.java | 246 ++++
.../FixedSplitTokenRangeSplitterNoVNodesTest.java | 84 ++
.../FixedSplitTokenRangeSplitterVNodesTest.java | 84 ++
.../autorepair/PrioritizedRepairPlanTest.java | 182 +++
.../autorepair/RepairTokenRangeSplitterTest.java | 604 +++++++++
.../repair/autorepair/SSTableRepairedAtTest.java | 175 +++
.../cassandra/service/ActiveRepairServiceTest.java | 105 +-
.../service/AutoRepairServiceBasicTest.java | 160 +++
.../service/AutoRepairServiceRepairTypeTest.java | 88 ++
.../service/AutoRepairServiceSetterTest.java | 153 +++
.../service/StorageServiceServerTest.java | 163 ++-
.../apache/cassandra/tools/JMXStandardsTest.java | 7 +
.../tools/nodetool/AutoRepairStatusTest.java | 108 ++
.../tools/nodetool/SSTableRepairedSetTest.java | 132 ++
.../tools/nodetool/SetAutoRepairConfigTest.java | 320 +++++
.../tools/nodetool/TableHistogramsTest.java | 2 +-
98 files changed, 14524 insertions(+), 120 deletions(-)
diff --git a/.build/run-tests.sh b/.build/run-tests.sh
index 344f3d9e18..68ecbff7f6 100755
--- a/.build/run-tests.sh
+++ b/.build/run-tests.sh
@@ -188,13 +188,13 @@ _build_all_dtest_jars() {
if [ -d ${TMP_DIR}/cassandra-dtest-jars/.git ] && [ "https://github.com/apache/cassandra.git" == "$(git -C ${TMP_DIR}/cassandra-dtest-jars remote get-url origin)" ] ; then
echo "Reusing ${TMP_DIR}/cassandra-dtest-jars for past branch dtest jars"
if [ "x" == "x${OFFLINE}" ] ; then
- until git -C ${TMP_DIR}/cassandra-dtest-jars fetch --quiet origin ; do echo "git -C ${TMP_DIR}/cassandra-dtest-jars fetch failed… trying again… " ; done
+ until git -C ${TMP_DIR}/cassandra-dtest-jars fetch --quiet --tags origin ; do echo "git -C ${TMP_DIR}/cassandra-dtest-jars fetch failed… trying again… " ; done
fi
else
echo "Cloning cassandra to ${TMP_DIR}/cassandra-dtest-jars for past branch dtest jars"
rm -fR ${TMP_DIR}/cassandra-dtest-jars
pushd $TMP_DIR >/dev/null
- until git clone --quiet --depth 1 --no-single-branch https://github.com/apache/cassandra.git cassandra-dtest-jars ; do echo "git clone failed… trying again… " ; done
+ until git clone --quiet --depth 1 --no-single-branch --tags https://github.com/apache/cassandra.git cassandra-dtest-jars ; do echo "git clone failed… trying again… " ; done
popd >/dev/null
fi
@@ -202,7 +202,9 @@ _build_all_dtest_jars() {
[ "${java_version}" -eq 11 ] && export CASSANDRA_USE_JDK11=true
pushd ${TMP_DIR}/cassandra-dtest-jars >/dev/null
- for branch in cassandra-4.0 cassandra-4.1 cassandra-5.0 ; do
+ # Note: cassandra-5.0.7 tag is used instead of cassandra-5.0 branch to enable
+ # testing upgrades from 5.0.7 to the current local build for autorepair feature
+ for branch in cassandra-4.0 cassandra-4.1 cassandra-5.0.7 ; do
git clean -qxdff && git reset --hard HEAD || echo "failed to reset/clean ${TMP_DIR}/cassandra-dtest-jars… continuing…"
git checkout --quiet $branch
dtest_jar_version=$(grep 'property\s*name=\"base.version\"' build.xml |sed -ne 's/.*value=\"\([^"]*\)\".*/\1/p')
diff --git a/CHANGES.txt b/CHANGES.txt
index 5ff7427a39..9a575f7f02 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,8 +1,18 @@
5.0.8
+ * Backport Automated Repair Inside Cassandra for CEP-37 (CASSANDRA-21138)
* Update cassandra-stress to support TLS 1.3 by default by auto-negotiation (CASSANDRA-21007)
* Ensure schema created before 2.1 without tableId in folder name can be loaded in SnapshotLoader (CASSANDRA-21173)
Merged from 4.1:
Merged from 4.0:
+Backported from 6.0:
+ * Improved observability in AutoRepair to report both expected vs. actual repair bytes and expected vs. actual keyspaces (CASSANDRA-20581)
+ * Stop repair scheduler if two major versions are detected (CASSANDRA-20048)
+ * AutoRepair: Safeguard Full repair against disk protection (CASSANDRA-20045)
+ * Stop AutoRepair monitoring thread upon Cassandra shutdown (CASSANDRA-20623)
+ * Fix race condition in auto-repair scheduler (CASSANDRA-20265)
+ * Implement minimum repair task duration setting for auto-repair scheduler (CASSANDRA-20160)
+ * Implement preview_repaired auto-repair type (CASSANDRA-20046)
+ * Automated Repair Inside Cassandra for CEP-37 (CASSANDRA-19918)
5.0.7
diff --git a/NEWS.txt b/NEWS.txt
index 1192f4178a..7151e94dcd 100644
--- a/NEWS.txt
+++ b/NEWS.txt
@@ -65,6 +65,45 @@ restore snapshots created with the previous major version using the
'sstableloader' tool. You can upgrade the file format of your snapshots
using the provided 'sstableupgrade' tool.
+5.0.8
+======
+
+New features
+------------
+ - CEP-37 Auto Repair is a fully automated scheduler that provides repair orchestration within Apache Cassandra. This
+ significantly reduces operational overhead by eliminating the need for operators to deploy external tools to submit
+ and manage repairs. See
+ https://cwiki.apache.org/confluence/display/CASSANDRA/CEP-37+Apache+Cassandra+Unified+Repair+Solution for more
+ details on the motivation and design.
+
+Upgrading
+---------
+ - The auto-repair feature requires enabling the JVM property `cassandra.autorepair.enable=true` (add
+ `-Dcassandra.autorepair.enable=true` to JVM options) before starting the node. This property creates the required
+ schema elements for auto-repair, including the auto_repair column in system_schema.tables and system_schema.views,
+ as well as the auto_repair_history and auto_repair_priority tables in system_distributed. After enabling this
+ property, you still need to enable auto-repair scheduling either in cassandra.yaml under the `auto_repair` section
+ or at runtime via JMX.
+
+ Users who do not intend to use auto-repair can leave this property disabled (the default) to maintain schema
+ compatibility with pre-5.0.8 nodes during rolling upgrades. This property must be set consistently across all
+ nodes before startup and cannot be changed at runtime.
+
+ WARNING: This property is non-reversible. Once enabled, it cannot be disabled. Attempting to start a node
+ with `cassandra.autorepair.enable=false` after it was previously enabled will cause the node to fail during
+ initialization due to schema incompatibility (the persisted schema contains auto-repair columns that are not
+ recognized when the property is disabled). To disable auto-repair scheduling after the property has been
+ enabled, use cassandra.yaml or JMX instead of changing the JVM property.
+
+ IMPORTANT: The `cassandra.autorepair.enable` property must be enabled consistently across all nodes in the
+ cluster before any schema changes are made. When some nodes have the property enabled and others do not, the
+ system_distributed keyspace schema generation will differ between nodes (generation 7 with auto-repair vs
+ generation 6 without), causing schema disagreement. This is similar to what happens during a major version
+ upgrade when new system tables are added. Any schema change (e.g. CREATE KEYSPACE) attempted while nodes
+ are in this inconsistent state will time out and schema versions will not converge until all nodes are
+ brought up with the same setting. Once all nodes have the property set consistently, schema will converge
+ automatically.
+
5.0.7
======
diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml
index e09518188a..e3472fc227 100644
--- a/conf/cassandra.yaml
+++ b/conf/cassandra.yaml
@@ -1951,6 +1951,13 @@ report_unconfirmed_repaired_data_mismatches: false
# Materialized views are considered experimental and are not recommended for production use.
materialized_views_enabled: false
+# Specify whether Materialized View mutations are replayed through the write path on streaming, e.g. repair.
+# When enabled, Materialized View data streamed to the destination node will be written into commit log first. When setting to false,
+# the streamed Materialized View data is written into SSTables just the same as normal streaming. The default is true.
+# If this is set to false, streaming will be considerably faster however it's possible that, in extreme situations
+# (losing > quorum # nodes in a replica set), you may have data in your SSTables that never makes it to the Materialized View.
+# materialized_views_on_repair_enabled: true
+
# Enables SASI index creation on this node.
# SASI indexes are considered experimental and are not recommended for production use.
sasi_indexes_enabled: false
@@ -2253,6 +2260,7 @@ drop_compact_storage_enabled: false
# excluded_keyspaces: # comma separated list of keyspaces to exclude from the check
# excluded_tables: # comma separated list of keyspace.table pairs to exclude from the check
+
# This property indicates with what Cassandra major version the storage format will be compatible with.
#
# The chosen storage compatibility mode will determine the versions of the written sstables, commitlogs, hints, etc.
@@ -2281,3 +2289,168 @@ drop_compact_storage_enabled: false
# compatibility mode would no longer toggle behaviors as when it was running in the UPGRADING mode.
#
storage_compatibility_mode: CASSANDRA_4
+
+
+# Prevents preparing a repair session or beginning a repair streaming session if pending compactions is over
+# the given value. Defaults to disabled.
+# reject_repair_compaction_threshold: 1024
+
+# Ratio of disk that must be unused to run repair. It is useful to avoid disks filling up during
+# repair as anti-compaction during repair may contribute to additional space temporarily.
+# For example, setting this to 0.2 means at least 20% of disk must be unused.
+# Set to 0.0 to disable this check. Defaults to 0.0 (disabled) on 5.0 for backward-compatibility.
+# repair_disk_headroom_reject_ratio: 0.0
+
+# Configuration for Auto Repair Scheduler.
+#
+# This feature is disabled by default.
+#
+# NOTE: The auto-repair feature requires enabling the JVM property `cassandra.autorepair.enable=true`.
+#
+# See: https://cassandra.apache.org/doc/latest/cassandra/managing/operating/auto_repair.html for an overview of this
+# feature.
+#
+# auto_repair:
+# # Enable/Disable the auto-repair scheduler.
+# # If set to false, the scheduler thread will not be started.
+# # If set to true, the repair scheduler thread will be created. The thread will
+# # check for secondary configuration available for each repair type (full, incremental,
+# # and preview_repaired), and based on that, it will schedule repairs.
+# enabled: true
+# repair_type_overrides:
+# full:
+# # Enable/Disable full auto-repair
+# enabled: true
+# # Minimum duration between repairing the same node again. This is useful for tiny clusters,
+# # such as clusters with 5 nodes that finish repairs quickly. This means that if the scheduler completes one
+# # round on all nodes in less than this duration, it will not start a new repair round on a given node until
+# # this much time has passed since the last repair completed. Consider increasing to a larger value to reduce
+# # the impact of repairs, however note that one should attempt to run repairs at a smaller interval than
+# # gc_grace_seconds to avoid potential data resurrection.
+# min_repair_interval: 24h
+# token_range_splitter:
+# # Implementation of IAutoRepairTokenRangeSplitter; responsible for splitting token ranges
+# # for repair assignments.
+# #
+# # Out of the box, Cassandra provides org.apache.cassandra.repair.autorepair.{RepairTokenRangeSplitter,
+# # FixedTokenRangeSplitter}.
+# #
+# # - RepairTokenRangeSplitter (default) attempts to intelligently split ranges based on data size and partition
+# # count.
+# # - FixedTokenRangeSplitter splits into fixed ranges based on the 'number_of_subranges' option.
+# # class_name: org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter
+#
+# # Optional parameters can be specified in the form of:
+# # parameters:
+# # param_key1: param_value1
+# parameters:
+# # The target and maximum amount of compressed bytes that should be included in a repair assignment.
+# # This scopes the amount of work involved in a repair and includes the data covering the range being
+# # repaired.
+# bytes_per_assignment: 50GiB
+# # The maximum number of bytes to cover in an individual schedule. This serves as
+# # a mechanism to throttle the work done in each repair cycle. You may reduce this
+# # value if the impact of repairs is causing too much load on the cluster or increase it
+# # if writes outpace the amount of data being repaired. Alternatively, adjust the
+# # min_repair_interval.
+# # This is set to a large value for full repair to attempt to repair all data per repair schedule.
+# max_bytes_per_schedule: 100000GiB
+# incremental:
+# enabled: false
+# # Incremental repairs operate over unrepaired data and should finish quickly. Running incremental repair
+# # frequently keeps the unrepaired set smaller and thus causes repairs to operate over a smaller set of data,
+# # so a more frequent schedule such as 1h is recommended.
+# # NOTE: Please consult
+# # https://cassandra.apache.org/doc/latest/cassandra/managing/operating/auto_repair.html#enabling-ir
+# # for guidance on enabling incremental repair on ane exiting cluster.
+# min_repair_interval: 24h
+# token_range_splitter:
+# parameters:
+# # Configured to attempt repairing 50GiB of compressed data per repair.
+# # This throttles the amount of incremental repair and anticompaction done per schedule after incremental
+# # repairs are turned on.
+# bytes_per_assignment: 50GiB
+# # Restricts the maximum number of bytes to cover in an individual schedule to the configured
+# # max_bytes_per_schedule value (defaults to 100GiB for incremental).
+# # Consider increasing this value if more data is written than this limit within the min_repair_interval.
+# max_bytes_per_schedule: 100GiB
+# preview_repaired:
+# # Performs preview repair over repaired SSTables, useful to detect possible inconsistencies in the repaired
+# # data set.
+# enabled: false
+# min_repair_interval: 24h
+# token_range_splitter:
+# parameters:
+# bytes_per_assignment: 50GiB
+# max_bytes_per_schedule: 100000GiB
+# # Time interval between successive checks to see if ongoing repairs are complete or if it is time to schedule
+# # repairs.
+# repair_check_interval: 5m
+# # Minimum duration for the execution of a single repair task. This prevents the scheduler from overwhelming
+# # the node by scheduling too many repair tasks in a short period of time.
+# repair_task_min_duration: 5s
+# # The scheduler needs to adjust its order when nodes leave the ring. Deleted hosts are tracked in metadata
+# # for a specified duration to ensure they are indeed removed before adjustments are made to the schedule.
+# history_clear_delete_hosts_buffer_interval: 2h
+# # By default repair is disabled if there are mixed major versions detected - which would happen
+# # if a major version upgrade is being performed on the cluster, but a user can enable it using this flag
+# mixed_major_version_repair_enabled: false
+# # NOTE: Each of the below settings can be overridden per repair type under repair_type_overrides
+# global_settings:
+# # If true, attempts to group tables in the same keyspace into one repair; otherwise, each table is repaired
+# # individually.
+# repair_by_keyspace: true
+# # Number of threads to use for each repair job scheduled by the scheduler. Similar to the -j option in nodetool
+# # repair.
+# number_of_repair_threads: 1
+# # Number of nodes running repair in parallel. If parallel_repair_percentage is set, the larger value is used.
+# parallel_repair_count: 3
+# # Percentage of nodes in the cluster running repair in parallel. If parallel_repair_count is set, the larger value
+# # is used.
+# parallel_repair_percentage: 3
+# # Whether to allow a node to take its turn running repair while one or more of its replicas are running repair.
+# # Defaults to false, as running repairs concurrently on replicas can increase load and also cause anticompaction
+# # conflicts while running incremental repair.
+# allow_parallel_replica_repair: false
+# # An addition to allow_parallel_replica_repair that also blocks repairs when replicas (including this node itself)
+# # are repairing in any schedule. For example, if a replica is executing full repairs, a value of false will
+# # prevent starting incremental repairs for this node. Defaults to true and is only evaluated when
+# # allow_parallel_replica_repair is false.
+# allow_parallel_replica_repair_across_schedules: true
+# # Repairs materialized views if true.
+# materialized_view_repair_enabled: false
+# # Delay before starting repairs after a node restarts to avoid repairs starting immediately after a restart.
+# initial_scheduler_delay: 5m
+# # Timeout for retrying stuck repair sessions.
+# repair_session_timeout: 3h
+# # Force immediate repair on new nodes after they join the ring.
+# force_repair_new_node: false
+# # Threshold to skip repairing tables with too many SSTables. Defaults to 10,000 SSTables to avoid penalizing good
+# # tables.
+# sstable_upper_threshold: 50000
+# # Maximum time allowed for repairing one table on a given node. If exceeded, the repair proceeds to the
+# # next table.
+# table_max_repair_time: 6h
+# # Avoid running repairs in specific data centers. By default, repairs run in all data centers. Specify data
+# # centers to exclude in this list. Note that repair sessions will still consider all replicas from excluded
+# # data centers. Useful if you have keyspaces that are not replicated in certain data centers, and you want to
+# # not run repair schedule in certain data centers.
+# ignore_dcs: []
+# # Repair only the primary ranges owned by a node. Equivalent to the -pr option in nodetool repair. Defaults
+# # to true. General advice is to keep this true.
+# repair_primary_token_range_only: true
+# # Maximum number of retries for a repair session.
+# repair_max_retries: 3
+# # Backoff time before retrying a repair session.
+# repair_retry_backoff: 30s
+# token_range_splitter:
+# # Splitter implementation to generate repair assignments. Defaults to RepairTokenRangeSplitter.
+# class_name: org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter
+# parameters:
+# # Maximum number of partitions to include in a repair assignment. Used to reduce number of partitions
+# # present in merkle tree leaf nodes to avoid overstreaming.
+# partitions_per_assignment: 1048576
+# # Maximum number of tables to include in a repair assignment. This reduces the number of repairs,
+# # especially in keyspaces with many tables. The splitter avoids batching tables together if they
+# # exceed other configuration parameters like bytes_per_assignment or partitions_per_assignment.
+# max_tables_per_assignment: 64
diff --git a/conf/cassandra_latest.yaml b/conf/cassandra_latest.yaml
index 0c7a792c8f..956e1e803b 100644
--- a/conf/cassandra_latest.yaml
+++ b/conf/cassandra_latest.yaml
@@ -2248,3 +2248,167 @@ default_secondary_index_enabled: true
# compatibility mode would no longer toggle behaviors as when it was running in the UPGRADING mode.
#
storage_compatibility_mode: NONE
+
+# Prevents preparing a repair session or beginning a repair streaming session if pending compactions is over
+# the given value. Defaults to disabled.
+# reject_repair_compaction_threshold: 1024
+
+# Ratio of disk that must be unused to run repair. It is useful to avoid disks filling up during
+# repair as anti-compaction during repair may contribute to additional space temporarily.
+# For example, setting this to 0.2 means at least 20% of disk must be unused.
+# Set to 0.0 to disable this check. Defaults to 0.0 (disabled) on 5.0 for backward-compatibility.
+# repair_disk_headroom_reject_ratio: 0.0
+
+# Configuration for Auto Repair Scheduler.
+#
+# This feature is disabled by default.
+#
+# NOTE: The auto-repair feature requires enabling the JVM property `cassandra.autorepair.enable=true`.
+#
+# See: https://cassandra.apache.org/doc/latest/cassandra/managing/operating/auto_repair.html for an overview of this
+# feature.
+#
+# auto_repair:
+# # Enable/Disable the auto-repair scheduler.
+# # If set to false, the scheduler thread will not be started.
+# # If set to true, the repair scheduler thread will be created. The thread will
+# # check for secondary configuration available for each repair type (full, incremental,
+# # and preview_repaired), and based on that, it will schedule repairs.
+# enabled: true
+# repair_type_overrides:
+# full:
+# # Enable/Disable full auto-repair
+# enabled: true
+# # Minimum duration between repairing the same node again. This is useful for tiny clusters,
+# # such as clusters with 5 nodes that finish repairs quickly. This means that if the scheduler completes one
+# # round on all nodes in less than this duration, it will not start a new repair round on a given node until
+# # this much time has passed since the last repair completed. Consider increasing to a larger value to reduce
+# # the impact of repairs, however note that one should attempt to run repairs at a smaller interval than
+# # gc_grace_seconds to avoid potential data resurrection.
+# min_repair_interval: 24h
+# token_range_splitter:
+# # Implementation of IAutoRepairTokenRangeSplitter; responsible for splitting token ranges
+# # for repair assignments.
+# #
+# # Out of the box, Cassandra provides org.apache.cassandra.repair.autorepair.{RepairTokenRangeSplitter,
+# # FixedTokenRangeSplitter}.
+# #
+# # - RepairTokenRangeSplitter (default) attempts to intelligently split ranges based on data size and partition
+# # count.
+# # - FixedTokenRangeSplitter splits into fixed ranges based on the 'number_of_subranges' option.
+# # class_name: org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter
+#
+# # Optional parameters can be specified in the form of:
+# # parameters:
+# # param_key1: param_value1
+# parameters:
+# # The target and maximum amount of compressed bytes that should be included in a repair assignment.
+# # This scopes the amount of work involved in a repair and includes the data covering the range being
+# # repaired.
+# bytes_per_assignment: 50GiB
+# # The maximum number of bytes to cover in an individual schedule. This serves as
+# # a mechanism to throttle the work done in each repair cycle. You may reduce this
+# # value if the impact of repairs is causing too much load on the cluster or increase it
+# # if writes outpace the amount of data being repaired. Alternatively, adjust the
+# # min_repair_interval.
+# # This is set to a large value for full repair to attempt to repair all data per repair schedule.
+# max_bytes_per_schedule: 100000GiB
+# incremental:
+# enabled: false
+# # Incremental repairs operate over unrepaired data and should finish quickly. Running incremental repair
+# # frequently keeps the unrepaired set smaller and thus causes repairs to operate over a smaller set of data,
+# # so a more frequent schedule such as 1h is recommended.
+# # NOTE: Please consult
+# # https://cassandra.apache.org/doc/latest/cassandra/managing/operating/auto_repair.html#enabling-ir
+# # for guidance on enabling incremental repair on ane exiting cluster.
+# min_repair_interval: 24h
+# token_range_splitter:
+# parameters:
+# # Configured to attempt repairing 50GiB of compressed data per repair.
+# # This throttles the amount of incremental repair and anticompaction done per schedule after incremental
+# # repairs are turned on.
+# bytes_per_assignment: 50GiB
+# # Restricts the maximum number of bytes to cover in an individual schedule to the configured
+# # max_bytes_per_schedule value (defaults to 100GiB for incremental).
+# # Consider increasing this value if more data is written than this limit within the min_repair_interval.
+# max_bytes_per_schedule: 100GiB
+# preview_repaired:
+# # Performs preview repair over repaired SSTables, useful to detect possible inconsistencies in the repaired
+# # data set.
+# enabled: false
+# min_repair_interval: 24h
+# token_range_splitter:
+# parameters:
+# bytes_per_assignment: 50GiB
+# max_bytes_per_schedule: 100000GiB
+# # Time interval between successive checks to see if ongoing repairs are complete or if it is time to schedule
+# # repairs.
+# repair_check_interval: 5m
+# # Minimum duration for the execution of a single repair task. This prevents the scheduler from overwhelming
+# # the node by scheduling too many repair tasks in a short period of time.
+# repair_task_min_duration: 5s
+# # The scheduler needs to adjust its order when nodes leave the ring. Deleted hosts are tracked in metadata
+# # for a specified duration to ensure they are indeed removed before adjustments are made to the schedule.
+# history_clear_delete_hosts_buffer_interval: 2h
+# # By default repair is disabled if there are mixed major versions detected - which would happen
+# # if a major version upgrade is being performed on the cluster, but a user can enable it using this flag
+# mixed_major_version_repair_enabled: false
+# # NOTE: Each of the below settings can be overridden per repair type under repair_type_overrides
+# global_settings:
+# # If true, attempts to group tables in the same keyspace into one repair; otherwise, each table is repaired
+# # individually.
+# repair_by_keyspace: true
+# # Number of threads to use for each repair job scheduled by the scheduler. Similar to the -j option in nodetool
+# # repair.
+# number_of_repair_threads: 1
+# # Number of nodes running repair in parallel. If parallel_repair_percentage is set, the larger value is used.
+# parallel_repair_count: 3
+# # Percentage of nodes in the cluster running repair in parallel. If parallel_repair_count is set, the larger value
+# # is used.
+# parallel_repair_percentage: 3
+# # Whether to allow a node to take its turn running repair while one or more of its replicas are running repair.
+# # Defaults to false, as running repairs concurrently on replicas can increase load and also cause anticompaction
+# # conflicts while running incremental repair.
+# allow_parallel_replica_repair: false
+# # An addition to allow_parallel_replica_repair that also blocks repairs when replicas (including this node itself)
+# # are repairing in any schedule. For example, if a replica is executing full repairs, a value of false will
+# # prevent starting incremental repairs for this node. Defaults to true and is only evaluated when
+# # allow_parallel_replica_repair is false.
+# allow_parallel_replica_repair_across_schedules: true
+# # Repairs materialized views if true.
+# materialized_view_repair_enabled: false
+# # Delay before starting repairs after a node restarts to avoid repairs starting immediately after a restart.
+# initial_scheduler_delay: 5m
+# # Timeout for retrying stuck repair sessions.
+# repair_session_timeout: 3h
+# # Force immediate repair on new nodes after they join the ring.
+# force_repair_new_node: false
+# # Threshold to skip repairing tables with too many SSTables. Defaults to 10,000 SSTables to avoid penalizing good
+# # tables.
+# sstable_upper_threshold: 50000
+# # Maximum time allowed for repairing one table on a given node. If exceeded, the repair proceeds to the
+# # next table.
+# table_max_repair_time: 6h
+# # Avoid running repairs in specific data centers. By default, repairs run in all data centers. Specify data
+# # centers to exclude in this list. Note that repair sessions will still consider all replicas from excluded
+# # data centers. Useful if you have keyspaces that are not replicated in certain data centers, and you want to
+# # not run repair schedule in certain data centers.
+# ignore_dcs: []
+# # Repair only the primary ranges owned by a node. Equivalent to the -pr option in nodetool repair. Defaults
+# # to true. General advice is to keep this true.
+# repair_primary_token_range_only: true
+# # Maximum number of retries for a repair session.
+# repair_max_retries: 3
+# # Backoff time before retrying a repair session.
+# repair_retry_backoff: 30s
+# token_range_splitter:
+# # Splitter implementation to generate repair assignments. Defaults to RepairTokenRangeSplitter.
+# class_name: org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter
+# parameters:
+# # Maximum number of partitions to include in a repair assignment. Used to reduce number of partitions
+# # present in merkle tree leaf nodes to avoid overstreaming.
+# partitions_per_assignment: 1048576
+# # Maximum number of tables to include in a repair assignment. This reduces the number of repairs,
+# # especially in keyspaces with many tables. The splitter avoids batching tables together if they
+# # exceed other configuration parameters like bytes_per_assignment or partitions_per_assignment.
+# max_tables_per_assignment: 64
diff --git a/doc/modules/cassandra/nav.adoc b/doc/modules/cassandra/nav.adoc
index 7c1a02cfaa..d9a028427d 100644
--- a/doc/modules/cassandra/nav.adoc
+++ b/doc/modules/cassandra/nav.adoc
@@ -84,6 +84,7 @@
**** xref:cassandra:managing/configuration/cass_jvm_options_file.adoc[jvm-* files]
**** xref:cassandra:managing/configuration/configuration.adoc[Liberating cassandra.yaml Parameters' Names from Their Units]
*** xref:cassandra:managing/operating/index.adoc[]
+**** xref:cassandra:managing/operating/auto_repair.adoc[Auto Repair]
**** xref:cassandra:managing/operating/backups.adoc[Backups]
**** xref:cassandra:managing/operating/bloom_filters.adoc[Bloom filters]
**** xref:cassandra:managing/operating/bulk_loading.adoc[Bulk loading]
@@ -125,4 +126,4 @@
*** xref:reference/static.adoc[Static columns]
*** xref:reference/vector-data-type.adoc[Vector data type]
-** xref:integrating/plugins/index.adoc[]
\ No newline at end of file
+** xref:integrating/plugins/index.adoc[]
diff --git a/doc/modules/cassandra/pages/managing/operating/auto_repair.adoc b/doc/modules/cassandra/pages/managing/operating/auto_repair.adoc
new file mode 100644
index 0000000000..7971263b50
--- /dev/null
+++ b/doc/modules/cassandra/pages/managing/operating/auto_repair.adoc
@@ -0,0 +1,473 @@
+= Auto Repair
+:navtitle: Auto Repair
+:description: Auto Repair concepts - How it works, how to configure it, and more.
+:keywords: CEP-37, Repair, Incremental, Preview
+
+Auto Repair is a fully automated scheduler that provides repair orchestration within Apache Cassandra. This
+significantly reduces operational overhead by eliminating the need for operators to deploy external tools to submit and
+manage repairs.
+
+At a high level, a dedicated thread pool is assigned to the repair scheduler. The repair scheduler in Cassandra
+maintains a new replicated table, `system_distributed.auto_repair_history`, which stores the repair history for all
+nodes, including details such as the last repair time. The scheduler selects the node(s) to begin repairs and
+orchestrates the process to ensure that every table and its token ranges are repaired.
+
+The algorithm can run repairs simultaneously on multiple nodes and splits token ranges into subranges, with necessary
+retries to handle transient failures. Automatic repair starts as soon as a Cassandra cluster is launched, similar to
+compaction, and if configured appropriately, does not require human intervention.
+
+The scheduler currently supports Full, Incremental, and Preview repair types with the following features. New repair
+types, such as Paxos repair or other future repair mechanisms, can be integrated with minimal development effort!
+
+
+== Features
+- Capability to run repairs on multiple nodes simultaneously.
+- A default implementation and an interface to override the dataset being repaired per session.
+- Extendable token split algorithms with two implementations readily available:
+. Splits token ranges by placing a cap on the size of data repaired in one session and a maximum cap at the schedule
+level using xref:#repair-token-range-splitter[RepairTokenRangeSplitter] (default).
+. Splits tokens evenly based on the specified number of splits using
+xref:#fixed-split-token-range-splitter[FixedSplitTokenRangeSplitter].
+- A new xref:#table-configuration[CQL table property] (`auto_repair`) offering:
+. The ability to disable specific repair types at the table level, allowing the scheduler to skip one or more tables.
+. Configuring repair priorities for certain tables to prioritize them over others.
+- Dynamic enablement or disablement of the scheduler for each repair type.
+- Configurable settings tailored to each repair job.
+- Rich configuration options for each repair type (e.g., Full, Incremental, or Preview repairs).
+- Comprehensive observability features that allow operators to configure alarms as needed.
+
+== Availability
+
+Auto Repair was introduced in Cassandra 6.0 via CEP-37 and backported to 5.0.8.
+
+In 5.0.8, auto-repair requires enabling the JVM property `-Dcassandra.autorepair.enable=true` before starting the
+node. This property creates the required schema elements (the `auto_repair` column in `system_schema.tables` and
+`system_schema.views`, and the `auto_repair_history` and `auto_repair_priority` tables in `system_distributed`).
+After enabling this property, auto-repair scheduling still needs to be enabled either in `cassandra.yaml` under
+the `auto_repair` section or at runtime via JMX.
+
+WARNING: The `cassandra.autorepair.enable` property is non-reversible. Once enabled, it cannot be disabled.
+See the xref:#upgrading[Upgrading] section in NEWS.txt for details.
+
+== Considerations
+
+Before enabling Auto Repair, please consult the xref:managing/operating/repair.adoc[Repair] guide to establish a base
+understanding of repairs.
+
+=== Full Repair
+
+Full Repairs operate over all data in the token range being repaired. It is therefore important to run full repair
+with a longer schedule and with smaller assignments.
+
+=== Incremental Repair
+
+When enabled from the inception of a cluster, incremental repairs operate over unrepaired data and should finish
+quickly when run more frequently.
+
+Once incremental repair has been run, SSTables will be separated between data that have been incrementally repaired
+and data that have not. Therefore, it is important to continually run incremental repair once it has been enabled so
+newly written data can be compacted together with previously repaired data, allowing overwritten and expired data to
+be eventually purged.
+
+Running incremental repair more frequently keeps the unrepaired set smaller and thus causes repairs to operate over
+a smaller set of data, so a shorter `min_repair_interval` such as `1h` is recommended for new clusters.
+
+==== Enabling Incremental Repair on existing clusters with a large amount of data
+[#enabling-ir]
+One should be careful when enabling incremental repair on a cluster for the first time. While
+xref:#repair-token-range-splitter[RepairTokenRangeSplitter] includes a default configuration to attempt to gracefully
+migrate to incremental repair over time, failure to take proper precaution could overwhelm the cluster with
+xref:managing/operating/compaction/overview.adoc#types-of-compaction[anticompactions].
+
+No matter how one goes about enabling and running incremental repair, it is recommended to run a cycle of full repairs
+for the entire cluster as pre-flight step to running incremental repair. This will put the cluster into a more
+consistent state which will reduce the amount of streaming between replicas when incremental repair initially runs.
+
+If you do not have strong data consistency requirements, one may consider using
+xref:managing/tools/sstable/sstablerepairedset.adoc[nodetool sstablerepairedset] to mark all SSTables as repaired
+before enabling incremental repair scheduling using Auto Repair. This will reduce the burden of initially running
+incremental repair because all existing data will be considered as repaired, so subsequent incremental repairs will
+only run against new data.
+
+If you do have strong data consistency requirements, then one must treat all data as initially unrepaired and run
+incremental repair against it. Consult
+xref:#incremental-repair-defaults[RepairTokenRangeSplitter's Incremental repair defaults].
+
+In particular one should be mindful of the xref:managing/operating/compaction/overview.adoc[compaction strategy]
+you use for your tables and how it might impact incremental repair before running incremental repair for the first
+time:
+
+- *Large SSTables*: When using xref:managing/operating/compaction/stcs.adoc[SizeTieredCompactionStrategy] or any
+ compaction strategy which can create large SSTables including many partitions the amount of
+ xref:managing/operating/compaction/overview.adoc#types-of-compaction[anticompaction] that might be required could be
+ excessive. Using a small `bytes_per_assignment` might contribute to repeated anticompactions over the same
+ unrepaired data.
+- *Partitions overlapping many SSTables*: If partitions overlap between many SSTables, the amount of SSTables included
+ in a repair might be large. Therefore it is important to consider that many SSTables may be included in a repair
+ session and must all be anticompacted. xref:managing/operating/compaction/lcs.adoc[LeveledCompactionStrategy] is less
+ susceptible to this issue as it prevents overlapping of partitions within levels outside of L0, but if SSTables
+ start accumulating in L0 between incremental repairs, the cost of anticompaction will increase.
+ xref:managing/operating/compaction/ucs#sharding[UnifiedCompactionStrategy's sharding] can also be used to avoid
+ partitions overlapping SSTables.
+
+The xref:#repair-token-range-splitter[token_range_splitter] configuration for incremental repair includes a default
+configuration that attempts to conservatively migrate 100GiB of compressed data every day per node. Depending on
+requirements, data set and capability of a cluster's hardware, one may consider tuning these values to be more
+aggressive or conservative.
+
+=== Previewing Repaired Data
+
+The `preview_repaired` repair type executes repairs over the repaired data set to detect possible data inconsistencies.
+
+Inconsistencies in the repaired data set should not happen in practice and could indicate a possible bug in incremental
+repair.
+
+Running preview repairs is useful when considering using the
+xref:cassandra:managing/operating/compaction/tombstones.adoc#deletion[only_purge_repaired_tombstones] table compaction
+option to prevent data from possibly being resurrected when inconsistent replicas are missing tombstones from deletes.
+
+When enabled, the `BytesPreviewedDesynchronized` and `TokenRangesPreviewedDesynchronized`
+xref:cassandra:managing/operating/metrics.adoc#table-metrics[table metrics] can be used to detect inconsistencies in the
+repaired data set.
+
+== Configuring Auto Repair in cassandra.yaml
+
+Configuration for Auto Repair is managed in the `cassandra.yaml` file by the `auto_repair` property.
+
+A rich set of configuration exists for configuring Auto Repair with sensible defaults. However, the expectation
+is that some tuning might be needed particulary when it comes to tuning how often repair should run
+(`min_repair_interval`) and how repair assignments as created (`token_range_splitter`).
+
+The following is a practical example of an auto_repair configuration that one might use.
+
+[source, yaml]
+----
+auto_repair:
+ enabled: true
+ repair_type_overrides:
+ full:
+ enabled: true
+ min_repair_interval: 5d
+ incremental:
+ enabled: true
+ min_repair_interval: 1h
+ token_range_splitter:
+ parameters:
+ bytes_per_assignment: 50GiB
+ max_bytes_per_schedule: 100GiB
+ preview_repaired:
+ enabled: true
+ min_repair_interval: 1d
+ global_settings:
+ repair_by_keyspace: true
+ parallel_repair_count: 1
+----
+
+
+=== Top level settings
+The following settings are defined at the top level of the configuration file and apply universally across all
+repair types.
+
+[cols=",,",options="header",]
+|===
+| Name | Default | Description
+| enabled | false | Enable/Disable the auto-repair scheduler. If set to false, the scheduler thread will not be started.
+If set to true, the repair scheduler thread will be created. The thread will check for secondary configuration available
+for each repair type (full, incremental, and preview_repaired), and based on that, it will schedule repairs.
+| repair_check_interval | 5m | Time interval between successive checks to see if ongoing repairs are complete or if it
+is time to schedule repairs.
+| repair_max_retries | 3 | Maximum number of retries for a repair session.
+| history_clear_delete_hosts_buffer_interval | 2h | The scheduler needs to adjust its order when nodes leave the ring.
+Deleted hosts are tracked in metadata for a specified duration to ensure they are indeed removed before adjustments
+are made to the schedule.
+| mixed_major_version_repair_enabled | false | Enable/Disable running repairs on the cluster when there are mixed
+major versions detected, which usually occurs when the cluster is being upgraded. Repairs between nodes of
+different major versions is not something that is tested, so this may lead to data compatibility issues.
+It is strongly discouraged to set this to true without doing extensive testing beforehand.
+|===
+
+
+=== Repair level settings
+The following settings can be configured globally using `global_settings` or tailored individually for each repair
+type by using `repair_type_overrides`.
+
+[cols=",,",options="header",]
+|===
+| Name | Default | Description
+| enabled | false | Whether the given repair types should be enabled
+| min_repair_interval | 24h | Minimum duration between repairing the same node again. This is useful for tiny clusters,
+such as clusters with 5 nodes that finish repairs quickly. This means that if the scheduler completes one round on all
+nodes in less than this duration, it will not start a new repair round on a given node until this much time has
+passed since the last repair completed. Consider increasing to a larger value to reduce the impact of repairs,
+however note that one should attempt to run repairs at a smaller interval than gc_grace_seconds to
+avoid xref:cassandra:managing/operating/compaction/tombstones.adoc#zombies[data resurrection].
+| token_range_splitter.class_name | org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter | Implementation of
+IAutoRepairTokenRangeSplitter to use; responsible for splitting token ranges for repair assignments. Out of the box,
+Cassandra provides org.apache.cassandra.repair.autorepair.{RepairTokenRangeSplitter,FixedTokenRangeSplitter}.
+| repair_by_keyspace | true | If true, attempts to group tables in the same keyspace into one repair; otherwise,
+each table is repaired individually.
+| number_of_repair_threads | 1 | Number of threads to use for each repair job scheduled by the scheduler. Similar to
+the -j option in nodetool repair.
+| parallel_repair_count | 3 | Number of nodes running repair in parallel. If `parallel_repair_percentage` is set, the
+larger value is used.
+| parallel_repair_percentage | 3 | Percentage of nodes in the cluster running repair in parallel. If
+`parallel_repair_count is set`, the larger value is used.
+| allow_parallel_replica_repair | false | Whether to allow a node to take its turn running repair while one or more of
+its replicas are running repair. Defaults to false, as running repairs concurrently on replicas can increase load and
+also cause anticompaction conflicts while running incremental repair.
+| allow_parallel_replica_repair_across_schedules | true | An addition to allow_parallel_repair that also blocks repairs
+when replicas (including this node itself) are repairing in any schedule.
+For example, if a replica is executing full repairs, a value of false will prevent starting incremental repairs for this
+node. Defaults to true and is only evaluated when allow_parallel_replica_repair is false.
+| materialized_view_repair_enabled | false | Repairs materialized views if true.
+| initial_scheduler_delay | 5m | Delay before starting repairs after a node restarts to avoid repairs starting
+immediately after a restart.
+| repair_session_timeout | 3h | Timeout for retrying stuck repair sessions.
+| force_repair_new_node | false | Force immediate repair on new nodes after they join the ring.
+| sstable_upper_threshold | 50000 | Threshold to skip repairing tables with too many SSTables.
+| table_max_repair_time | 6h | Maximum time allowed for repairing one table on a given node. If exceeded, the repair
+proceeds to the next table.
+| ignore_dcs | [] | Avoid running repairs in specific data centers. By default, repairs run in all data centers. Specify
+data centers to exclude in this list. Note that repair sessions will still consider all replicas from excluded data
+centers. Useful if you have keyspaces that are not replicated in certain data centers, and you want to not run repair
+schedule in certain data centers.
+| repair_primary_token_range_only | true | Repair only the primary ranges owned by a node. Equivalent to the -pr option
+in nodetool repair. General advice is to keep this true.
+| repair_retry_backoff | 30s | Backoff time before retrying a repair session.
+| repair_task_min_duration | 5s | Minimum duration for the execution of a single repair task. This prevents the
+scheduler from overwhelming the node by scheduling too many repair tasks in a short period of time.
+|===
+
+=== `RepairTokenRangeSplitter` configuration
+[#repair-token-range-splitter]
+
+`RepairTokenRangeSplitter` is the default implementation of `IAutoRepairTokenRangeSplitter` that attempts to create
+token range assignments meeting the following goals:
+
+- *Create smaller, consistent repair times*: Long repairs, such as those lasting 15 hours, can be problematic. If a
+node fails 14 hours into the repair, the entire process must be restarted. The goal is to reduce the impact of
+disturbances or failures. However, making the repairs too short can lead to overhead from repair orchestration becoming
+the main bottleneck.
+
+- *Minimize the impact on hosts*: Repairs should not heavily affect the host systems. For incremental repairs, this
+might involve anti-compaction work. In full repairs, streaming large amounts of data—especially with wide partitions
+can lead to issues with disk usage and higher compaction costs.
+
+- *Reduce overstreaming*: The Merkle tree, which represents data within each partition and range, has a maximum size.
+If a repair covers too many partitions, the tree’s leaves represent larger data ranges. Even a small change in a leaf
+can trigger excessive data streaming, making the process inefficient.
+
+- *Reduce number of repairs*: If there are many small tables, it's beneficial to batch these tables together under a
+single parent repair. This prevents the repair overhead from becoming a bottleneck, especially when dealing with
+hundreds of tables. Running individual repairs for each table can significantly impact performance and efficiency.
+
+To achieve these goals, this implementation inspects SSTable metadata to estimate the bytes and number of partitions
+within a range and splits it accordingly to bound the size of the token ranges used for repair assignments.
+
+==== Parameter defaults
+
+The following `parameters` include the same defaults for all repair types.
+
+[cols=",,",options="header",]
+|===
+| Name | Default | Description
+| partitions_per_assignment | 1048576 | Maximum number of partitions to include in a repair
+assignment. Used to reduce number of partitions present in merkle tree leaf nodes to avoid overstreaming.
+| max_tables_per_assignment | 64 | Maximum number of tables to include in a repair assignment.
+This reduces the number of repairs, especially in keyspaces with many tables. The splitter avoids batching tables
+together if they exceed other configuration parameters like `bytes_per_assignment` or `partitions_per_assignment`.
+|===
+
+==== Full & Preview Repaired repair defaults
+
+The following `parameters` defaults are established for both `full` and `preview_repaired` repair scheduling:
+
+[cols=",,",options="header",]
+|===
+| Name | Default | Description
+| bytes_per_assignment | 50GiB | The target and maximum amount of *compressed* bytes that should be included in a
+repair assignment. *Note*: For full and preview_repaired, only the portion of an SSTable that covers the ranges
+being repaired are accounted for in this calculation.
+| max_bytes_per_schedule | 100000GiB | The maximum number of bytes to cover in an individual
+schedule. This serves as a mechanism to throttle the work done in each repair cycle. You may reduce this value if the
+impact of repairs is causing too much load on the cluster or increase it if writes outpace the amount of data being
+repaired. Alternatively, adjust the `min_repair_interval`. This is set to a large value for full repair to attempt to
+repair all data per repair schedule.
+|===
+
+==== Incremental repair defaults
+
+The following `parameters` defaults are established for `incremental` repair scheduling:
+
+[cols=",,",options="header",]
+|===
+| Name | Default | Description
+| bytes_per_assignment | 50GiB | The target and maximum amount of *compressed* bytes that should be
+included in a repair assignment. *Note*: For incremental repair, the *entire size* of *unrepaired* SSTables
+including ranges being repaired are accounted for in this calculation. This is to account for the anticompaction
+work required to split the candidate data to repair from the data that won't be repaired.
+| max_bytes_per_schedule | 100GiB | The maximum number of bytes to cover in an individual schedule.
+Consider increasing if more data is written than this limit within the `min_repair_interval`.
+|===
+
+=== `FixedSplitTokenRangeSplitter` configuration
+[#fixed-split-token-range-splitter]
+
+`FixedSplitTokenRangeSplitter` is a more simple implementation of `IAutoRepairTokenRangeSplitter` that creates repair
+assignments by splitting a node's token ranges into an even number of splits.
+
+The following `parameters` apply for `FixedSplitTokenRangeSplitter` configuration:
+
+[cols=",,",options="header",]
+|===
+| Name | Default | Description
+| number_of_subranges | 32 | Number of evenly split subranges to create for each node that repair runs for.
+If vnodes are configured using `num_tokens`, attempts to evenly subdivide subranges by each range. For example, for
+`num_tokens: 16` and `number_of_subranges: 32`, 2 (32/16) repair assignments will be created for each token range. At
+least one repair assignment will be created for each token range.
+|===
+
+=== Other cassandra.yaml Considerations
+
+==== Enable `reject_repair_compaction_threshold`
+
+When enabling auto_repair, it is advisable to configure the top level `reject_repair_compaction_threshold`
+configuration in cassandra.yaml as a backpressure mechanism to reject new repairs on instances that have many
+pending compactions.
+
+==== Tune `repair_disk_headroom_reject_ratio`
+
+By default, repairs will be rejected if less than 20% of disk is available. If one wishes to be
+conservative this top level configuration could be increased to a larger value to prevent filling your data directories.
+
+== Table configuration
+
+If Auto Repair is enabled in cassandra.yaml, the `auto_repair` property may be optionally configured at the table
+level, e.g.:
+
+[source,cql]
+----
+ALTER TABLE cycling.cyclist_races
+WITH auto_repair = {'incremental_enabled': 'false', 'priority': '0'};
+----
+
+[cols=",,",options="header",]
+|===
+| Name | Default | Description
+| priority | 0 | Indicates the priority at which this table should be given when issuing repairs. The higher the number
+the more priority will be given to repair the table (e.g. 3 will be repaired before 2). When `repair_by_keyspace` is
+set to `true` tables sharing the same priority may be grouped in the same repair assignment.
+| full_enabled | true | Whether full repair is enabled for this table. If full.enabled is not true in cassandra.yaml
+this will not be evaluated.
+| incremental_enabled | true | Whether incremental repair is enabled for this table. If incremental.enabled is not
+true in cassandra.yaml this will not be evaluated.
+| preview_repaired_enabled | true | Whether preview repair is enabled for this table. If preview_repaired.enabled is
+not true in cassandra.yaml this will not be evaluated.
+|===
+
+== Nodetool Configuration
+=== nodetool getautorepairconfig
+
+Retrieves the runtime configuration of Auto Repair for the targeted node.
+
+[source,none]
+----
+$> nodetool getautorepairconfig
+repair scheduler configuration:
+ repair_check_interval: 5m
+ repair_max_retries: 3
+ history_clear_delete_hosts_buffer_interval: 2h
+configuration for repair_type: full
+ enabled: true
+ min_repair_interval: 24h
+ repair_by_keyspace: true
+ number_of_repair_threads: 1
+ sstable_upper_threshold: 50000
+ table_max_repair_time: 6h
+ ignore_dcs: []
+ repair_primary_token_range_only: true
+ parallel_repair_count: 3
+ parallel_repair_percentage: 3
+ materialized_view_repair_enabled: false
+ initial_scheduler_delay: 5m
+ repair_session_timeout: 3h
+ force_repair_new_node: false
+ repair_retry_backoff: 30s
+ repair_task_min_duration: 5s
+ token_range_splitter: org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter
+ token_range_splitter.bytes_per_assignment: 50GiB
+ token_range_splitter.partitions_per_assignment: 1048576
+ token_range_splitter.max_tables_per_assignment: 64
+ token_range_splitter.max_bytes_per_schedule: 100000GiB
+configuration for repair_type: incremental
+ enabled: true
+ min_repair_interval: 1h
+ repair_by_keyspace: true
+ number_of_repair_threads: 1
+ sstable_upper_threshold: 50000
+ table_max_repair_time: 6h
+ ignore_dcs: []
+ repair_primary_token_range_only: true
+ parallel_repair_count: 3
+ parallel_repair_percentage: 3
+ materialized_view_repair_enabled: false
+ initial_scheduler_delay: 5m
+ repair_session_timeout: 3h
+ force_repair_new_node: false
+ repair_retry_backoff: 30s
+ repair_task_min_duration: 5s
+ token_range_splitter: org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter
+ token_range_splitter.bytes_per_assignment: 50GiB
+ token_range_splitter.partitions_per_assignment: 1048576
+ token_range_splitter.max_tables_per_assignment: 64
+ token_range_splitter.max_bytes_per_schedule: 100GiB
+configuration for repair_type: preview_repaired
+ enabled: false
+----
+
+=== nodetool autorepairstatus
+
+Provides currently running Auto Repair status.
+
+[source,none]
+----
+$> nodetool autorepairstatus -t incremental
+Active Repairs
+425cea55-09aa-46e0-8911-9f37a4424574
+
+
+$> nodetool autorepairstatus -t full
+Active Repairs
+NONE
+
+----
+
+=== nodetool setautorepairconfig
+
+Dynamic configuration changes can be made by using `setautorepairconfig`. Note that this only applies on the node being
+targeted and these changes are not retained when a node is bounced.
+
+The following disables the `incremental` repair schedule:
+
+[source,none]
+----
+$> nodetool setautorepairconfig -t incremental enabled false
+----
+
+The following adjusts the `min_repair_interval` option to `5d` specifically for the `full` repair schedule:
+
+[source,none]
+----
+$> nodetool setautorepairconfig -t full min_repair_interval 5d
+----
+
+The following configures the `bytes_per_assignment` parameter for `incremental` repair's `token_range_splitter` to
+`10GiB`:
+
+[source,none]
+----
+$> nodetool setautorepairconfig -t incremental token_range_splitter.bytes_per_assignment 10GiB
+----
+
+==== More details
+https://cwiki.apache.org/confluence/display/CASSANDRA/CEP-37+Apache+Cassandra+Unified+Repair+Solution[CEP-37]
diff --git a/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java b/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
index f447e812ee..e34e4414ef 100644
--- a/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
+++ b/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
@@ -56,6 +56,10 @@ public enum CassandraRelevantProperties
AUTH_CACHE_WARMING_MAX_RETRIES("cassandra.auth_cache.warming.max_retries"),
AUTH_CACHE_WARMING_RETRY_INTERVAL_MS("cassandra.auth_cache.warming.retry_interval_ms"),
AUTOCOMPACTION_ON_STARTUP_ENABLED("cassandra.autocompaction_on_startup_enabled", "true"),
+ /** When true, checks that all nodes are above the minimum supported version before running auto-repair */
+ AUTOREPAIR_CHECK_MIN_VERSION("cassandra.autorepair.check_min_version", "true"),
+ /** When false, completely disables the auto-repair feature including schema changes and JMX registration */
+ AUTOREPAIR_ENABLE("cassandra.autorepair.enable", "false"),
AUTO_BOOTSTRAP("cassandra.auto_bootstrap"),
AUTO_REPAIR_FREQUENCY_SECONDS("cassandra.auto_repair_frequency_seconds", convertToString(TimeUnit.MINUTES.toSeconds(5))),
BATCHLOG_REPLAY_TIMEOUT_IN_MS("cassandra.batchlog.replay_timeout_in_ms"),
diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java
index 9b16609154..87cc553176 100644
--- a/src/java/org/apache/cassandra/config/Config.java
+++ b/src/java/org/apache/cassandra/config/Config.java
@@ -41,6 +41,7 @@ import org.apache.cassandra.fql.FullQueryLoggerOptions;
import org.apache.cassandra.index.internal.CassandraIndex;
import org.apache.cassandra.io.compress.BufferType;
import org.apache.cassandra.io.sstable.format.big.BigFormat;
+import org.apache.cassandra.repair.autorepair.AutoRepairConfig;
import org.apache.cassandra.service.StartupChecks.StartupCheckType;
import org.apache.cassandra.utils.StorageCompatibilityMode;
@@ -348,6 +349,8 @@ public class Config
// The number of executors to use for building secondary indexes
public volatile int concurrent_index_builders = 2;
+ public volatile double repair_disk_headroom_reject_ratio = 0.0; // disabled by default on 5.0 for backward-compatibility
+
/**
* @deprecated retry support removed on CASSANDRA-10992
*/
@@ -592,6 +595,10 @@ public class Config
@Replaces(oldName = "enable_materialized_views", converter = Converters.IDENTITY, deprecated = true)
public boolean materialized_views_enabled = false;
+ // When true, materialized views data in SSTable go through commit logs during internodes streaming, e.g. repair
+ // When false, it behaves the same as normal streaming.
+ public volatile boolean materialized_views_on_repair_enabled = true;
+
@Replaces(oldName = "enable_transient_replication", converter = Converters.IDENTITY, deprecated = true)
public boolean transient_replication_enabled = false;
@@ -956,6 +963,8 @@ public class Config
public volatile DurationSpec.LongMicrosecondsBound minimum_timestamp_warn_threshold = null;
public volatile DurationSpec.LongMicrosecondsBound minimum_timestamp_fail_threshold = null;
+ public volatile AutoRepairConfig auto_repair = new AutoRepairConfig();
+
/**
* The variants of paxos implementation and semantics supported by Cassandra.
*/
diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index 3361385083..49bb7a67fb 100644
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@ -58,6 +58,7 @@ import com.google.common.collect.Sets;
import com.google.common.primitives.Ints;
import com.google.common.primitives.Longs;
import com.google.common.util.concurrent.RateLimiter;
+import org.apache.cassandra.repair.autorepair.AutoRepairConfig;
import org.apache.cassandra.utils.Pair;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
@@ -4223,6 +4224,16 @@ public class DatabaseDescriptor
conf.materialized_views_enabled = enableMaterializedViews;
}
+ public static boolean isMaterializedViewsOnRepairEnabled()
+ {
+ return conf.materialized_views_on_repair_enabled;
+ }
+
+ public static void setMaterializedViewsOnRepairEnabled(boolean val)
+ {
+ conf.materialized_views_on_repair_enabled = val;
+ }
+
public static boolean getSASIIndexesEnabled()
{
return conf.sasi_indexes_enabled;
@@ -5319,4 +5330,23 @@ public class DatabaseDescriptor
{
conf.paxos_repair_race_wait = paxosRepairRaceWait;
}
+
+ public static AutoRepairConfig getAutoRepairConfig()
+ {
+ return conf.auto_repair;
+ }
+
+ public static double getRepairDiskHeadroomRejectRatio()
+ {
+ return conf.repair_disk_headroom_reject_ratio;
+ }
+
+ public static void setRepairDiskHeadroomRejectRatio(double value)
+ {
+ if (value < 0.0 || value > 1.0)
+ {
+ throw new IllegalArgumentException("Value must be >= 0 and <= 1 for repair_disk_headroom_reject_ratio");
+ }
+ conf.repair_disk_headroom_reject_ratio = value;
+ }
}
diff --git a/src/java/org/apache/cassandra/cql3/UntypedResultSet.java b/src/java/org/apache/cassandra/cql3/UntypedResultSet.java
index a0201c500a..0c6802c84e 100644
--- a/src/java/org/apache/cassandra/cql3/UntypedResultSet.java
+++ b/src/java/org/apache/cassandra/cql3/UntypedResultSet.java
@@ -449,6 +449,14 @@ public abstract class UntypedResultSet implements Iterable<UntypedResultSet.Row>
return LongType.instance.compose(data.get(column));
}
+ // this function will return the default value if the row doesn't have that column or the column data is null
+ // This function is used to avoid the nullpointerexception
+ public long getLong(String column, long ifNull)
+ {
+ ByteBuffer bytes = data.get(column);
+ return bytes == null ? ifNull : LongType.instance.compose(bytes);
+ }
+
public <T> Set<T> getSet(String column, AbstractType<T> type)
{
ByteBuffer raw = data.get(column);
diff --git a/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java b/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java
index a538348cea..f021d4b8f3 100644
--- a/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java
@@ -82,7 +82,7 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa
public static final String CUSTOM_EXPRESSIONS_NOT_ALLOWED =
"Custom index expressions cannot be used in WHERE clauses for UPDATE or DELETE statements";
- private static final ColumnIdentifier CAS_RESULT_COLUMN = new ColumnIdentifier("[applied]", false);
+ public static final ColumnIdentifier CAS_RESULT_COLUMN = new ColumnIdentifier("[applied]", false);
protected final StatementType type;
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java b/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java
index 93d477c847..3f2ffa2b57 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java
@@ -23,8 +23,10 @@ import java.util.Set;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
+import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.cql3.statements.PropertyDefinitions;
import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.schema.AutoRepairParams;
import org.apache.cassandra.schema.CachingParams;
import org.apache.cassandra.schema.CompactionParams;
import org.apache.cassandra.schema.CompressionParams;
@@ -57,6 +59,10 @@ public final class TableAttributes extends PropertyDefinitions
public void validate()
{
validate(validKeywords, obsoleteKeywords);
+
+ if (hasOption(AUTO_REPAIR) && !CassandraRelevantProperties.AUTOREPAIR_ENABLE.getBoolean())
+ throw new ConfigurationException("auto_repair option is not supported unless auto-repair is enabled with -Dcassandra.autorepair.enable=true");
+
build(TableParams.builder()).validate();
}
@@ -151,6 +157,9 @@ public final class TableAttributes extends PropertyDefinitions
if (hasOption(READ_REPAIR))
builder.readRepair(ReadRepairStrategy.fromString(getString(READ_REPAIR)));
+ if (hasOption(Option.AUTO_REPAIR))
+ builder.automatedRepair(AutoRepairParams.fromMap(getMap(Option.AUTO_REPAIR)));
+
return builder.build();
}
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index e16e46c17c..7c93558004 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@ -867,7 +867,7 @@ public class CompactionManager implements CompactionManagerMBean, ICompactionMan
Set<SSTableReader> fullyContainedSSTables = findSSTablesToAnticompact(sstableIterator, normalizedRanges, sessionID);
- cfs.metric.bytesMutatedAnticompaction.inc(SSTableReader.getTotalBytes(fullyContainedSSTables));
+ cfs.metric.bytesMutatedAnticompaction.mark(SSTableReader.getTotalBytes(fullyContainedSSTables));
cfs.getCompactionStrategyManager().mutateRepaired(fullyContainedSSTables, UNREPAIRED_SSTABLE, sessionID, isTransient);
// since we're just re-writing the sstable metdata for the fully contained sstables, we don't want
// them obsoleted when the anti-compaction is complete. So they're removed from the transaction here
@@ -1702,7 +1702,7 @@ public class CompactionManager implements CompactionManagerMBean, ICompactionMan
// repairedAt values for these, we still avoid anti-compacting already repaired sstables, as we currently don't
// make use of any actual repairedAt value and splitting up sstables just for that is not worth it at this point.
Set<SSTableReader> unrepairedSSTables = sstables.stream().filter((s) -> !s.isRepaired()).collect(Collectors.toSet());
- cfs.metric.bytesAnticompacted.inc(SSTableReader.getTotalBytes(unrepairedSSTables));
+ cfs.metric.bytesAnticompacted.mark(SSTableReader.getTotalBytes(unrepairedSSTables));
Collection<Collection<SSTableReader>> groupedSSTables = cfs.getCompactionStrategyManager().groupSSTablesForAntiCompaction(unrepairedSSTables);
// iterate over sstables to check if the full / transient / unrepaired ranges intersect them.
diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java
index 50f87c799e..d248ceae54 100644
--- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java
+++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java
@@ -23,6 +23,7 @@ import java.util.Collection;
import java.util.List;
import java.util.Set;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.Iterables;
import org.slf4j.Logger;
@@ -175,7 +176,7 @@ public class CassandraStreamReceiver implements StreamReceiver
return cfs.metadata().params.cdc;
}
- // returns true iif it is a cdc table and cdc on repair is enabled.
+ // returns true if it is a cdc table and cdc on repair is enabled.
private boolean cdcRequiresWriteCommitLog(ColumnFamilyStore cfs)
{
return DatabaseDescriptor.isCDCOnRepairEnabled() && hasCDC(cfs);
@@ -190,11 +191,12 @@ public class CassandraStreamReceiver implements StreamReceiver
* For CDC-enabled tables and write path for CDC is enabled, we want to ensure that the mutations are
* run through the CommitLog, so they can be archived by the CDC process on discard.
*/
- private boolean requiresWritePath(ColumnFamilyStore cfs)
+ @VisibleForTesting
+ boolean requiresWritePath(ColumnFamilyStore cfs)
{
return cdcRequiresWriteCommitLog(cfs)
|| cfs.streamToMemtable()
- || (session.streamOperation().requiresViewBuild() && hasViews(cfs));
+ || (session.streamOperation().requiresViewBuild() && hasViews(cfs) && DatabaseDescriptor.isMaterializedViewsOnRepairEnabled());
}
private void sendThroughWritePath(ColumnFamilyStore cfs, Collection<SSTableReader> readers)
diff --git a/src/java/org/apache/cassandra/gms/Gossiper.java b/src/java/org/apache/cassandra/gms/Gossiper.java
index 647441ffba..54dd87091a 100644
--- a/src/java/org/apache/cassandra/gms/Gossiper.java
+++ b/src/java/org/apache/cassandra/gms/Gossiper.java
@@ -163,7 +163,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean,
/* live member set */
@VisibleForTesting
- final Set<InetAddressAndPort> liveEndpoints = new ConcurrentSkipListSet<>();
+ public final Set<InetAddressAndPort> liveEndpoints = new ConcurrentSkipListSet<>();
/* Inflight echo requests. */
private final Map<InetAddressAndPort, EndpointState> inflightEcho = new ConcurrentHashMap<>();
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
index 535fdf577c..96010ded32 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
@@ -433,6 +433,11 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource,
}
catch (CorruptSSTableException ex)
{
+ if (hasAutoRepairSchemaIncompatibility(ex))
+ logger.error("The SSTable {} contains an auto_repair column that is not recognized. " +
+ "This occurs when cassandra.autorepair.enable was previously set to true " +
+ "and is now false. Set -Dcassandra.autorepair.enable=true and restart.",
+ entry.getKey().baseFile());
JVMStabilityInspector.inspectThrowable(ex);
logger.error("Corrupt sstable {}; skipping table", entry, ex);
return;
@@ -465,6 +470,14 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource,
return sstables;
}
+ private static boolean hasAutoRepairSchemaIncompatibility(Throwable t)
+ {
+ for (Throwable cause = t; cause != null; cause = cause.getCause())
+ if (cause.getMessage() != null && cause.getMessage().contains("Unknown column auto_repair during deserialization"))
+ return true;
+ return false;
+ }
+
protected SSTableReader(Builder<?, ?> builder, Owner owner)
{
super(builder, owner);
diff --git a/src/java/org/apache/cassandra/locator/InetAddressAndPort.java b/src/java/org/apache/cassandra/locator/InetAddressAndPort.java
index c954a05b1c..be86d5fd56 100644
--- a/src/java/org/apache/cassandra/locator/InetAddressAndPort.java
+++ b/src/java/org/apache/cassandra/locator/InetAddressAndPort.java
@@ -25,15 +25,22 @@ import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
-import java.util.regex.Pattern;
+import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
+import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import com.google.common.base.Splitter;
import com.google.common.net.HostAndPort;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
@@ -58,6 +65,7 @@ import org.apache.cassandra.utils.FastByteOperations;
public final class InetAddressAndPort extends InetSocketAddress implements Comparable<InetAddressAndPort>, Serializable
{
private static final long serialVersionUID = 0;
+ private static final Logger logger = LoggerFactory.getLogger(InetAddressAndPort.class);
//Store these here to avoid requiring DatabaseDescriptor to be loaded. DatabaseDescriptor will set
//these when it loads the config. A lot of unit tests won't end up loading DatabaseDescriptor.
@@ -311,6 +319,52 @@ public final class InetAddressAndPort extends InetSocketAddress implements Compa
defaultPort = port;
}
+ public static List<String> stringify(Iterable<InetAddressAndPort> endpoints)
+ {
+ return stringify(endpoints, true);
+ }
+
+ public static List<String> stringify(Iterable<InetAddressAndPort> endpoints, boolean withPort)
+ {
+ List<String> stringEndpoints = new ArrayList<>();
+ for (InetAddressAndPort ep : endpoints)
+ {
+ stringEndpoints.add(ep.getHostAddress(withPort));
+ }
+ return stringEndpoints;
+ }
+
+ /**
+ * Parses a comma-separated list of hosts to a set of {@link InetAddressAndPort}
+ *
+ * @param value the comma-separated list of hosts to parse
+ * @param failOnError whether to fail when encountering an invalid hostname
+ * @return the set of parsed {@link InetAddressAndPort}
+ */
+ public static Set<InetAddressAndPort> parseHosts(String value, boolean failOnError)
+ {
+ Set<InetAddressAndPort> hosts = new HashSet<>();
+ for (String host : Splitter.on(',').split(value))
+ {
+ try
+ {
+ hosts.add(InetAddressAndPort.getByName(host));
+ }
+ catch (UnknownHostException e)
+ {
+ if (failOnError)
+ {
+ throw new IllegalArgumentException("Failed to parse host: " + host, e);
+ }
+ else
+ {
+ logger.warn("Invalid ip address {} from input={}", host, value);
+ }
+ }
+ }
+ return hosts;
+ }
+
static int getDefaultPort()
{
return defaultPort;
diff --git a/src/java/org/apache/cassandra/metrics/AutoRepairMetrics.java b/src/java/org/apache/cassandra/metrics/AutoRepairMetrics.java
new file mode 100644
index 0000000000..e1355dbfa6
--- /dev/null
+++ b/src/java/org/apache/cassandra/metrics/AutoRepairMetrics.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.metrics;
+
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.Gauge;
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType;
+import org.apache.cassandra.repair.autorepair.AutoRepairUtils;
+import org.apache.cassandra.repair.autorepair.AutoRepair;
+import org.apache.cassandra.service.AutoRepairService;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
+
+/**
+ * Metrics related to AutoRepair.
+ */
+public class AutoRepairMetrics
+{
+ public final Gauge<Integer> repairsInProgress;
+ public final Gauge<Integer> nodeRepairTimeInSec;
+ public final Gauge<Integer> clusterRepairTimeInSec;
+ public final Gauge<Integer> longestUnrepairedSec;
+ public final Gauge<Integer> repairStartLagSec;
+ public final Gauge<Integer> succeededTokenRangesCount;
+ public final Gauge<Integer> failedTokenRangesCount;
+ public final Gauge<Integer> skippedTokenRangesCount;
+ public final Gauge<Integer> skippedTablesCount;
+ public final Gauge<Integer> totalMVTablesConsideredForRepair;
+ public final Gauge<Integer> totalDisabledRepairTables;
+ public final Gauge<Long> totalBytesToRepair;
+ public final Gauge<Long> bytesAlreadyRepaired;
+ public final Gauge<Integer> totalKeyspaceRepairPlansToRepair;
+ public final Gauge<Integer> keyspaceRepairPlansAlreadyRepaired;
+
+ public Counter repairTurnMyTurn;
+ public Counter repairTurnMyTurnDueToPriority;
+ public Counter repairTurnMyTurnForceRepair;
+ public Counter repairDelayedByReplica;
+ public Counter repairDelayedBySchedule;
+
+ private final RepairType repairType;
+
+ private volatile int repairStartLagSecVal;
+
+ public AutoRepairMetrics(RepairType repairType)
+ {
+ this.repairType = repairType;
+ AutoRepairMetricsFactory factory = new AutoRepairMetricsFactory(repairType);
+
+ repairsInProgress = Metrics.register(factory.createMetricName("RepairsInProgress"), new Gauge<Integer>()
+ {
+ public Integer getValue()
+ {
+ return AutoRepair.instance.getRepairState(repairType).isRepairInProgress() ? 1 : 0;
+ }
+ });
+
+ nodeRepairTimeInSec = Metrics.register(factory.createMetricName("NodeRepairTimeInSec"), new Gauge<Integer>()
+ {
+ public Integer getValue()
+ {
+ return AutoRepair.instance.getRepairState(repairType).getNodeRepairTimeInSec();
+ }
+ });
+
+ clusterRepairTimeInSec = Metrics.register(factory.createMetricName("ClusterRepairTimeInSec"), new Gauge<Integer>()
+ {
+ public Integer getValue()
+ {
+ return AutoRepair.instance.getRepairState(repairType).getClusterRepairTimeInSec();
+ }
+ });
+
+ skippedTokenRangesCount = Metrics.register(factory.createMetricName("SkippedTokenRangesCount"), new Gauge<Integer>()
+ {
+ public Integer getValue()
+ {
+ return AutoRepair.instance.getRepairState(repairType).getSkippedTokenRangesCount();
+ }
+ });
+
+ skippedTablesCount = Metrics.register(factory.createMetricName("SkippedTablesCount"), new Gauge<Integer>()
+ {
+ public Integer getValue()
+ {
+ return AutoRepair.instance.getRepairState(repairType).getSkippedTablesCount();
+ }
+ });
+
+ longestUnrepairedSec = Metrics.register(factory.createMetricName("LongestUnrepairedSec"), new Gauge<Integer>()
+ {
+ public Integer getValue()
+ {
+ return AutoRepair.instance.getRepairState(repairType).getLongestUnrepairedSec();
+ }
+ });
+
+ repairStartLagSec = Metrics.register(factory.createMetricName("RepairStartLagSec"), new Gauge<Integer>()
+ {
+ public Integer getValue()
+ {
+ return repairStartLagSecVal;
+ }
+ });
+
+ succeededTokenRangesCount = Metrics.register(factory.createMetricName("SucceededTokenRangesCount"), new Gauge<Integer>()
+ {
+ public Integer getValue()
+ {
+ return AutoRepair.instance.getRepairState(repairType).getSucceededTokenRangesCount();
+ }
+ });
+
+ failedTokenRangesCount = Metrics.register(factory.createMetricName("FailedTokenRangesCount"), new Gauge<Integer>()
+ {
+ public Integer getValue()
+ {
+ return AutoRepair.instance.getRepairState(repairType).getFailedTokenRangesCount();
+ }
+ });
+
+ repairTurnMyTurn = Metrics.counter(factory.createMetricName("RepairTurnMyTurn"));
+ repairTurnMyTurnDueToPriority = Metrics.counter(factory.createMetricName("RepairTurnMyTurnDueToPriority"));
+ repairTurnMyTurnForceRepair = Metrics.counter(factory.createMetricName("RepairTurnMyTurnForceRepair"));
+
+ repairDelayedByReplica = Metrics.counter(factory.createMetricName("RepairDelayedByReplica"));
+ repairDelayedBySchedule = Metrics.counter(factory.createMetricName("RepairDelayedBySchedule"));
+
+ totalMVTablesConsideredForRepair = Metrics.register(factory.createMetricName("TotalMVTablesConsideredForRepair"), new Gauge<Integer>()
+ {
+ public Integer getValue()
+ {
+ return AutoRepair.instance.getRepairState(repairType).getTotalMVTablesConsideredForRepair();
+ }
+ });
+
+ totalDisabledRepairTables = Metrics.register(factory.createMetricName("TotalDisabledRepairTables"), new Gauge<Integer>()
+ {
+ public Integer getValue()
+ {
+ return AutoRepair.instance.getRepairState(repairType).getTotalDisabledTablesRepairCount();
+ }
+ });
+ totalBytesToRepair = Metrics.register(factory.createMetricName("TotalBytesToRepair"), new Gauge<Long>()
+ {
+ public Long getValue()
+ {
+ return AutoRepair.instance.getRepairState(repairType).getTotalBytesToRepair();
+ }
+ });
+ bytesAlreadyRepaired = Metrics.register(factory.createMetricName("BytesAlreadyRepaired"), new Gauge<Long>()
+ {
+ public Long getValue()
+ {
+ return AutoRepair.instance.getRepairState(repairType).getBytesAlreadyRepaired();
+ }
+ });
+ totalKeyspaceRepairPlansToRepair = Metrics.register(factory.createMetricName("TotalKeyspaceRepairPlansToRepair"), new Gauge<Integer>()
+ {
+ public Integer getValue()
+ {
+ return AutoRepair.instance.getRepairState(repairType).getTotalKeyspaceRepairPlansToRepair();
+ }
+ });
+ keyspaceRepairPlansAlreadyRepaired = Metrics.register(factory.createMetricName("KeyspaceRepairPlansAlreadyRepaired"), new Gauge<Integer>()
+ {
+ public Integer getValue()
+ {
+ return AutoRepair.instance.getRepairState(repairType).getKeyspaceRepairPlansAlreadyRepaired();
+ }
+ });
+ }
+
+ public void recordTurn(AutoRepairUtils.RepairTurn turn)
+ {
+ switch (turn)
+ {
+ case MY_TURN:
+ repairTurnMyTurn.inc();
+ break;
+ case MY_TURN_FORCE_REPAIR:
+ repairTurnMyTurnForceRepair.inc();
+ break;
+ case MY_TURN_DUE_TO_PRIORITY:
+ repairTurnMyTurnDueToPriority.inc();
+ break;
+ default:
+ throw new RuntimeException(String.format("Unrecoginized turn: %s", turn.name()));
+ }
+ this.repairStartLagSecVal = 0;
+ }
+
+ /**
+ * Record perceived lag in scheduling repair.
+ * <p/>
+ * Takes the current time and subtracts it from the given last repair finish time. It then compares the difference
+ * with the min repair interval for this repair type, and if that value is greater than 0, records it.
+ */
+ public void recordRepairStartLag(long lastFinishTimeInMs)
+ {
+ long now = AutoRepair.instance.currentTimeMs();
+ long deltaFinish = now - lastFinishTimeInMs;
+ long deltaMinRepairInterval = deltaFinish - AutoRepairService.instance
+ .getAutoRepairConfig().getRepairMinInterval(repairType)
+ .toMilliseconds();
+ this.repairStartLagSecVal = deltaMinRepairInterval > 0 ? (int) MILLISECONDS.toSeconds(deltaMinRepairInterval) : 0;
+ }
+
+ @VisibleForTesting
+ protected static class AutoRepairMetricsFactory implements MetricNameFactory
+ {
+ private static final String TYPE = "AutoRepair";
+ @VisibleForTesting
+ protected final String repairType;
+
+ protected AutoRepairMetricsFactory(RepairType repairType)
+ {
+ this.repairType = repairType.toString().toLowerCase();
+ }
+
+ @Override
+ public CassandraMetricsRegistry.MetricName createMetricName(String metricName)
+ {
+ StringBuilder mbeanName = new StringBuilder();
+ mbeanName.append(DefaultNameFactory.GROUP_NAME).append(':');
+ mbeanName.append("type=").append(TYPE);
+ mbeanName.append(",name=").append(metricName);
+ mbeanName.append(",repairType=").append(repairType);
+
+ StringBuilder scope = new StringBuilder();
+ scope.append("repairType=").append(repairType);
+
+ return new CassandraMetricsRegistry.MetricName(DefaultNameFactory.GROUP_NAME, TYPE.toLowerCase(),
+ metricName, scope.toString(), mbeanName.toString());
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/metrics/AutoRepairMetricsManager.java b/src/java/org/apache/cassandra/metrics/AutoRepairMetricsManager.java
new file mode 100644
index 0000000000..e97ce34e5a
--- /dev/null
+++ b/src/java/org/apache/cassandra/metrics/AutoRepairMetricsManager.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.metrics;
+
+import org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * AutoRepair metrics manager holding all the auto-repair related metrics.
+ */
+public class AutoRepairMetricsManager
+{
+ private static final Map<RepairType, AutoRepairMetrics> metrics = new ConcurrentHashMap<>();
+
+ public static AutoRepairMetrics getMetrics(RepairType repairType)
+ {
+ return metrics.computeIfAbsent(repairType, k -> new AutoRepairMetrics(repairType));
+ }
+}
diff --git a/src/java/org/apache/cassandra/metrics/CassandraMetricsRegistry.java b/src/java/org/apache/cassandra/metrics/CassandraMetricsRegistry.java
index 598e484f6b..d104fecb50 100644
--- a/src/java/org/apache/cassandra/metrics/CassandraMetricsRegistry.java
+++ b/src/java/org/apache/cassandra/metrics/CassandraMetricsRegistry.java
@@ -68,20 +68,30 @@ public class CassandraMetricsRegistry extends MetricRegistry
}
public Meter meter(MetricName name)
+ {
+ return meter(name, false);
+ }
+
+ public Meter meter(MetricName name, boolean gaugeCompatible)
{
Meter meter = meter(name.getMetricName());
- registerMBean(meter, name.getMBeanName());
+ registerMBean(meter, name.getMBeanName(), gaugeCompatible);
return meter;
}
- public Meter meter(MetricName name, MetricName alias)
+ public Meter meter(MetricName name, MetricName alias, boolean gaugeCompatible)
{
- Meter meter = meter(name);
- registerAlias(name, alias);
+ Meter meter = meter(name, gaugeCompatible);
+ registerAlias(name, alias, gaugeCompatible);
return meter;
}
+ public Meter meter(MetricName name, MetricName alias)
+ {
+ return meter(name, alias, false);
+ }
+
public Histogram histogram(MetricName name, boolean considerZeroes)
{
Histogram histogram = register(name, new ClearableHistogram(new DecayingEstimatedHistogramReservoir(considerZeroes)));
@@ -217,6 +227,11 @@ public class CassandraMetricsRegistry extends MetricRegistry
}
public void registerMBean(Metric metric, ObjectName name)
+ {
+ registerMBean(metric, name, false);
+ }
+
+ public void registerMBean(Metric metric, ObjectName name, boolean gaugeCompatible)
{
AbstractBean mbean;
@@ -229,7 +244,18 @@ public class CassandraMetricsRegistry extends MetricRegistry
else if (metric instanceof Timer)
mbean = new JmxTimer((Timer) metric, name, TimeUnit.SECONDS, DEFAULT_TIMER_UNIT);
else if (metric instanceof Metered)
- mbean = new JmxMeter((Metered) metric, name, TimeUnit.SECONDS);
+ {
+ // If a gauge compatible meter is requested, create a special implementation which
+ // also yields a 'Value' attribute for backwards compatibility.
+ if (gaugeCompatible)
+ {
+ mbean = new JmxMeterGaugeCompatible((Metered) metric, name, TimeUnit.SECONDS);
+ }
+ else
+ {
+ mbean = new JmxMeter((Metered) metric, name, TimeUnit.SECONDS);
+ }
+ }
else
throw new IllegalArgumentException("Unknown metric type: " + metric.getClass());
@@ -238,11 +264,16 @@ public class CassandraMetricsRegistry extends MetricRegistry
}
private void registerAlias(MetricName existingName, MetricName aliasName)
+ {
+ registerAlias(existingName, aliasName, false);
+ }
+
+ private void registerAlias(MetricName existingName, MetricName aliasName, boolean gaugeCompatible)
{
Metric existing = Metrics.getMetrics().get(existingName.getMetricName());
assert existing != null : existingName + " not registered";
- registerMBean(existing, aliasName.getMBeanName());
+ registerMBean(existing, aliasName.getMBeanName(), gaugeCompatible);
}
private void removeAlias(MetricName name)
@@ -531,6 +562,30 @@ public class CassandraMetricsRegistry extends MetricRegistry
}
}
+ public interface JmxMeterGaugeCompatibleMBean extends JmxMeterMBean, JmxGaugeMBean {}
+
+ /**
+ * An implementation of {@link JmxMeter} that is compatible with {@link JmxGaugeMBean} in that it also
+ * implements {@link JmxGaugeMBean}. This is useful for metrics that were migrated from {@link JmxGauge}
+ * to {@link JmxMeter} like {@link TableMetrics#bytesAnticompacted} and
+ * {@link TableMetrics#bytesMutatedAnticompaction}.
+ */
+ private static class JmxMeterGaugeCompatible extends JmxMeter implements JmxMeterGaugeCompatibleMBean
+ {
+
+ private JmxMeterGaugeCompatible(Metered metric, ObjectName objectName, TimeUnit rateUnit)
+ {
+ super(metric, objectName, rateUnit);
+ }
+
+ @Override
+ public Object getValue()
+ {
+ return getCount();
+ }
+ }
+
+
public interface JmxTimerMBean extends JmxMeterMBean
{
double getMin();
diff --git a/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java b/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java
index ae15bbf95a..5ddab582c0 100644
--- a/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java
@@ -184,6 +184,12 @@ public class KeyspaceMetrics
public final ImmutableMap<SSTableFormat<?, ?>, ImmutableMap<String, Gauge<? extends Number>>> formatSpecificGauges;
+ public final Meter bytesAnticompacted;
+ public final Meter bytesMutatedAnticompaction;
+ public final Meter bytesPreviewed;
+ public final Meter tokenRangesPreviewedDesynchronized;
+ public final Meter bytesPreviewedDesynchronized;
+
public final MetricNameFactory factory;
private final Keyspace keyspace;
@@ -292,6 +298,11 @@ public class KeyspaceMetrics
outOfRangeTokenReads = createKeyspaceCounter("ReadOutOfRangeToken");
outOfRangeTokenWrites = createKeyspaceCounter("WriteOutOfRangeToken");
outOfRangeTokenPaxosRequests = createKeyspaceCounter("PaxosOutOfRangeToken");
+ bytesAnticompacted = createKeyspaceMeter("BytesAnticompacted");
+ bytesMutatedAnticompaction = createKeyspaceMeter("BytesMutatedAnticompaction");
+ bytesPreviewed = createKeyspaceMeter("BytesPreviewed");
+ tokenRangesPreviewedDesynchronized = createKeyspaceMeter("TokenRangesPreviewedDesynchronized");
+ bytesPreviewedDesynchronized = createKeyspaceMeter("BytesPreviewedDesynchronized");
}
/**
diff --git a/src/java/org/apache/cassandra/metrics/TableMetrics.java b/src/java/org/apache/cassandra/metrics/TableMetrics.java
index b2a8e61fe4..49f1327446 100644
--- a/src/java/org/apache/cassandra/metrics/TableMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/TableMetrics.java
@@ -202,9 +202,15 @@ public class TableMetrics
/** number of partitions read creating merkle trees */
public final TableHistogram partitionsValidated;
/** number of bytes read while doing anticompaction */
- public final Counter bytesAnticompacted;
+ public final TableMeter bytesAnticompacted;
/** number of bytes where the whole sstable was contained in a repairing range so that we only mutated the repair status */
- public final Counter bytesMutatedAnticompaction;
+ public final TableMeter bytesMutatedAnticompaction;
+ /** number of bytes that were scanned during preview repair */
+ public final TableMeter bytesPreviewed;
+ /** number of desynchronized token ranges that were detected during preview repair */
+ public final TableMeter tokenRangesPreviewedDesynchronized;
+ /** number of desynchronized bytes that were detected during preview repair */
+ public final TableMeter bytesPreviewedDesynchronized;
/** ratio of how much we anticompact vs how much we could mutate the repair status*/
public final Gauge<Double> mutatedAnticompactionGauge;
@@ -811,12 +817,15 @@ public class TableMetrics
bytesValidated = createTableHistogram("BytesValidated", cfs.keyspace.metric.bytesValidated, false);
partitionsValidated = createTableHistogram("PartitionsValidated", cfs.keyspace.metric.partitionsValidated, false);
- bytesAnticompacted = createTableCounter("BytesAnticompacted");
- bytesMutatedAnticompaction = createTableCounter("BytesMutatedAnticompaction");
+ bytesAnticompacted = createTableMeter("BytesAnticompacted", cfs.keyspace.metric.bytesAnticompacted, true);
+ bytesMutatedAnticompaction = createTableMeter("BytesMutatedAnticompaction", cfs.keyspace.metric.bytesMutatedAnticompaction, true);
+ bytesPreviewed = createTableMeter("BytesPreviewed", cfs.keyspace.metric.bytesPreviewed);
+ tokenRangesPreviewedDesynchronized = createTableMeter("TokenRangesPreviewedDesynchronized", cfs.keyspace.metric.tokenRangesPreviewedDesynchronized);
+ bytesPreviewedDesynchronized = createTableMeter("BytesPreviewedDesynchronized", cfs.keyspace.metric.bytesPreviewedDesynchronized);
mutatedAnticompactionGauge = createTableGauge("MutatedAnticompactionGauge", () ->
{
- double bytesMutated = bytesMutatedAnticompaction.getCount();
- double bytesAnticomp = bytesAnticompacted.getCount();
+ double bytesMutated = bytesMutatedAnticompaction.table.getCount();
+ double bytesAnticomp = bytesAnticompacted.table.getCount();
if (bytesAnticomp + bytesMutated > 0)
return bytesMutated / (bytesAnticomp + bytesMutated);
return 0.0;
@@ -1094,17 +1103,23 @@ public class TableMetrics
protected TableMeter createTableMeter(String name, Meter keyspaceMeter)
{
- return createTableMeter(name, name, keyspaceMeter);
+ return createTableMeter(name, keyspaceMeter, false);
}
- protected TableMeter createTableMeter(String name, String alias, Meter keyspaceMeter)
+ protected TableMeter createTableMeter(String name, Meter keyspaceMeter, boolean globalMeterGaugeCompatible)
+ {
+ return createTableMeter(name, name, keyspaceMeter, globalMeterGaugeCompatible);
+ }
+
+ protected TableMeter createTableMeter(String name, String alias, Meter keyspaceMeter, boolean globalMeterGaugeCompatible)
{
Meter meter = Metrics.meter(factory.createMetricName(name), aliasFactory.createMetricName(alias));
register(name, alias, meter);
return new TableMeter(meter,
keyspaceMeter,
Metrics.meter(GLOBAL_FACTORY.createMetricName(name),
- GLOBAL_ALIAS_FACTORY.createMetricName(alias)));
+ GLOBAL_ALIAS_FACTORY.createMetricName(alias),
+ globalMeterGaugeCompatible));
}
private LatencyMetrics createLatencyMetrics(String namePrefix, LatencyMetrics ... parents)
@@ -1176,10 +1191,15 @@ public class TableMetrics
}
public void mark()
+ {
+ mark(1L);
+ }
+
+ public void mark(long val)
{
for (Meter meter : all)
{
- meter.mark();
+ meter.mark(val);
}
}
}
diff --git a/src/java/org/apache/cassandra/repair/PreviewRepairTask.java b/src/java/org/apache/cassandra/repair/PreviewRepairTask.java
index edee11cf20..600e64610a 100644
--- a/src/java/org/apache/cassandra/repair/PreviewRepairTask.java
+++ b/src/java/org/apache/cassandra/repair/PreviewRepairTask.java
@@ -26,6 +26,7 @@ import com.google.common.base.Preconditions;
import org.apache.cassandra.concurrent.ExecutorPlus;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
@@ -85,10 +86,10 @@ public class PreviewRepairTask extends AbstractRepairTask
else
{
message = (previewKind == PreviewKind.REPAIRED ? "Repaired data is inconsistent\n" : "Preview complete\n") + summary;
- RepairMetrics.previewFailures.inc();
if (previewKind == PreviewKind.REPAIRED)
maybeSnapshotReplicas(parentSession, keyspace, result.results.get()); // we know its present as summary used it
}
+ emitMetrics(summary);
successMessage += "; " + message;
coordinator.notification(message);
@@ -96,6 +97,21 @@ public class PreviewRepairTask extends AbstractRepairTask
});
}
+ private void emitMetrics(SyncStatSummary summary)
+ {
+ if (!summary.isEmpty())
+ RepairMetrics.previewFailures.inc();
+
+ summary.getTotals().forEach((key, table) -> {
+ if (table.isCounter())
+ return;
+
+ ColumnFamilyStore cfs = Keyspace.open(key.left).getColumnFamilyStore(key.right);
+ cfs.metric.tokenRangesPreviewedDesynchronized.mark(table.getRanges());
+ cfs.metric.bytesPreviewedDesynchronized.mark(table.getBytes());
+ });
+ }
+
private void maybeSnapshotReplicas(TimeUUID parentSession, String keyspace, List<RepairSessionResult> results)
{
if (!DatabaseDescriptor.snapshotOnRepairedDataMismatch())
diff --git a/src/java/org/apache/cassandra/repair/RepairCoordinator.java b/src/java/org/apache/cassandra/repair/RepairCoordinator.java
index 091819c65d..b5ad02cd21 100644
--- a/src/java/org/apache/cassandra/repair/RepairCoordinator.java
+++ b/src/java/org/apache/cassandra/repair/RepairCoordinator.java
@@ -32,6 +32,7 @@ import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.function.Supplier;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
@@ -102,7 +103,8 @@ public class RepairCoordinator implements Runnable, ProgressEventNotifier, Repai
private final Function<String, RangesAtEndpoint> getLocalReplicas;
private final List<ProgressListener> listeners = new ArrayList<>();
- private final AtomicReference<Throwable> firstError = new AtomicReference<>(null);
+ @VisibleForTesting
+ protected final AtomicReference<Throwable> firstError = new AtomicReference<>(null);
final SharedContext ctx;
final Scheduler validationScheduler;
@@ -208,12 +210,12 @@ public class RepairCoordinator implements Runnable, ProgressEventNotifier, Repai
complete(null);
}
- private void fail(String reason)
+ protected void fail(String reason)
{
if (reason == null)
{
Throwable error = firstError.get();
- reason = error != null ? error.toString() : "Some repair failed";
+ reason = (error != null && error.getMessage() != null) ? error.getMessage() : "Some repair failed";
}
state.phase.fail(reason);
ParticipateState p = ctx.repair().participate(state.id);
diff --git a/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java b/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java
index 621aacc378..1f4fa5b568 100644
--- a/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java
+++ b/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java
@@ -114,6 +114,13 @@ public class RepairMessageVerbHandler implements IVerbHandler<RepairMessage>
replyDedup(ctx.repair().participate(state.id), message);
return;
}
+ if (!ctx.repair().verifyDiskHeadroomThreshold(prepareMessage.parentRepairSession, prepareMessage.previewKind))
+ {
+ // error is logged in verifyDiskHeadroomThreshold
+ state.phase.fail("Not enough disk headroom to perform repair");
+ sendFailureResponse(message);
+ return;
+ }
if (!ctx.repair().verifyCompactionsPendingThreshold(prepareMessage.parentRepairSession, prepareMessage.previewKind))
{
// error is logged in verifyCompactionsPendingThreshold
diff --git a/src/java/org/apache/cassandra/repair/ValidationManager.java b/src/java/org/apache/cassandra/repair/ValidationManager.java
index e3598cd38f..c4f4d7419c 100644
--- a/src/java/org/apache/cassandra/repair/ValidationManager.java
+++ b/src/java/org/apache/cassandra/repair/ValidationManager.java
@@ -37,6 +37,7 @@ import org.apache.cassandra.dht.Token;
import org.apache.cassandra.metrics.TableMetrics;
import org.apache.cassandra.metrics.TopPartitionTracker;
import org.apache.cassandra.repair.state.ValidationState;
+import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.utils.Clock;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.MerkleTree;
@@ -143,6 +144,10 @@ public class ValidationManager implements IValidationManager
{
cfs.metric.bytesValidated.update(state.estimatedTotalBytes);
cfs.metric.partitionsValidated.update(state.partitionsProcessed);
+ if (validator.getPreviewKind() != PreviewKind.NONE)
+ {
+ cfs.metric.bytesPreviewed.mark(state.estimatedTotalBytes);
+ }
if (topPartitionCollector != null)
cfs.topPartitions.merge(topPartitionCollector);
}
diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java
new file mode 100644
index 0000000000..692ef6641e
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java
@@ -0,0 +1,629 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.autorepair;
+
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.EnumMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
+import com.google.common.util.concurrent.Uninterruptibles;
+
+import org.apache.cassandra.config.DurationSpec;
+import org.apache.cassandra.repair.RepairCoordinator;
+import org.apache.cassandra.utils.Clock;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.concurrent.ScheduledExecutorPlus;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.Tables;
+import org.apache.cassandra.service.AutoRepairService;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.repair.autorepair.AutoRepairUtils.RepairTurn;
+import org.apache.cassandra.utils.concurrent.Condition;
+import org.apache.cassandra.utils.concurrent.Future;
+import org.apache.cassandra.utils.progress.ProgressEvent;
+import org.apache.cassandra.utils.progress.ProgressEventType;
+import org.apache.cassandra.utils.progress.ProgressListener;
+
+import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
+import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.RepairTurn.MY_TURN;
+import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.RepairTurn.MY_TURN_DUE_TO_PRIORITY;
+import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.RepairTurn.MY_TURN_FORCE_REPAIR;
+import static org.apache.cassandra.utils.concurrent.Condition.newOneTimeCondition;
+
+/**
+ * AutoRepair scheduler responsible for running different types of repairs.
+ */
+public class AutoRepair
+{
+ private static final Logger logger = LoggerFactory.getLogger(AutoRepair.class);
+ private static final SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,SSS");
+
+ @VisibleForTesting
+ protected static Supplier<Long> timeFunc = Clock.Global::currentTimeMillis;
+
+ // Sleep for 5 seconds if repair finishes quickly to flush JMX metrics; it happens only for Cassandra nodes with tiny amount of data.
+ public static DurationSpec.IntSecondsBound SLEEP_IF_REPAIR_FINISHES_QUICKLY = new DurationSpec.IntSecondsBound("5s");
+
+ @VisibleForTesting
+ public Map<AutoRepairConfig.RepairType, AutoRepairState> repairStates;
+
+ @VisibleForTesting
+ protected Map<AutoRepairConfig.RepairType, ScheduledExecutorPlus> repairExecutors;
+
+ protected Map<AutoRepairConfig.RepairType, ScheduledExecutorPlus> repairRunnableExecutors;
+
+ @VisibleForTesting
+ // Auto-repair is likely to be run on multiple nodes independently, we want to avoid running multiple repair
+ // sessions on overlapping datasets at the same time. Shuffling keyspaces reduces the likelihood of this happening.
+ protected static Consumer<List<String>> shuffleFunc = java.util.Collections::shuffle;
+
+ @VisibleForTesting
+ protected static BiConsumer<Long, TimeUnit> sleepFunc = Uninterruptibles::sleepUninterruptibly;
+
+ @VisibleForTesting
+ public boolean isSetupDone = false;
+ public static AutoRepair instance = new AutoRepair();
+
+ public volatile boolean isShutDown = false;
+
+ private AutoRepair()
+ {
+ // Private constructor to prevent instantiation
+ }
+
+ public void setup()
+ {
+ // Ensure setup is done only once; this is only for unit tests
+ // For production, this method should be called only once.
+ synchronized (this)
+ {
+ if (isSetupDone)
+ {
+ return;
+ }
+ repairExecutors = new EnumMap<>(AutoRepairConfig.RepairType.class);
+ repairRunnableExecutors = new EnumMap<>(AutoRepairConfig.RepairType.class);
+ repairStates = new EnumMap<>(AutoRepairConfig.RepairType.class);
+ AutoRepairConfig config = DatabaseDescriptor.getAutoRepairConfig();
+
+ for (AutoRepairConfig.RepairType repairType : AutoRepairConfig.RepairType.values())
+ {
+ repairExecutors.put(repairType, executorFactory().scheduled(false, "AutoRepair-Repair-" + repairType.getConfigName(), Thread.NORM_PRIORITY));
+ repairRunnableExecutors.put(repairType, executorFactory().scheduled(false, "AutoRepair-RepairRunnable-" + repairType.getConfigName(), Thread.NORM_PRIORITY));
+ repairStates.put(repairType, AutoRepairConfig.RepairType.getAutoRepairState(repairType, config));
+ }
+
+ AutoRepairUtils.setup();
+
+ for (AutoRepairConfig.RepairType repairType : AutoRepairConfig.RepairType.values())
+ {
+ if (config.isAutoRepairEnabled(repairType))
+ AutoRepairService.instance.checkCanRun(repairType);
+
+ repairExecutors.get(repairType).scheduleWithFixedDelay(
+ () -> repair(repairType),
+ config.getInitialSchedulerDelay(repairType).toSeconds(),
+ config.getRepairCheckInterval().toSeconds(),
+ TimeUnit.SECONDS);
+ }
+ isSetupDone = true;
+ }
+ }
+
+ /**
+ * @return The current observed system time in ms.
+ */
+ public long currentTimeMs()
+ {
+ return timeFunc.get();
+ }
+
+ // repair runs a repair session of the given type synchronously.
+ public void repair(AutoRepairConfig.RepairType repairType)
+ {
+ AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig();
+ if (!config.isAutoRepairEnabled(repairType))
+ {
+ logger.debug("Auto-repair is disabled for repair type {}", repairType);
+ return;
+ }
+ if (!config.isMixedMajorVersionRepairEnabled() && AutoRepairUtils.hasMultipleLiveMajorVersions())
+ {
+ logger.info("Auto-repair is disabled when nodes in the cluster have different major versions");
+ return;
+ }
+ if (AutoRepairUtils.hasNodesBelowMinimumVersion())
+ {
+ logger.info("Auto-repair is disabled because some nodes are running unsupported versions " +
+ "(auto-repair requires all nodes to be above version {})",
+ AutoRepairUtils.LAST_UNSUPPORTED_VERSION_FOR_AUTO_REPAIR);
+ return;
+ }
+ AutoRepairService.instance.checkCanRun(repairType);
+ AutoRepairState repairState = repairStates.get(repairType);
+ try
+ {
+ String localDC = DatabaseDescriptor.getLocalDataCenter();
+ if (config.getIgnoreDCs(repairType).contains(localDC))
+ {
+ logger.info("Not running repair as this node belongs to datacenter {}", localDC);
+ return;
+ }
+
+ // refresh the longest unrepaired node
+ repairState.setLongestUnrepairedNode(AutoRepairUtils.getHostWithLongestUnrepairTime(repairType));
+
+ //consistency level to use for local query
+ UUID myId = Gossiper.instance.getHostId(FBUtilities.getBroadcastAddressAndPort());
+
+ // If it's too soon to run repair, don't bother checking if it's our turn.
+ if (tooSoonToRunRepair(repairType, repairState, config, myId))
+ {
+ return;
+ }
+
+ RepairTurn turn = AutoRepairUtils.myTurnToRunRepair(repairType, myId);
+ if (turn == MY_TURN || turn == MY_TURN_DUE_TO_PRIORITY || turn == MY_TURN_FORCE_REPAIR)
+ {
+ repairState.recordTurn(turn);
+ repairState.setBytesAlreadyRepaired(0L);
+ repairState.setKeyspaceRepairPlansAlreadyRepaired(0);
+ // For normal auto repair, we will use primary range only repairs (Repair with -pr option).
+ // For some cases, we may set the auto_repair_primary_token_range_only flag to false then we will do repair
+ // without -pr. We may also do force repair for certain node that we want to repair all the data on one node
+ // When doing force repair, we want to repair without -pr.
+ boolean primaryRangeOnly = config.getRepairPrimaryTokenRangeOnly(repairType)
+ && turn != MY_TURN_FORCE_REPAIR;
+
+ long startTimeInMillis = timeFunc.get();
+ logger.info("My host id: {}, my turn to run repair...repair primary-ranges only? {}", myId,
+ config.getRepairPrimaryTokenRangeOnly(repairType));
+ AutoRepairUtils.updateStartAutoRepairHistory(repairType, myId, timeFunc.get(), turn);
+
+ repairState.setRepairKeyspaceCount(0);
+ repairState.setRepairInProgress(true);
+ repairState.setTotalTablesConsideredForRepair(0);
+ repairState.setTotalMVTablesConsideredForRepair(0);
+
+ CollectedRepairStats collectedRepairStats = new CollectedRepairStats();
+
+ List<Keyspace> keyspaces = new ArrayList<>();
+ Keyspace.all().forEach(keyspaces::add);
+ // Filter out keyspaces and tables to repair and group into a map by keyspace.
+ Map<String, List<String>> keyspacesAndTablesToRepair = new LinkedHashMap<>();
+ for (Keyspace keyspace : keyspaces)
+ {
+ if (!AutoRepairUtils.shouldConsiderKeyspace(keyspace))
+ {
+ continue;
+ }
+ List<String> tablesToBeRepairedList = retrieveTablesToBeRepaired(keyspace, config, repairType, repairState, collectedRepairStats);
+ keyspacesAndTablesToRepair.put(keyspace.getName(), tablesToBeRepairedList);
+ }
+
+ // Separate out the keyspaces and tables to repair based on their priority, with each repair plan representing a uniquely occuring priority.
+ List<PrioritizedRepairPlan> repairPlans = PrioritizedRepairPlan.build(keyspacesAndTablesToRepair, repairType, shuffleFunc, primaryRangeOnly);
+ repairState.updateRepairScheduleStatistics(repairPlans);
+
+ // calculate the repair assignments for each priority:keyspace.
+ Iterator<KeyspaceRepairAssignments> repairAssignmentsIterator = config.getTokenRangeSplitterInstance(repairType).getRepairAssignments(primaryRangeOnly, repairPlans);
+
+ int keyspaceRepairAssignmentsAlreadyRepaired = 0;
+ while (repairAssignmentsIterator.hasNext())
+ {
+ KeyspaceRepairAssignments repairAssignments = repairAssignmentsIterator.next();
+ List<RepairAssignment> assignments = repairAssignments.getRepairAssignments();
+ if (assignments.isEmpty())
+ {
+ keyspaceRepairAssignmentsAlreadyRepaired++;
+ logger.info("Skipping repairs for priorityBucket={} for keyspace={} since it yielded no assignments", repairAssignments.getPriority(), repairAssignments.getKeyspaceName());
+ continue;
+ }
+
+ logger.info("Submitting repairs for priorityBucket={} for keyspace={} with assignmentCount={} and keyspaceRepairAssignmentsAlreadyRepaired={}/{}",
+ repairAssignments.getPriority(), repairAssignments.getKeyspaceName(), repairAssignments.getRepairAssignments().size(),
+ keyspaceRepairAssignmentsAlreadyRepaired, repairState.getTotalKeyspaceRepairPlansToRepair());
+ repairKeyspace(repairType, primaryRangeOnly, repairAssignments.getKeyspaceName(), repairAssignments.getRepairAssignments(), collectedRepairStats);
+ keyspaceRepairAssignmentsAlreadyRepaired++;
+ repairState.setKeyspaceRepairPlansAlreadyRepaired(keyspaceRepairAssignmentsAlreadyRepaired);
+ }
+
+ cleanupAndUpdateStats(turn, repairType, repairState, myId, startTimeInMillis, collectedRepairStats);
+ }
+ else
+ {
+ logger.info("Waiting for my turn...");
+ }
+ }
+ catch (Exception e)
+ {
+ logger.error("Exception in autorepair:", e);
+ }
+ }
+
+ private void repairKeyspace(AutoRepairConfig.RepairType repairType, boolean primaryRangeOnly, String keyspaceName, List<RepairAssignment> repairAssignments, CollectedRepairStats collectedRepairStats)
+ {
+ AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig();
+ AutoRepairState repairState = repairStates.get(repairType);
+
+ // evaluate over each keyspace's repair assignments.
+ repairState.setRepairKeyspaceCount(repairState.getRepairKeyspaceCount() + 1);
+
+ int totalRepairAssignments = repairAssignments.size();
+ long keyspaceStartTime = timeFunc.get();
+ RepairAssignment previousAssignment = null;
+ long tableStartTime = timeFunc.get();
+ int totalProcessedAssignments = 0;
+ Set<Range<Token>> ranges = new HashSet<>();
+ long bytesAlreadyRepaired = repairState.getBytesAlreadyRepaired();
+ for (RepairAssignment curRepairAssignment : repairAssignments)
+ {
+ try
+ {
+ totalProcessedAssignments++;
+ boolean repairOneTableAtATime = !config.getRepairByKeyspace(repairType);
+ if (previousAssignment != null && repairOneTableAtATime && !previousAssignment.tableNames.equals(curRepairAssignment.tableNames))
+ {
+ // In the repair assignment, all the tables are appended sequnetially.
+ // Check if we have a different table, and if so, we should reset the table start time.
+ tableStartTime = timeFunc.get();
+ }
+ previousAssignment = curRepairAssignment;
+ if (!config.isAutoRepairEnabled(repairType))
+ {
+ logger.error("Auto-repair for type {} is disabled hence not running repair", repairType);
+ repairState.setRepairInProgress(false);
+ return;
+ }
+ if (AutoRepairUtils.keyspaceMaxRepairTimeExceeded(repairType, keyspaceStartTime, repairAssignments.size()))
+ {
+ collectedRepairStats.skippedTokenRanges += totalRepairAssignments - totalProcessedAssignments;
+ logger.info("Keyspace took too much time to repair hence skipping it {}",
+ keyspaceName);
+ break;
+ }
+ if (repairOneTableAtATime && AutoRepairUtils.tableMaxRepairTimeExceeded(repairType, tableStartTime))
+ {
+ collectedRepairStats.skippedTokenRanges += 1;
+ logger.info("Table took too much time to repair hence skipping it table name {}.{}, token range {}",
+ keyspaceName, curRepairAssignment.tableNames, curRepairAssignment.tokenRange);
+ continue;
+ }
+
+ Range<Token> tokenRange = curRepairAssignment.getTokenRange();
+ logger.debug("Current Token Left side {}, right side {}",
+ tokenRange.left.toString(),
+ tokenRange.right.toString());
+
+ ranges.add(curRepairAssignment.getTokenRange());
+ if ((totalProcessedAssignments % config.getRepairThreads(repairType) == 0) ||
+ (totalProcessedAssignments == totalRepairAssignments))
+ {
+ boolean success = false;
+ int retryCount = 0;
+ Future<?> f = null;
+ while (retryCount <= config.getRepairMaxRetries(repairType))
+ {
+ RepairCoordinator task = repairState.getRepairRunnable(keyspaceName,
+ Lists.newArrayList(curRepairAssignment.getTableNames()),
+ ranges, primaryRangeOnly);
+ RepairProgressListener listener = new RepairProgressListener(repairType);
+ task.addProgressListener(listener);
+ f = repairRunnableExecutors.get(repairType).submit(task);
+ try
+ {
+ long jobStartTime = timeFunc.get();
+ listener.await(config.getRepairSessionTimeout(repairType));
+ success = listener.isSuccess();
+ soakAfterRepair(jobStartTime, config.getRepairTaskMinDuration().toMilliseconds());
+ }
+ catch (InterruptedException e)
+ {
+ logger.error("Exception in cond await:", e);
+ }
+ if (success)
+ {
+ break;
+ }
+ else if (retryCount < config.getRepairMaxRetries(repairType))
+ {
+ boolean cancellationStatus = f.cancel(true);
+ logger.warn("Repair failed for range {}-{} for {} tables {} with cancellationStatus: {} retrying after {} seconds...",
+ tokenRange.left, tokenRange.right,
+ keyspaceName, curRepairAssignment.getTableNames(),
+ cancellationStatus, config.getRepairRetryBackoff(repairType).toSeconds());
+ sleepFunc.accept(config.getRepairRetryBackoff(repairType).toSeconds(), TimeUnit.SECONDS);
+ }
+ retryCount++;
+ }
+ //check repair status
+ if (success)
+ {
+ logger.info("Repair completed for range {}-{} for {} tables {}, total assignments: {}," +
+ "processed assignments: {}", tokenRange.left, tokenRange.right,
+ keyspaceName, curRepairAssignment.getTableNames(), totalRepairAssignments, totalProcessedAssignments);
+ collectedRepairStats.succeededTokenRanges += ranges.size();
+ }
+ else
+ {
+ boolean cancellationStatus = true;
+ if (f != null)
+ {
+ cancellationStatus = f.cancel(true);
+ }
+ //in the future we can add retry, etc.
+ logger.error("Repair failed for range {}-{} for {} tables {} after {} retries, total assignments: {}," +
+ "processed assignments: {}, cancellationStatus: {}", tokenRange.left, tokenRange.right, keyspaceName,
+ curRepairAssignment.getTableNames(), retryCount, totalRepairAssignments, totalProcessedAssignments, cancellationStatus);
+ collectedRepairStats.failedTokenRanges += ranges.size();
+ }
+ ranges.clear();
+ }
+ bytesAlreadyRepaired += curRepairAssignment.getEstimatedBytes();
+ repairState.setBytesAlreadyRepaired(bytesAlreadyRepaired);
+ logger.info("Repair completed for {} tables {}, range {}, bytesAlreadyRepaired {}/{}",
+ keyspaceName, curRepairAssignment.getTableNames(), curRepairAssignment.getTokenRange(), bytesAlreadyRepaired, repairState.getTotalBytesToRepair());
+ }
+ catch (Exception e)
+ {
+ logger.error("Exception while repairing keyspace {}:", keyspaceName, e);
+ }
+ }
+ }
+
+ private boolean tooSoonToRunRepair(AutoRepairConfig.RepairType repairType, AutoRepairState repairState, AutoRepairConfig config, UUID myId)
+ {
+ if (repairState.getLastRepairTime() == 0)
+ {
+ // the node has either just boooted or has not run repair before,
+ // we should check for the node's repair history in the DB
+ repairState.setLastRepairTime(AutoRepairUtils.getLastRepairTimeForNode(repairType, myId));
+ }
+ /*
+ * check if it is too soon to run repair. one of the reason we
+ * should not run frequent repair is that repair triggers
+ * memtable flush
+ */
+ long timeElapsedSinceLastRepair = TimeUnit.MILLISECONDS.toSeconds(timeFunc.get() - repairState.getLastRepairTime());
+ if (timeElapsedSinceLastRepair < config.getRepairMinInterval(repairType).toSeconds())
+ {
+ logger.info("Too soon to run repair, last repair was done {} seconds ago",
+ timeElapsedSinceLastRepair);
+ return true;
+ }
+ return false;
+ }
+
+ private List<String> retrieveTablesToBeRepaired(Keyspace keyspace, AutoRepairConfig config, AutoRepairConfig.RepairType repairType, AutoRepairState repairState, CollectedRepairStats collectedRepairStats)
+ {
+ Tables tables = keyspace.getMetadata().tables;
+ List<String> tablesToBeRepaired = new ArrayList<>();
+ Iterator<TableMetadata> iter = tables.iterator();
+ while (iter.hasNext())
+ {
+ repairState.setTotalTablesConsideredForRepair(repairState.getTotalTablesConsideredForRepair() + 1);
+ TableMetadata tableMetadata = iter.next();
+ String tableName = tableMetadata.name;
+
+ ColumnFamilyStore columnFamilyStore = keyspace.getColumnFamilyStore(tableName);
+ if (!columnFamilyStore.metadata().params.autoRepair.repairEnabled(repairType))
+ {
+ logger.info("Repair is disabled for keyspace {} for tables: {}", keyspace.getName(), tableName);
+ repairState.setTotalDisabledTablesRepairCount(repairState.getTotalDisabledTablesRepairCount() + 1);
+ collectedRepairStats.skippedTables++;
+ continue;
+ }
+
+ // this is done to make autorepair safe as running repair on table with more sstables
+ // may have its own challenges
+ int totalSSTables = columnFamilyStore.getLiveSSTables().size();
+ if (totalSSTables > config.getRepairSSTableCountHigherThreshold(repairType))
+ {
+ logger.info("Too many SSTables for repair for table {}.{}" +
+ "totalSSTables {}", keyspace.getName(), tableName, totalSSTables);
+ collectedRepairStats.skippedTables++;
+ continue;
+ }
+
+ tablesToBeRepaired.add(tableName);
+
+ // See if we should repair MVs as well that are associated with this given table
+ List<String> mvs = AutoRepairUtils.getAllMVs(repairType, keyspace, tableMetadata);
+ if (!mvs.isEmpty())
+ {
+ tablesToBeRepaired.addAll(mvs);
+ repairState.setTotalMVTablesConsideredForRepair(repairState.getTotalMVTablesConsideredForRepair() + mvs.size());
+ }
+ }
+ return tablesToBeRepaired;
+ }
+
+ private void cleanupAndUpdateStats(RepairTurn turn, AutoRepairConfig.RepairType repairType, AutoRepairState repairState, UUID myId,
+ long startTimeInMillis, CollectedRepairStats collectedRepairStats) throws InterruptedException
+ {
+ //if it was due to priority then remove it now
+ if (turn == MY_TURN_DUE_TO_PRIORITY)
+ {
+ logger.info("Remove current host from priority list");
+ AutoRepairUtils.removePriorityStatus(repairType, myId);
+ }
+ long repairScheduleElapsedInMillis = timeFunc.get() - startTimeInMillis;
+ if (repairScheduleElapsedInMillis < SLEEP_IF_REPAIR_FINISHES_QUICKLY.toMilliseconds())
+ {
+ //If repair finished quickly, happens for Cassndra cluster with empty (or tiny) data, in such cases,
+ //wait for some duration so that the JMX metrics can detect the repairInProgress
+ logger.info("Wait for {}ms for repair type {}.", SLEEP_IF_REPAIR_FINISHES_QUICKLY.toMilliseconds() - repairScheduleElapsedInMillis, repairType);
+ Thread.sleep(SLEEP_IF_REPAIR_FINISHES_QUICKLY.toMilliseconds() - repairScheduleElapsedInMillis);
+ }
+ repairState.setFailedTokenRangesCount(collectedRepairStats.failedTokenRanges);
+ repairState.setSucceededTokenRangesCount(collectedRepairStats.succeededTokenRanges);
+ repairState.setSkippedTokenRangesCount(collectedRepairStats.skippedTokenRanges);
+ repairState.setSkippedTablesCount(collectedRepairStats.skippedTables);
+ repairState.setNodeRepairTimeInSec((int) TimeUnit.MILLISECONDS.toSeconds(timeFunc.get() - startTimeInMillis));
+ long timeInHours = TimeUnit.SECONDS.toHours(repairState.getNodeRepairTimeInSec());
+ logger.info("Local {} repair time {} hour(s), stats: repairKeyspaceCount {}, " +
+ "repairTokenRangesSuccessCount {}, repairTokenRangesFailureCount {}, " +
+ "repairTokenRangesSkipCount {}, repairTablesSkipCount {}", repairType, timeInHours, repairState.getRepairKeyspaceCount(),
+ repairState.getSucceededTokenRangesCount(), repairState.getFailedTokenRangesCount(),
+ repairState.getSkippedTokenRangesCount(), repairState.getSkippedTablesCount());
+ if (repairState.getLastRepairTime() != 0)
+ {
+ repairState.setClusterRepairTimeInSec((int) TimeUnit.MILLISECONDS.toSeconds(timeFunc.get() -
+ repairState.getLastRepairTime()));
+ logger.info("Cluster repair time for repair type {}: {} day(s)", repairType,
+ TimeUnit.SECONDS.toDays(repairState.getClusterRepairTimeInSec()));
+ }
+ repairState.setLastRepairTime(timeFunc.get());
+ repairState.setRepairInProgress(false);
+
+ AutoRepairUtils.updateFinishAutoRepairHistory(repairType, myId, timeFunc.get());
+ }
+
+ public AutoRepairState getRepairState(AutoRepairConfig.RepairType repairType)
+ {
+ return repairStates.get(repairType);
+ }
+
+ private void soakAfterRepair(long startTimeMilis, long minDurationMilis)
+ {
+ long currentTime = timeFunc.get();
+ long timeElapsed = currentTime - startTimeMilis;
+ if (timeElapsed < minDurationMilis)
+ {
+ long timeToSoak = minDurationMilis - timeElapsed;
+ logger.info("Soaking for {} ms after repair", timeToSoak);
+ sleepFunc.accept(timeToSoak, TimeUnit.MILLISECONDS);
+ }
+ }
+
+ static class CollectedRepairStats
+ {
+ int failedTokenRanges = 0;
+ int succeededTokenRanges = 0;
+ int skippedTokenRanges = 0;
+ int skippedTables = 0;
+ }
+
+ @VisibleForTesting
+ protected static class RepairProgressListener implements ProgressListener
+ {
+ private final AutoRepairConfig.RepairType repairType;
+ @VisibleForTesting
+ protected boolean success;
+ @VisibleForTesting
+ protected final Condition condition = newOneTimeCondition();
+
+ public RepairProgressListener(AutoRepairConfig.RepairType repairType)
+ {
+ this.repairType = repairType;
+ }
+
+ public void await(DurationSpec.IntSecondsBound repairSessionTimeout) throws InterruptedException
+ {
+ //if for some reason we don't hear back on repair progress for sometime
+ if (!condition.await(repairSessionTimeout.to(TimeUnit.SECONDS), TimeUnit.SECONDS))
+ {
+ success = false;
+ }
+ }
+
+ public boolean isSuccess()
+ {
+ return success;
+ }
+
+ @Override
+ public void progress(String tag, ProgressEvent event)
+ {
+ ProgressEventType type = event.getType();
+ String message = String.format("[%s] %s", format.format(timeFunc.get()), event.getMessage());
+ if (type == ProgressEventType.ERROR)
+ {
+ logger.error("Repair failure for repair {}: {}", repairType.toString(), message);
+ success = false;
+ condition.signalAll();
+ }
+ if (type == ProgressEventType.PROGRESS)
+ {
+ message = message + " (progress: " + (int) event.getProgressPercentage() + "%)";
+ logger.debug("Repair progress for repair {}: {}", repairType.toString(), message);
+ }
+ if (type == ProgressEventType.COMPLETE)
+ {
+ logger.debug("Repair completed for repair {}: {}", repairType.toString(), message);
+ success = true;
+ condition.signalAll();
+ }
+ }
+ }
+
+ public synchronized void shutdownBlocking() throws ExecutionException, InterruptedException
+ {
+ if (!isSetupDone)
+ {
+ // By default, executors within AutoRepair are not initialized as the feature is opt-in.
+ // If the AutoRepair has not been set up, then there is no need to worry about shutting it down
+ return;
+ }
+ if (isShutDown)
+ {
+ throw new IllegalStateException("AutoRepair has already been shut down");
+ }
+ isShutDown = true;
+ for (AutoRepairConfig.RepairType repairType : AutoRepairConfig.RepairType.values())
+ {
+ repairRunnableExecutors.get(repairType).shutdown();
+ repairExecutors.get(repairType).shutdown();
+ }
+ logger.info("Paused AutoRepair");
+ }
+
+ public Map<AutoRepairConfig.RepairType, ScheduledExecutorPlus> getRepairExecutors()
+ {
+ return repairExecutors;
+ }
+
+ public Map<AutoRepairConfig.RepairType, ScheduledExecutorPlus> getRepairRunnableExecutors()
+ {
+ return repairRunnableExecutors;
+ }
+}
diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java
new file mode 100644
index 0000000000..8846d3c088
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java
@@ -0,0 +1,616 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.autorepair;
+
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
+import java.util.function.Function;
+
+import javax.annotation.Nonnull;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Maps;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.config.DurationSpec;
+import org.apache.cassandra.config.ParameterizedClass;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.utils.FBUtilities;
+
+/**
+ * Defines configurations for AutoRepair.
+ */
+public class AutoRepairConfig implements Serializable
+{
+ // Enable/Disable the auto-repair scheduler.
+ // If set to false, the scheduler thread will not be started.
+ // If set to true, the repair scheduler thread will be created. The thread will
+ // check for secondary configuration available for each repair type (full, incremental,
+ // and preview_repaired), and based on that, it will schedule repairs.
+ public volatile Boolean enabled;
+ // Time interval between successive checks to see if ongoing repairs are complete or if it is time to schedule
+ // repairs.
+ public final DurationSpec.IntSecondsBound repair_check_interval = new DurationSpec.IntSecondsBound("5m");
+ // The scheduler needs to adjust its order when nodes leave the ring. Deleted hosts are tracked in metadata
+ // for a specified duration to ensure they are indeed removed before adjustments are made to the schedule.
+ public volatile DurationSpec.IntSecondsBound history_clear_delete_hosts_buffer_interval = new DurationSpec.IntSecondsBound("2h");
+ // Minimum duration for the execution of a single repair task. This prevents the scheduler from overwhelming
+ // the node by scheduling too many repair tasks in a short period of time.
+ public volatile DurationSpec.LongSecondsBound repair_task_min_duration = new DurationSpec.LongSecondsBound("5s");
+ // by default repair is disabled if there are mixed major versions detected, but you can enable it using this flag
+ public volatile boolean mixed_major_version_repair_enabled = false;
+
+ // global_settings overides Options.defaultOptions for all repair types
+ public volatile Options global_settings;
+
+ public static final Class<? extends IAutoRepairTokenRangeSplitter> DEFAULT_SPLITTER = RepairTokenRangeSplitter.class;
+
+ // make transient so gets consturcted in the implementation.
+ private final transient Map<RepairType, IAutoRepairTokenRangeSplitter> tokenRangeSplitters = new EnumMap<>(RepairType.class);
+
+ public enum RepairType implements Serializable
+ {
+ FULL,
+ INCREMENTAL,
+ PREVIEW_REPAIRED;
+
+ private final String configName;
+
+ RepairType()
+ {
+ this.configName = name().toLowerCase();
+ }
+
+ /**
+ * @return Format of the repair type as it should be represented in configuration.
+ * Canonically this is the enum name in lowerCase.
+ */
+ public String getConfigName()
+ {
+ return configName;
+ }
+
+ public static AutoRepairState getAutoRepairState(RepairType repairType, AutoRepairConfig config)
+ {
+ switch (repairType)
+ {
+ case FULL:
+ return new FullRepairState(config);
+ case INCREMENTAL:
+ return new IncrementalRepairState(config);
+ case PREVIEW_REPAIRED:
+ return new PreviewRepairedState(config);
+ }
+
+ throw new IllegalArgumentException("Invalid repair type: " + repairType);
+ }
+
+ /**
+ * Case-insensitive parsing of the repair type string into {@link RepairType}
+ *
+ * @param repairTypeStr the repair type string
+ * @return the {@link RepairType} represented by the {@code repairTypeStr} string
+ * @throws IllegalArgumentException when the repair type string does not match any repair type
+ */
+ public static RepairType parse(String repairTypeStr)
+ {
+ return RepairType.valueOf(Objects.requireNonNull(repairTypeStr, "repairTypeStr cannot be null").toUpperCase());
+ }
+ }
+
+ // repair_type_overrides overrides the global_settings for a specific repair type. String used as key instead
+ // of enum to allow lower case key in yaml.
+ public volatile ConcurrentMap<String, Options> repair_type_overrides = Maps.newConcurrentMap();
+
+ public AutoRepairConfig()
+ {
+ this(false);
+ }
+
+ public AutoRepairConfig(boolean enabled)
+ {
+ this.enabled = enabled;
+ global_settings = Options.getDefaultOptions();
+ }
+
+ public DurationSpec.IntSecondsBound getRepairCheckInterval()
+ {
+ return repair_check_interval;
+ }
+
+ public boolean isAutoRepairSchedulingEnabled()
+ {
+ return CassandraRelevantProperties.AUTOREPAIR_ENABLE.getBoolean() && enabled;
+ }
+
+ @VisibleForTesting
+ public void setAutoRepairSchedulingEnabled(boolean enabled)
+ {
+ this.enabled = enabled;
+ }
+
+ public boolean isMixedMajorVersionRepairEnabled()
+ {
+ return mixed_major_version_repair_enabled;
+ }
+
+ public DurationSpec.IntSecondsBound getAutoRepairHistoryClearDeleteHostsBufferInterval()
+ {
+ return history_clear_delete_hosts_buffer_interval;
+ }
+
+ public void startScheduler()
+ {
+ enabled = true;
+ AutoRepair.instance.setup();
+ }
+
+ public void setAutoRepairHistoryClearDeleteHostsBufferInterval(String duration)
+ {
+ history_clear_delete_hosts_buffer_interval = new DurationSpec.IntSecondsBound(duration);
+ }
+
+ public DurationSpec.LongSecondsBound getRepairTaskMinDuration()
+ {
+ return repair_task_min_duration;
+ }
+
+ public void setRepairTaskMinDuration(String duration)
+ {
+ repair_task_min_duration = new DurationSpec.LongSecondsBound(duration);
+ }
+
+ public boolean isAutoRepairEnabled(RepairType repairType)
+ {
+ return enabled && applyOverrides(repairType, opt -> opt.enabled);
+ }
+
+ public void setAutoRepairEnabled(RepairType repairType, boolean enabled)
+ {
+ getOptions(repairType).enabled = enabled;
+ }
+
+ public void setRepairByKeyspace(RepairType repairType, boolean repairByKeyspace)
+ {
+ getOptions(repairType).repair_by_keyspace = repairByKeyspace;
+ }
+
+ public boolean getRepairByKeyspace(RepairType repairType)
+ {
+ return applyOverrides(repairType, opt -> opt.repair_by_keyspace);
+ }
+
+ public int getRepairThreads(RepairType repairType)
+ {
+ return applyOverrides(repairType, opt -> opt.number_of_repair_threads);
+ }
+
+ public void setRepairThreads(RepairType repairType, int repairThreads)
+ {
+ getOptions(repairType).number_of_repair_threads = repairThreads;
+ }
+
+ public DurationSpec.IntSecondsBound getRepairMinInterval(RepairType repairType)
+ {
+ return applyOverrides(repairType, opt -> opt.min_repair_interval);
+ }
+
+ public void setRepairMinInterval(RepairType repairType, String minRepairInterval)
+ {
+ getOptions(repairType).min_repair_interval = new DurationSpec.IntSecondsBound(minRepairInterval);
+ }
+
+ public int getRepairSSTableCountHigherThreshold(RepairType repairType)
+ {
+ return applyOverrides(repairType, opt -> opt.sstable_upper_threshold);
+ }
+
+ public void setRepairSSTableCountHigherThreshold(RepairType repairType, int sstableHigherThreshold)
+ {
+ getOptions(repairType).sstable_upper_threshold = sstableHigherThreshold;
+ }
+
+ public DurationSpec.IntSecondsBound getAutoRepairTableMaxRepairTime(RepairType repairType)
+ {
+ return applyOverrides(repairType, opt -> opt.table_max_repair_time);
+ }
+
+ public void setAutoRepairTableMaxRepairTime(RepairType repairType, String autoRepairTableMaxRepairTime)
+ {
+ getOptions(repairType).table_max_repair_time = new DurationSpec.IntSecondsBound(autoRepairTableMaxRepairTime);
+ }
+
+ public Set<String> getIgnoreDCs(RepairType repairType)
+ {
+ return applyOverrides(repairType, opt -> opt.ignore_dcs);
+ }
+
+ public void setIgnoreDCs(RepairType repairType, Set<String> ignoreDCs)
+ {
+ getOptions(repairType).ignore_dcs = ignoreDCs;
+ }
+
+ public boolean getRepairPrimaryTokenRangeOnly(RepairType repairType)
+ {
+ return applyOverrides(repairType, opt -> opt.repair_primary_token_range_only);
+ }
+
+ public void setRepairPrimaryTokenRangeOnly(RepairType repairType, boolean primaryTokenRangeOnly)
+ {
+ getOptions(repairType).repair_primary_token_range_only = primaryTokenRangeOnly;
+ }
+
+ public int getParallelRepairPercentage(RepairType repairType)
+ {
+ return applyOverrides(repairType, opt -> opt.parallel_repair_percentage);
+ }
+
+ public void setParallelRepairPercentage(RepairType repairType, int percentage)
+ {
+ getOptions(repairType).parallel_repair_percentage = percentage;
+ }
+
+ public int getParallelRepairCount(RepairType repairType)
+ {
+ return applyOverrides(repairType, opt -> opt.parallel_repair_count);
+ }
+
+ public void setParallelRepairCount(RepairType repairType, int count)
+ {
+ getOptions(repairType).parallel_repair_count = count;
+ }
+
+ public boolean getAllowParallelReplicaRepair(RepairType repairType)
+ {
+ return applyOverrides(repairType, opt -> opt.allow_parallel_replica_repair);
+ }
+
+ public void setAllowParallelReplicaRepair(RepairType repairType, boolean enabled)
+ {
+ getOptions(repairType).allow_parallel_replica_repair = enabled;
+ }
+
+ public boolean getAllowParallelReplicaRepairAcrossSchedules(RepairType repairType)
+ {
+ return applyOverrides(repairType, opt -> opt.allow_parallel_replica_repair_across_schedules);
+ }
+
+ public void setAllowParallelReplicaRepairAcrossSchedules(RepairType repairType, boolean enabled)
+ {
+ getOptions(repairType).allow_parallel_replica_repair_across_schedules = enabled;
+ }
+
+ public boolean getMaterializedViewRepairEnabled(RepairType repairType)
+ {
+ return applyOverrides(repairType, opt -> opt.materialized_view_repair_enabled);
+ }
+
+ public void setMaterializedViewRepairEnabled(RepairType repairType, boolean enabled)
+ {
+ getOptions(repairType).materialized_view_repair_enabled = enabled;
+ }
+
+ public void setForceRepairNewNode(RepairType repairType, boolean forceRepairNewNode)
+ {
+ getOptions(repairType).force_repair_new_node = forceRepairNewNode;
+ }
+
+ public boolean getForceRepairNewNode(RepairType repairType)
+ {
+ return applyOverrides(repairType, opt -> opt.force_repair_new_node);
+ }
+
+ public ParameterizedClass getTokenRangeSplitter(RepairType repairType)
+ {
+ return applyOverrides(repairType, opt -> opt.token_range_splitter);
+ }
+
+ public IAutoRepairTokenRangeSplitter getTokenRangeSplitterInstance(RepairType repairType)
+ {
+ return tokenRangeSplitters.computeIfAbsent(repairType,
+ key -> newAutoRepairTokenRangeSplitter(key, getTokenRangeSplitter(key)));
+ }
+
+ public void setInitialSchedulerDelay(RepairType repairType, String initialSchedulerDelay)
+ {
+ getOptions(repairType).initial_scheduler_delay = new DurationSpec.IntSecondsBound(initialSchedulerDelay);
+ }
+
+ public DurationSpec.IntSecondsBound getInitialSchedulerDelay(RepairType repairType)
+ {
+ return applyOverrides(repairType, opt -> opt.initial_scheduler_delay);
+ }
+
+ public DurationSpec.IntSecondsBound getRepairSessionTimeout(RepairType repairType)
+ {
+ return applyOverrides(repairType, opt -> opt.repair_session_timeout);
+ }
+
+ public void setRepairSessionTimeout(RepairType repairType, String repairSessionTimeout)
+ {
+ getOptions(repairType).repair_session_timeout = new DurationSpec.IntSecondsBound(repairSessionTimeout);
+ }
+
+ public int getRepairMaxRetries(RepairType repairType)
+ {
+ return applyOverrides(repairType, opt -> opt.repair_max_retries);
+ }
+
+ public void setRepairMaxRetries(RepairType repairType, int maxRetries)
+ {
+ getOptions(repairType).repair_max_retries = maxRetries;
+ }
+
+ public DurationSpec.LongSecondsBound getRepairRetryBackoff(RepairType repairType)
+ {
+ return applyOverrides(repairType, opt -> opt.repair_retry_backoff);
+ }
+
+ public void setRepairRetryBackoff(RepairType repairType, String interval)
+ {
+ getOptions(repairType).repair_retry_backoff = new DurationSpec.LongSecondsBound(interval);
+ }
+
+ public boolean getMixedMajorVersionRepairEnabled()
+ {
+ return this.mixed_major_version_repair_enabled;
+ }
+
+ public void setMixedMajorVersionRepairEnabled(boolean enabled)
+ {
+ this.mixed_major_version_repair_enabled = enabled;
+ }
+
+ @VisibleForTesting
+ static IAutoRepairTokenRangeSplitter newAutoRepairTokenRangeSplitter(RepairType repairType, ParameterizedClass parameterizedClass) throws ConfigurationException
+ {
+ try
+ {
+ Class<? extends IAutoRepairTokenRangeSplitter> tokenRangeSplitterClass;
+ final String className;
+ if (parameterizedClass.class_name != null && !parameterizedClass.class_name.isEmpty())
+ {
+ className = parameterizedClass.class_name.contains(".") ?
+ parameterizedClass.class_name :
+ "org.apache.cassandra.repair.autorepair." + parameterizedClass.class_name;
+ tokenRangeSplitterClass = FBUtilities.classForName(className, "token_range_splitter");
+ }
+ else
+ {
+ // If token_range_splitter.class_name is not defined, just use default, this is for convenience.
+ tokenRangeSplitterClass = AutoRepairConfig.DEFAULT_SPLITTER;
+ }
+ try
+ {
+ Map<String, String> parameters = parameterizedClass.parameters != null ? parameterizedClass.parameters : Collections.emptyMap();
+ // first attempt to initialize with RepairType and Map arguments.
+ return tokenRangeSplitterClass.getConstructor(RepairType.class, Map.class).newInstance(repairType, parameters);
+ }
+ catch (NoSuchMethodException nsme)
+ {
+ // fall back on no argument constructor.
+ return tokenRangeSplitterClass.getConstructor().newInstance();
+ }
+ }
+ catch (Exception ex)
+ {
+ throw new ConfigurationException("Unable to create instance of IAutoRepairTokenRangeSplitter", ex);
+ }
+ }
+
+ // Options configures auto-repair behavior for a given repair type.
+ // All fields can be modified dynamically.
+ public static class Options implements Serializable
+ {
+ // defaultOptions defines the default auto-repair behavior when no overrides are defined
+ @VisibleForTesting
+ private static Map<AutoRepairConfig.RepairType, Options> defaultOptions;
+
+ private static Map<AutoRepairConfig.RepairType, Options> initializeDefaultOptions()
+ {
+ Map<AutoRepairConfig.RepairType, Options> options = new EnumMap<>(AutoRepairConfig.RepairType.class);
+ options.put(AutoRepairConfig.RepairType.FULL, getDefaultOptions());
+ options.put(RepairType.INCREMENTAL, getDefaultOptions());
+ options.put(RepairType.PREVIEW_REPAIRED, getDefaultOptions());
+
+ return options;
+ }
+
+ public static Map<AutoRepairConfig.RepairType, Options> getDefaultOptionsMap()
+ {
+ if (defaultOptions == null)
+ {
+ synchronized (AutoRepairConfig.class)
+ {
+ if (defaultOptions == null)
+ {
+ defaultOptions = initializeDefaultOptions();
+ }
+ }
+ }
+ return defaultOptions;
+ }
+
+ public Options()
+ {
+ }
+
+ @VisibleForTesting
+ protected static Options getDefaultOptions()
+ {
+ Options opts = new Options();
+
+ opts.enabled = false;
+ opts.repair_by_keyspace = true;
+ opts.number_of_repair_threads = 1;
+ opts.parallel_repair_count = 3;
+ opts.parallel_repair_percentage = 3;
+ opts.allow_parallel_replica_repair = false;
+ opts.allow_parallel_replica_repair_across_schedules = true;
+ opts.sstable_upper_threshold = 50000;
+ opts.ignore_dcs = new HashSet<>();
+ opts.repair_primary_token_range_only = true;
+ opts.force_repair_new_node = false;
+ opts.table_max_repair_time = new DurationSpec.IntSecondsBound("6h");
+ opts.materialized_view_repair_enabled = false;
+ opts.token_range_splitter = new ParameterizedClass(DEFAULT_SPLITTER.getName(), Collections.emptyMap());
+ opts.initial_scheduler_delay = new DurationSpec.IntSecondsBound("5m");
+ opts.repair_session_timeout = new DurationSpec.IntSecondsBound("3h");
+ opts.min_repair_interval = new DurationSpec.IntSecondsBound("24h");
+
+ return opts;
+ }
+
+ // Enable/Disable full or incremental or previewed_repair auto repair
+ public volatile Boolean enabled;
+ // If true, attempts to group tables in the same keyspace into one repair; otherwise, each table is repaired
+ // individually.
+ public volatile Boolean repair_by_keyspace;
+ // Number of threads to use for each repair job scheduled by the scheduler. Similar to the -j option in nodetool
+ // repair.
+ public volatile Integer number_of_repair_threads;
+ // Number of nodes running repair in parallel. If parallel_repair_percentage is set, the larger value is used.
+ public volatile Integer parallel_repair_count;
+ // Percentage of nodes in the cluster running repair in parallel. If parallel_repair_count is set, the larger value
+ // is used. Recommendation is that the repair cycle on the cluster should finish within gc_grace_seconds.
+ public volatile Integer parallel_repair_percentage;
+ // Whether to allow a node to take its turn running repair while one or more of its replicas are running repair.
+ // Defaults to false, as running repairs concurrently on replicas can increase load and also cause
+ // anticompaction conflicts while running incremental repair.
+ public volatile Boolean allow_parallel_replica_repair;
+ // An addition to allow_parallel_replica_repair that also blocks repairs when replicas (including this node itself)
+ // are repairing in any schedule. For example, if a replica is executing full repairs, a value of false will
+ // prevent starting incremental repairs for this node. Defaults to true and is only evaluated when
+ // allow_parallel_replica_repair is false.
+ public volatile Boolean allow_parallel_replica_repair_across_schedules;
+ // Threshold to skip repairing tables with too many SSTables. Defaults to 10,000 SSTables to avoid penalizing good
+ // tables.
+ public volatile Integer sstable_upper_threshold;
+ // Minimum duration between repairing the same node again. This is useful for tiny clusters, such as
+ // clusters with 5 nodes that finish repairs quickly. The default is 24 hours. This means that if the scheduler
+ // completes one round on all nodes in less than 24 hours, it will not start a new repair round on a given node
+ // until 24 hours have passed since the last repair.
+ public volatile DurationSpec.IntSecondsBound min_repair_interval;
+ // Avoid running repairs in specific data centers. By default, repairs run in all data centers. Specify data
+ // centers to exclude in this list. Note that repair sessions will still consider all replicas from excluded
+ // data centers. Useful if you have keyspaces that are not replicated in certain data centers, and you want to
+ // not run repair schedule in certain data centers.
+ public volatile Set<String> ignore_dcs;
+ // Repair only the primary ranges owned by a node. Equivalent to the -pr option in nodetool repair. Defaults
+ // to true. General advice is to keep this true.
+ public volatile Boolean repair_primary_token_range_only;
+ // Force immediate repair on new nodes after they join the ring.
+ public volatile Boolean force_repair_new_node;
+ // Maximum time allowed for repairing one table on a given node. If exceeded, the repair proceeds to the
+ // next table.
+ public volatile DurationSpec.IntSecondsBound table_max_repair_time;
+ // Repairs materialized views if true.
+ public volatile Boolean materialized_view_repair_enabled;
+ /**
+ * Splitter implementation to use for generating repair assignments.
+ * <p>
+ * The default is {@link RepairTokenRangeSplitter}. The class should implement {@link IAutoRepairTokenRangeSplitter}
+ * and have a constructor accepting ({@link RepairType}, {@link java.util.Map})
+ */
+ public volatile ParameterizedClass token_range_splitter;
+ // After a node restart, wait for this much delay before scheduler starts running repair; this is to avoid starting repair immediately after a node restart.
+ public volatile DurationSpec.IntSecondsBound initial_scheduler_delay;
+ // Timeout for retrying stuck repair sessions.
+ public volatile DurationSpec.IntSecondsBound repair_session_timeout;
+ // Maximum number of retries for a repair session.
+ public volatile Integer repair_max_retries = 3;
+ // Backoff time before retrying a repair session.
+ public volatile DurationSpec.LongSecondsBound repair_retry_backoff = new DurationSpec.LongSecondsBound("30s");
+
+ public String toString()
+ {
+ return "Options{" +
+ "enabled=" + enabled +
+ ", repair_by_keyspace=" + repair_by_keyspace +
+ ", number_of_repair_threads=" + number_of_repair_threads +
+ ", parallel_repair_count=" + parallel_repair_count +
+ ", parallel_repair_percentage=" + parallel_repair_percentage +
+ ", allow_parallel_replica_repair=" + allow_parallel_replica_repair +
+ ", allow_parallel_replica_repair_across_schedules=" + allow_parallel_replica_repair_across_schedules +
+ ", sstable_upper_threshold=" + sstable_upper_threshold +
+ ", min_repair_interval=" + min_repair_interval +
+ ", ignore_dcs=" + ignore_dcs +
+ ", repair_primary_token_range_only=" + repair_primary_token_range_only +
+ ", force_repair_new_node=" + force_repair_new_node +
+ ", table_max_repair_time=" + table_max_repair_time +
+ ", materialized_view_repair_enabled=" + materialized_view_repair_enabled +
+ ", token_range_splitter=" + token_range_splitter +
+ ", intial_scheduler_delay=" + initial_scheduler_delay +
+ ", repair_session_timeout=" + repair_session_timeout +
+ '}';
+ }
+ }
+
+ @Nonnull
+ protected Options getOptions(RepairType repairType)
+ {
+ return repair_type_overrides.computeIfAbsent(repairType.getConfigName(), k -> new Options());
+ }
+
+ private static <T> T getOverride(Options options, Function<Options, T> optionSupplier)
+ {
+ return options != null ? optionSupplier.apply(options) : null;
+ }
+
+ @VisibleForTesting
+ protected <T> T applyOverrides(RepairType repairType, Function<Options, T> optionSupplier)
+ {
+ // Check option by repair type first
+ Options repairTypeOverrides = getOptions(repairType);
+ T val = optionSupplier.apply(repairTypeOverrides);
+
+ if (val != null)
+ return val;
+
+ // Check option in global settings
+ if (global_settings != null)
+ {
+ val = getOverride(global_settings, optionSupplier);
+
+ if (val != null)
+ return val;
+ }
+
+ // Otherwise check defaults
+ return getOverride(Options.getDefaultOptionsMap().get(repairType), optionSupplier);
+ }
+
+ public String toString()
+ {
+ return "AutoRepairConfig{" +
+ "enabled=" + enabled +
+ ", repair_check_interval=" + repair_check_interval +
+ ", history_clear_delete_hosts_buffer_interval=" + history_clear_delete_hosts_buffer_interval +
+ ", repair_task_min_duration=" + repair_task_min_duration +
+ ", global_settings=" + global_settings +
+ ", repair_type_overrides=" + repair_type_overrides +
+ "}";
+ }
+}
diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java
new file mode 100644
index 0000000000..bf319b0fd0
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java
@@ -0,0 +1,386 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.autorepair;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.view.TableViews;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.metrics.AutoRepairMetricsManager;
+import org.apache.cassandra.metrics.AutoRepairMetrics;
+import org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType;
+import org.apache.cassandra.repair.autorepair.AutoRepairUtils.AutoRepairHistory;
+import org.apache.cassandra.repair.RepairParallelism;
+import org.apache.cassandra.repair.RepairCoordinator;
+import org.apache.cassandra.repair.messages.RepairOption;
+import org.apache.cassandra.service.AutoRepairService;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.utils.Clock;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.text.SimpleDateFormat;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
+/**
+ * AutoRepairState represents the state of automated repair for a given repair type.
+ */
+public abstract class AutoRepairState
+{
+ protected static final Logger logger = LoggerFactory.getLogger(AutoRepairState.class);
+ private final SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,SSS");
+ @VisibleForTesting
+ protected static Supplier<Long> timeFunc = Clock.Global::currentTimeMillis;
+
+ @VisibleForTesting
+ protected final RepairType repairType;
+ @VisibleForTesting
+ protected AutoRepairConfig config;
+ @VisibleForTesting
+ protected int totalTablesConsideredForRepair = 0;
+ @VisibleForTesting
+ protected long lastRepairTimeInMs;
+ @VisibleForTesting
+ protected int nodeRepairTimeInSec = 0;
+ @VisibleForTesting
+ protected int clusterRepairTimeInSec = 0;
+ @VisibleForTesting
+ protected boolean repairInProgress = false;
+ @VisibleForTesting
+ protected int repairKeyspaceCount = 0;
+ @VisibleForTesting
+ protected int totalMVTablesConsideredForRepair = 0;
+ @VisibleForTesting
+ protected int totalDisabledTablesRepairCount = 0;
+ @VisibleForTesting
+ protected int failedTokenRangesCount = 0;
+ @VisibleForTesting
+ protected int succeededTokenRangesCount = 0;
+ @VisibleForTesting
+ protected int skippedTokenRangesCount = 0;
+ @VisibleForTesting
+ protected int skippedTablesCount = 0;
+ @VisibleForTesting
+ protected long totalBytesToRepair = 0;
+ @VisibleForTesting
+ protected long bytesAlreadyRepaired = 0;
+ @VisibleForTesting
+ protected int totalKeyspaceRepairPlansToRepair = 0;
+ @VisibleForTesting
+ protected int keyspaceRepairPlansAlreadyRepaired = 0;
+ @VisibleForTesting
+ protected AutoRepairHistory longestUnrepairedNode;
+ protected final AutoRepairMetrics metrics;
+
+ protected AutoRepairState(RepairType repairType, AutoRepairConfig config)
+ {
+ metrics = AutoRepairMetricsManager.getMetrics(repairType);
+ this.repairType = repairType;
+ this.config = config;
+ }
+
+ public abstract RepairCoordinator getRepairRunnable(String keyspace, List<String> tables, Set<Range<Token>> ranges, boolean primaryRangeOnly);
+
+ protected RepairCoordinator getRepairRunnable(String keyspace, RepairOption options)
+ {
+ return new RepairCoordinator(StorageService.instance, StorageService.nextRepairCommand.incrementAndGet(),
+ options, keyspace);
+ }
+
+ public void updateRepairScheduleStatistics(List<PrioritizedRepairPlan> repairPlans)
+ {
+ setTotalBytesToRepair(repairPlans.stream().
+ flatMap(repairPlan -> repairPlan.getKeyspaceRepairPlans().
+ stream()).mapToLong(KeyspaceRepairPlan::getEstimatedBytes).sum());
+ setTotalKeyspaceRepairPlansToRepair(repairPlans.stream().mapToInt(repairPlan -> repairPlan.getKeyspaceRepairPlans().size()).sum());
+ }
+
+ public long getLastRepairTime()
+ {
+ return lastRepairTimeInMs;
+ }
+
+ public void setTotalTablesConsideredForRepair(int count)
+ {
+ totalTablesConsideredForRepair = count;
+ }
+
+ public int getTotalTablesConsideredForRepair()
+ {
+ return totalTablesConsideredForRepair;
+ }
+
+ public void setLastRepairTime(long lastRepairTime)
+ {
+ lastRepairTimeInMs = lastRepairTime;
+ }
+
+ public int getClusterRepairTimeInSec()
+ {
+ return clusterRepairTimeInSec;
+ }
+
+ public int getNodeRepairTimeInSec()
+ {
+ return nodeRepairTimeInSec;
+ }
+
+ public void setRepairInProgress(boolean repairInProgress)
+ {
+ this.repairInProgress = repairInProgress;
+ }
+
+ public boolean isRepairInProgress()
+ {
+ return repairInProgress;
+ }
+
+ public int getLongestUnrepairedSec()
+ {
+ if (longestUnrepairedNode == null)
+ {
+ return 0;
+ }
+ return (int) TimeUnit.MILLISECONDS.toSeconds(timeFunc.get() - longestUnrepairedNode.getLastRepairFinishTime());
+ }
+
+ public void setTotalMVTablesConsideredForRepair(int count)
+ {
+ totalMVTablesConsideredForRepair = count;
+ }
+
+ public int getTotalMVTablesConsideredForRepair()
+ {
+ return totalMVTablesConsideredForRepair;
+ }
+
+ public void setNodeRepairTimeInSec(int elapsed)
+ {
+ nodeRepairTimeInSec = elapsed;
+ }
+
+ public void setClusterRepairTimeInSec(int seconds)
+ {
+ clusterRepairTimeInSec = seconds;
+ }
+
+ public void setRepairKeyspaceCount(int count)
+ {
+ repairKeyspaceCount = count;
+ }
+
+ public int getRepairKeyspaceCount()
+ {
+ return repairKeyspaceCount;
+ }
+
+ public void setLongestUnrepairedNode(AutoRepairHistory longestUnrepairedNode)
+ {
+ this.longestUnrepairedNode = longestUnrepairedNode;
+ }
+
+ public void setFailedTokenRangesCount(int count)
+ {
+ failedTokenRangesCount = count;
+ }
+
+ public int getFailedTokenRangesCount()
+ {
+ return failedTokenRangesCount;
+ }
+
+ public void setSucceededTokenRangesCount(int count)
+ {
+ succeededTokenRangesCount = count;
+ }
+
+ public int getSucceededTokenRangesCount()
+ {
+ return succeededTokenRangesCount;
+ }
+
+ public void setSkippedTokenRangesCount(int count)
+ {
+ skippedTokenRangesCount = count;
+ }
+
+ public int getSkippedTokenRangesCount()
+ {
+ return skippedTokenRangesCount;
+ }
+
+ public void setSkippedTablesCount(int count)
+ {
+ skippedTablesCount = count;
+ }
+
+ public int getSkippedTablesCount()
+ {
+ return skippedTablesCount;
+ }
+
+ public void recordTurn(AutoRepairUtils.RepairTurn turn)
+ {
+ metrics.recordTurn(turn);
+ }
+
+ public void setTotalDisabledTablesRepairCount(int count)
+ {
+ totalDisabledTablesRepairCount = count;
+ }
+
+ public int getTotalDisabledTablesRepairCount()
+ {
+ return totalDisabledTablesRepairCount;
+ }
+
+ public void setTotalBytesToRepair(long totalBytesToRepair)
+ {
+ this.totalBytesToRepair = totalBytesToRepair;
+ }
+
+ public long getTotalBytesToRepair()
+ {
+ return totalBytesToRepair;
+ }
+
+ public void setBytesAlreadyRepaired(long bytesAlreadyRepaired)
+ {
+ this.bytesAlreadyRepaired = bytesAlreadyRepaired;
+ }
+
+ public long getBytesAlreadyRepaired()
+ {
+ return bytesAlreadyRepaired;
+ }
+
+ public void setTotalKeyspaceRepairPlansToRepair(int totalKeyspaceRepairPlansToRepair)
+ {
+ this.totalKeyspaceRepairPlansToRepair = totalKeyspaceRepairPlansToRepair;
+ }
+
+ public int getTotalKeyspaceRepairPlansToRepair()
+ {
+ return totalKeyspaceRepairPlansToRepair;
+ }
+
+ public void setKeyspaceRepairPlansAlreadyRepaired(int keyspaceRepairPlansAlreadyRepaired)
+ {
+ this.keyspaceRepairPlansAlreadyRepaired = keyspaceRepairPlansAlreadyRepaired;
+ }
+
+ public int getKeyspaceRepairPlansAlreadyRepaired()
+ {
+ return keyspaceRepairPlansAlreadyRepaired;
+ }
+}
+
+class PreviewRepairedState extends AutoRepairState
+{
+ public PreviewRepairedState(AutoRepairConfig config)
+ {
+ super(RepairType.PREVIEW_REPAIRED, config);
+ }
+
+ @Override
+ public RepairCoordinator getRepairRunnable(String keyspace, List<String> tables, Set<Range<Token>> ranges, boolean primaryRangeOnly)
+ {
+ RepairOption option = new RepairOption(RepairParallelism.PARALLEL, primaryRangeOnly, false, false,
+ AutoRepairService.instance.getAutoRepairConfig().getRepairThreads(repairType), ranges,
+ !ranges.isEmpty(), false, false, PreviewKind.REPAIRED, false, true, false, false);
+
+ option.getColumnFamilies().addAll(tables);
+
+ return getRepairRunnable(keyspace, option);
+ }
+}
+
+class IncrementalRepairState extends AutoRepairState
+{
+ public IncrementalRepairState(AutoRepairConfig config)
+ {
+ super(RepairType.INCREMENTAL, config);
+ }
+
+ @Override
+ public RepairCoordinator getRepairRunnable(String keyspace, List<String> tables, Set<Range<Token>> ranges, boolean primaryRangeOnly)
+ {
+ RepairOption option = new RepairOption(RepairParallelism.PARALLEL, primaryRangeOnly, true, false,
+ AutoRepairService.instance.getAutoRepairConfig().getRepairThreads(repairType), ranges,
+ !ranges.isEmpty(), false, false, PreviewKind.NONE, true, true, false, false);
+
+ option.getColumnFamilies().addAll(filterOutUnsafeTables(keyspace, tables));
+
+ return getRepairRunnable(keyspace, option);
+ }
+
+ @VisibleForTesting
+ protected List<String> filterOutUnsafeTables(String keyspaceName, List<String> tables)
+ {
+ Keyspace keyspace = Keyspace.open(keyspaceName);
+
+ return tables.stream()
+ .filter(table -> {
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(table);
+ TableViews views = keyspace.viewManager.forTable(cfs.metadata().id);
+ if (views != null && !views.isEmpty())
+ {
+ logger.debug("Skipping incremental repair for {}.{} as it has materialized views", keyspaceName, table);
+ return false;
+ }
+
+ if (cfs.metadata().params != null && cfs.metadata().params.cdc)
+ {
+ logger.debug("Skipping incremental repair for {}.{} as it has CDC enabled", keyspaceName, table);
+ return false;
+ }
+
+ return true;
+ }).collect(Collectors.toList());
+ }
+}
+
+class FullRepairState extends AutoRepairState
+{
+ public FullRepairState(AutoRepairConfig config)
+ {
+ super(RepairType.FULL, config);
+ }
+
+ @Override
+ public RepairCoordinator getRepairRunnable(String keyspace, List<String> tables, Set<Range<Token>> ranges, boolean primaryRangeOnly)
+ {
+ RepairOption option = new RepairOption(RepairParallelism.PARALLEL, primaryRangeOnly, false, false,
+ AutoRepairService.instance.getAutoRepairConfig().getRepairThreads(repairType), ranges,
+ !ranges.isEmpty(), false, false, PreviewKind.NONE, true, true, false, false);
+
+ option.getColumnFamilies().addAll(tables);
+
+ return getRepairRunnable(keyspace, option);
+ }
+}
diff --git a/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java
new file mode 100644
index 0000000000..82dd78caa1
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java
@@ -0,0 +1,1409 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.repair.autorepair;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import javax.annotation.Nullable;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.MoreObjects;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+
+import com.clearspring.analytics.stream.cardinality.CardinalityMergeException;
+import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus;
+import com.clearspring.analytics.stream.cardinality.ICardinality;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Splitter;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.metadata.CompactionMetadata;
+import org.apache.cassandra.io.sstable.metadata.MetadataType;
+import org.apache.cassandra.locator.EndpointsByRange;
+import org.apache.cassandra.locator.EndpointsForRange;
+import org.apache.cassandra.locator.LocalStrategy;
+
+import org.apache.cassandra.utils.CassandraVersion;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.cql3.statements.ModificationStatement;
+import org.apache.cassandra.cql3.statements.SelectStatement;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.UUIDType;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.locator.AbstractReplicationStrategy;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.NetworkTopologyStrategy;
+import org.apache.cassandra.locator.RangesAtEndpoint;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.metrics.AutoRepairMetricsManager;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.SystemDistributedKeyspace;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.ViewMetadata;
+import org.apache.cassandra.serializers.SetSerializer;
+import org.apache.cassandra.serializers.UUIDSerializer;
+import org.apache.cassandra.service.AutoRepairService;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.transport.Dispatcher;
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.transport.messages.ResultMessage;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType;
+import org.apache.cassandra.utils.NoSpamLogger;
+import org.apache.cassandra.utils.concurrent.Refs;
+
+import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.RepairTurn.MY_TURN;
+import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.RepairTurn.MY_TURN_DUE_TO_PRIORITY;
+import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.RepairTurn.NOT_MY_TURN;
+import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.RepairTurn.MY_TURN_FORCE_REPAIR;
+import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
+
+/**
+ * This class serves as a utility class for AutoRepair. It contains various helper APIs
+ * to store/retrieve repair status, decide whose turn is next, etc.
+ */
+public class AutoRepairUtils
+{
+ private static final Logger logger = LoggerFactory.getLogger(AutoRepairUtils.class);
+ static final String COL_REPAIR_TYPE = "repair_type";
+ static final String COL_HOST_ID = "host_id";
+ static final String COL_REPAIR_START_TS = "repair_start_ts";
+ static final String COL_REPAIR_FINISH_TS = "repair_finish_ts";
+ static final String COL_REPAIR_PRIORITY = "repair_priority";
+ static final String COL_DELETE_HOSTS = "delete_hosts"; // this set stores the host ids which think the row should be deleted
+ static final String COL_REPAIR_TURN = "repair_turn"; // this record the last repair turn. Normal turn or turn due to priority
+ static final String COL_DELETE_HOSTS_UPDATE_TIME = "delete_hosts_update_time"; // the time when delete hosts are upated
+ static final String COL_FORCE_REPAIR = "force_repair"; // if set to true, the node will do non-primary range rapair
+
+ final static String SELECT_REPAIR_HISTORY = String.format(
+ "SELECT * FROM %s.%s WHERE %s = ?", SchemaConstants.DISTRIBUTED_KEYSPACE_NAME,
+ SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, COL_REPAIR_TYPE);
+ final static String SELECT_REPAIR_PRIORITY = String.format(
+ "SELECT * FROM %s.%s WHERE %s = ?", SchemaConstants.DISTRIBUTED_KEYSPACE_NAME,
+ SystemDistributedKeyspace.AUTO_REPAIR_PRIORITY, COL_REPAIR_TYPE);
+ final static String DEL_REPAIR_PRIORITY = String.format(
+ "DELETE %s[?] FROM %s.%s WHERE %s = ?", COL_REPAIR_PRIORITY, SchemaConstants.DISTRIBUTED_KEYSPACE_NAME,
+ SystemDistributedKeyspace.AUTO_REPAIR_PRIORITY, COL_REPAIR_TYPE);
+ final static String ADD_PRIORITY_HOST = String.format(
+ "UPDATE %s.%s SET %s = %s + ? WHERE %s = ?", SchemaConstants.DISTRIBUTED_KEYSPACE_NAME,
+ SystemDistributedKeyspace.AUTO_REPAIR_PRIORITY, COL_REPAIR_PRIORITY, COL_REPAIR_PRIORITY, COL_REPAIR_TYPE);
+
+ final static String INSERT_NEW_REPAIR_HISTORY = String.format(
+ "INSERT INTO %s.%s (%s, %s, %s, %s, %s, %s) values (?, ? ,?, ?, {}, ?) IF NOT EXISTS",
+ SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, COL_REPAIR_TYPE,
+ COL_HOST_ID, COL_REPAIR_START_TS, COL_REPAIR_FINISH_TS, COL_DELETE_HOSTS, COL_DELETE_HOSTS_UPDATE_TIME);
+
+ final static String ADD_HOST_ID_TO_DELETE_HOSTS = String.format(
+ "UPDATE %s.%s SET %s = %s + ?, %s = ? WHERE %s = ? AND %s = ? IF EXISTS"
+ , SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, COL_DELETE_HOSTS,
+ COL_DELETE_HOSTS, COL_DELETE_HOSTS_UPDATE_TIME, COL_REPAIR_TYPE, COL_HOST_ID);
+
+ final static String DEL_AUTO_REPAIR_HISTORY = String.format(
+ "DELETE FROM %s.%s WHERE %s = ? AND %s = ?"
+ , SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, COL_REPAIR_TYPE,
+ COL_HOST_ID);
+
+ final static String RECORD_START_REPAIR_HISTORY = String.format(
+ "UPDATE %s.%s SET %s= ?, repair_turn = ? WHERE %s = ? AND %s = ?"
+ , SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, COL_REPAIR_START_TS,
+ COL_REPAIR_TYPE, COL_HOST_ID);
+
+ final static String RECORD_FINISH_REPAIR_HISTORY = String.format(
+ "UPDATE %s.%s SET %s= ?, %s=false WHERE %s = ? AND %s = ?"
+ , SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, COL_REPAIR_FINISH_TS,
+ COL_FORCE_REPAIR, COL_REPAIR_TYPE, COL_HOST_ID);
+
+ final static String CLEAR_DELETE_HOSTS = String.format(
+ "UPDATE %s.%s SET %s= {} WHERE %s = ? AND %s = ?"
+ , SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, COL_DELETE_HOSTS,
+ COL_REPAIR_TYPE, COL_HOST_ID);
+
+ final static String SET_FORCE_REPAIR = String.format(
+ "UPDATE %s.%s SET %s=true WHERE %s = ? AND %s = ?"
+ , SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, COL_FORCE_REPAIR,
+ COL_REPAIR_TYPE, COL_HOST_ID);
+
+ final static String SELECT_LAST_REPAIR_TIME_FOR_NODE = String.format(
+ "SELECT %s FROM %s.%s WHERE %s = ? AND %s = ?", COL_REPAIR_FINISH_TS, SchemaConstants.DISTRIBUTED_KEYSPACE_NAME,
+ SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, COL_REPAIR_TYPE, COL_HOST_ID);
+
+ static ModificationStatement delStatementRepairHistory;
+ static SelectStatement selectStatementRepairHistory;
+ static ModificationStatement delStatementPriorityStatus;
+ static SelectStatement selectStatementRepairPriority;
+ static SelectStatement selectLastRepairTimeForNode;
+ static ModificationStatement addPriorityHost;
+ static ModificationStatement insertNewRepairHistoryStatement;
+ static ModificationStatement recordStartRepairHistoryStatement;
+ static ModificationStatement recordFinishRepairHistoryStatement;
+ static ModificationStatement addHostIDToDeleteHostsStatement;
+ static ModificationStatement clearDeleteHostsStatement;
+ static ModificationStatement setForceRepairStatement;
+ static ConsistencyLevel internalQueryCL;
+
+ public enum RepairTurn
+ {
+ MY_TURN,
+ NOT_MY_TURN,
+ MY_TURN_DUE_TO_PRIORITY,
+ MY_TURN_FORCE_REPAIR
+ }
+
+ public static void setup()
+ {
+ selectStatementRepairHistory = (SelectStatement) QueryProcessor.getStatement(SELECT_REPAIR_HISTORY, ClientState
+ .forInternalCalls());
+ selectStatementRepairPriority = (SelectStatement) QueryProcessor.getStatement(SELECT_REPAIR_PRIORITY, ClientState
+ .forInternalCalls());
+ selectLastRepairTimeForNode = (SelectStatement) QueryProcessor.getStatement(SELECT_LAST_REPAIR_TIME_FOR_NODE, ClientState
+ .forInternalCalls());
+ delStatementPriorityStatus = (ModificationStatement) QueryProcessor.getStatement(DEL_REPAIR_PRIORITY, ClientState
+ .forInternalCalls());
+ addPriorityHost = (ModificationStatement) QueryProcessor.getStatement(ADD_PRIORITY_HOST, ClientState
+ .forInternalCalls());
+ insertNewRepairHistoryStatement = (ModificationStatement) QueryProcessor.getStatement(INSERT_NEW_REPAIR_HISTORY, ClientState
+ .forInternalCalls());
+ recordStartRepairHistoryStatement = (ModificationStatement) QueryProcessor.getStatement(RECORD_START_REPAIR_HISTORY, ClientState
+ .forInternalCalls());
+ recordFinishRepairHistoryStatement = (ModificationStatement) QueryProcessor.getStatement(RECORD_FINISH_REPAIR_HISTORY, ClientState
+ .forInternalCalls());
+ addHostIDToDeleteHostsStatement = (ModificationStatement) QueryProcessor.getStatement(ADD_HOST_ID_TO_DELETE_HOSTS, ClientState
+ .forInternalCalls());
+ setForceRepairStatement = (ModificationStatement) QueryProcessor.getStatement(SET_FORCE_REPAIR, ClientState
+ .forInternalCalls());
+ clearDeleteHostsStatement = (ModificationStatement) QueryProcessor.getStatement(CLEAR_DELETE_HOSTS, ClientState
+ .forInternalCalls());
+ delStatementRepairHistory = (ModificationStatement) QueryProcessor.getStatement(DEL_AUTO_REPAIR_HISTORY, ClientState
+ .forInternalCalls());
+ Keyspace autoRepairKS = Schema.instance.getKeyspaceInstance(SchemaConstants.DISTRIBUTED_KEYSPACE_NAME);
+ internalQueryCL = autoRepairKS.getReplicationStrategy().getClass() == NetworkTopologyStrategy.class ?
+ ConsistencyLevel.LOCAL_QUORUM : ConsistencyLevel.ONE;
+ }
+
+ public static class AutoRepairHistory
+ {
+ UUID hostId;
+ String repairTurn;
+ long lastRepairStartTime;
+ long lastRepairFinishTime;
+ Set<UUID> deleteHosts;
+ long deleteHostsUpdateTime;
+ boolean forceRepair;
+
+ public AutoRepairHistory(UUID hostId, String repairTurn, long lastRepairStartTime, long lastRepairFinishTime,
+ Set<UUID> deleteHosts, long deleteHostsUpateTime, boolean forceRepair)
+ {
+ this.hostId = hostId;
+ this.repairTurn = repairTurn;
+ this.lastRepairStartTime = lastRepairStartTime;
+ this.lastRepairFinishTime = lastRepairFinishTime;
+ this.deleteHosts = deleteHosts;
+ if (this.deleteHosts == null)
+ {
+ this.deleteHosts = new HashSet<>();
+ }
+ this.deleteHostsUpdateTime = deleteHostsUpateTime;
+ this.forceRepair = forceRepair;
+ }
+
+ public String toString()
+ {
+ return MoreObjects.toStringHelper(this).
+ add("hostId", hostId).
+ add("repairTurn", repairTurn).
+ add("lastRepairStartTime", lastRepairStartTime).
+ add("lastRepairFinishTime", lastRepairFinishTime).
+ add("deleteHosts", deleteHosts).
+ toString();
+ }
+
+ public boolean isRepairRunning()
+ {
+ // if a repair history record has start time laster than finish time, it means the repair is running
+ return lastRepairStartTime > lastRepairFinishTime;
+ }
+
+ public long getLastRepairFinishTime()
+ {
+ return lastRepairFinishTime;
+ }
+ }
+
+ public static class CurrentRepairStatus
+ {
+ public Set<UUID> hostIdsWithOnGoingRepair; // hosts that is running repair
+ public Set<UUID> hostIdsWithOnGoingForceRepair; // hosts that is running repair because of force repair
+ Set<UUID> priority;
+ public AutoRepairHistory myRepairHistory;
+ List<AutoRepairHistory> historiesWithoutOnGoingRepair; // hosts that is NOT running repair
+
+ public CurrentRepairStatus(List<AutoRepairHistory> repairHistories, Set<UUID> priority, UUID myId)
+ {
+ hostIdsWithOnGoingRepair = new HashSet<>();
+ hostIdsWithOnGoingForceRepair = new HashSet<>();
+ historiesWithoutOnGoingRepair = new ArrayList<>();
+
+ for (AutoRepairHistory history : repairHistories)
+ {
+ if (history.isRepairRunning())
+ {
+ if (history.forceRepair)
+ {
+ hostIdsWithOnGoingForceRepair.add(history.hostId);
+ }
+ else
+ {
+ hostIdsWithOnGoingRepair.add(history.hostId);
+ }
+ }
+ else
+ {
+ historiesWithoutOnGoingRepair.add(history);
+ }
+ if (history.hostId.equals(myId))
+ {
+ myRepairHistory = history;
+ }
+ }
+ this.priority = priority;
+ }
+
+ public Set<UUID> getAllHostsWithOngoingRepair()
+ {
+ return Sets.union(hostIdsWithOnGoingRepair, hostIdsWithOnGoingForceRepair);
+ }
+
+ public String toString()
+ {
+ return MoreObjects.toStringHelper(this).
+ add("hostIdsWithOnGoingRepair", hostIdsWithOnGoingRepair).
+ add("hostIdsWithOnGoingForceRepair", hostIdsWithOnGoingForceRepair).
+ add("historiesWithoutOnGoingRepair", historiesWithoutOnGoingRepair).
+ add("priority", priority).
+ add("myRepairHistory", myRepairHistory).
+ toString();
+ }
+ }
+
+ @VisibleForTesting
+ public static List<AutoRepairHistory> getAutoRepairHistory(RepairType repairType)
+ {
+ UntypedResultSet repairHistoryResult;
+
+ ResultMessage.Rows repairStatusRows = selectStatementRepairHistory.execute(QueryState.forInternalCalls(),
+ QueryOptions.forInternalCalls(internalQueryCL, Lists.newArrayList(ByteBufferUtil.bytes(repairType.toString()))), Dispatcher.RequestTime.forImmediateExecution());
+ repairHistoryResult = UntypedResultSet.create(repairStatusRows.result);
+
+ List<AutoRepairHistory> repairHistories = new ArrayList<>();
+ if (!repairHistoryResult.isEmpty())
+ {
+ for (UntypedResultSet.Row row : repairHistoryResult)
+ {
+ UUID hostId = row.getUUID(COL_HOST_ID);
+ String repairTurn = null;
+ if (row.has(COL_REPAIR_TURN))
+ repairTurn = row.getString(COL_REPAIR_TURN);
+ long lastRepairStartTime = row.getLong(COL_REPAIR_START_TS, 0);
+ long lastRepairFinishTime = row.getLong(COL_REPAIR_FINISH_TS, 0);
+ Set<UUID> deleteHosts = row.getSet(COL_DELETE_HOSTS, UUIDType.instance);
+ long deleteHostsUpdateTime = row.getLong(COL_DELETE_HOSTS_UPDATE_TIME, 0);
+ boolean forceRepair = row.has(COL_FORCE_REPAIR) && row.getBoolean(COL_FORCE_REPAIR);
+ repairHistories.add(new AutoRepairHistory(hostId, repairTurn, lastRepairStartTime, lastRepairFinishTime,
+ deleteHosts, deleteHostsUpdateTime, forceRepair));
+ }
+ return repairHistories;
+ }
+ logger.info("No repair history found");
+ return null;
+ }
+
+ // A host may add itself in delete hosts for some other hosts due to restart or some temp gossip issue. If a node's record
+ // delete_hosts is not growing for more than 2 hours, we consider it as a normal node so we clear the delete_hosts for that node
+ public static void clearDeleteHosts(RepairType repairType, UUID hostId)
+ {
+ clearDeleteHostsStatement.execute(QueryState.forInternalCalls(),
+ QueryOptions.forInternalCalls(internalQueryCL,
+ Lists.newArrayList(ByteBufferUtil.bytes(repairType.toString()),
+ ByteBufferUtil.bytes(hostId))), Dispatcher.RequestTime.forImmediateExecution());
+ }
+
+ public static void setForceRepairNewNode(RepairType repairType)
+ {
+ // this function will be called when a node bootstrap finished
+ UUID hostId = Gossiper.instance.getHostId(FBUtilities.getBroadcastAddressAndPort());
+ // insert the data first
+ insertNewRepairHistory(repairType, currentTimeMillis(), currentTimeMillis());
+ setForceRepair(repairType, hostId);
+ }
+
+ public static void setForceRepair(RepairType repairType, Set<InetAddressAndPort> hosts)
+ {
+ // this function is used by nodetool
+ for (InetAddressAndPort host : hosts)
+ {
+ UUID hostId = Gossiper.instance.getHostId(host);
+ setForceRepair(repairType, hostId);
+ }
+ }
+
+ public static void setForceRepair(RepairType repairType, UUID hostId)
+ {
+ setForceRepairStatement.execute(QueryState.forInternalCalls(),
+ QueryOptions.forInternalCalls(internalQueryCL,
+ Lists.newArrayList(ByteBufferUtil.bytes(repairType.toString()),
+ ByteBufferUtil.bytes(hostId))),
+ Dispatcher.RequestTime.forImmediateExecution());
+
+ logger.info("Set force repair repair type: {}, node: {}", repairType, hostId);
+ }
+
+ public static long getLastRepairTimeForNode(RepairType repairType, UUID hostId)
+ {
+ ResultMessage.Rows rows = selectLastRepairTimeForNode.execute(QueryState.forInternalCalls(),
+ QueryOptions.forInternalCalls(internalQueryCL,
+ Lists.newArrayList(
+ ByteBufferUtil.bytes(repairType.toString()),
+ ByteBufferUtil.bytes(hostId))),
+ Dispatcher.RequestTime.forImmediateExecution());
+ UntypedResultSet repairTime = UntypedResultSet.create(rows.result);
+ if (repairTime.isEmpty())
+ {
+ return 0;
+ }
+ return repairTime.one().getLong(COL_REPAIR_FINISH_TS);
+ }
+
+ @VisibleForTesting
+ public static CurrentRepairStatus getCurrentRepairStatus(RepairType repairType, List<AutoRepairHistory> autoRepairHistories, UUID myId)
+ {
+ if (autoRepairHistories != null)
+ {
+ return new CurrentRepairStatus(autoRepairHistories, getPriorityHostIds(repairType), myId);
+ }
+ return null;
+ }
+
+ /**
+ * Checks whether the cluster has multiple major versions
+ * @return
+ * true if more than one major versions are detected
+ * false if only one major version is detected
+ *
+ */
+ public static boolean hasMultipleLiveMajorVersions()
+ {
+ Set<InetAddressAndPort> liveEndpoints = Gossiper.instance.getLiveMembers();
+ Set<Integer> majorVersions = new HashSet<>();
+ for (InetAddressAndPort endpoint : liveEndpoints)
+ {
+ CassandraVersion releaseVersion = Gossiper.instance.getReleaseVersion(endpoint);
+ if (releaseVersion != null)
+ {
+ majorVersions.add(releaseVersion.major);
+ }
+ }
+ return majorVersions.size() > 1;
+ }
+
+ /**
+ * Last version that does not support auto-repair.
+ * All nodes in the cluster must be running a version above this to enable auto-repair.
+ * Versions at or below this version (5.0.7) do not support auto-repair.
+ */
+ @VisibleForTesting
+ static final CassandraVersion LAST_UNSUPPORTED_VERSION_FOR_AUTO_REPAIR = new CassandraVersion("5.0.7");
+
+ /**
+ * Checks whether any node in the cluster is running an unsupported version for auto-repair.
+ *
+ * @return true if any live node has a version at or below 5.0.7 (unsupported) or has an unknown version,
+ * false if all nodes are running versions above 5.0.7 (supported)
+ */
+ public static boolean hasNodesBelowMinimumVersion()
+ {
+ if (!CassandraRelevantProperties.AUTOREPAIR_CHECK_MIN_VERSION.getBoolean())
+ return false;
+
+ Set<InetAddressAndPort> liveEndpoints = Gossiper.instance.getLiveMembers();
+ for (InetAddressAndPort endpoint : liveEndpoints)
+ {
+ CassandraVersion releaseVersion = Gossiper.instance.getReleaseVersion(endpoint);
+ if (releaseVersion == null)
+ {
+ logger.warn("Cannot determine version for endpoint {}, blocking auto-repair", endpoint);
+ return true;
+ }
+ if (releaseVersion.compareTo(LAST_UNSUPPORTED_VERSION_FOR_AUTO_REPAIR) <= 0)
+ {
+ logger.info("Endpoint {} is running version {} which does not support auto-repair " +
+ "(auto-repair requires version above {})",
+ endpoint, releaseVersion, LAST_UNSUPPORTED_VERSION_FOR_AUTO_REPAIR);
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @VisibleForTesting
+ protected static TreeSet<UUID> getHostIdsInCurrentRing(RepairType repairType, Set<InetAddressAndPort> allNodesInRing)
+ {
+ TreeSet<UUID> hostIdsInCurrentRing = new TreeSet<>();
+ for (InetAddressAndPort node : allNodesInRing)
+ {
+ String nodeDC = DatabaseDescriptor.getEndpointSnitch().getDatacenter(node);
+ if (AutoRepairService.instance.getAutoRepairConfig().getIgnoreDCs(repairType).contains(nodeDC))
+ {
+ logger.info("Ignore node {} because its datacenter is {}", node, nodeDC);
+ continue;
+ }
+ /*
+ * Check if endpoint state exists in gossip or not. If it
+ * does not then this maybe a ghost node so ignore it
+ */
+ if (Gossiper.instance.isAlive(node))
+ {
+ UUID hostId = Gossiper.instance.getHostId(node);
+ hostIdsInCurrentRing.add(hostId);
+ }
+ else
+ {
+ logger.warn("Node is not present in Gossip cache node {}, node data center {}", node, nodeDC);
+ }
+ }
+ return hostIdsInCurrentRing;
+ }
+
+ public static TreeSet<UUID> getHostIdsInCurrentRing(RepairType repairType)
+ {
+ Set<InetAddressAndPort> allNodesInRing = StorageService.instance.getTokenMetadata().getAllEndpoints();
+ return getHostIdsInCurrentRing(repairType, allNodesInRing);
+ }
+
+ // This function will return the host ID for the node which has not been repaired for longest time
+ public static AutoRepairHistory getHostWithLongestUnrepairTime(RepairType repairType)
+ {
+ List<AutoRepairHistory> autoRepairHistories = getAutoRepairHistory(repairType);
+ return getHostWithLongestUnrepairTime(autoRepairHistories);
+ }
+
+ /**
+ * Convenience method to resolve the broadcast address of a host id
+ *
+ * @return broadcast address if it exists, otherwise null.
+ */
+ @Nullable
+ private static InetAddressAndPort getBroadcastAddress(UUID hostId)
+ {
+ return StorageService.instance.getEndpointForHostId(hostId);
+ }
+
+ /**
+ * @return Map of broadcast address to host id, if a broadcast address cannot be found for a host, it is
+ * not included in the map.
+ */
+ private static Map<InetAddressAndPort, UUID> getBroadcastAddressToHostIdMap(Set<UUID> hosts)
+ {
+ // Get a mapping of endpoint : host id
+ Map<InetAddressAndPort, UUID> broadcastAddressMap = new HashMap<>(hosts.size());
+ for (UUID hostId : hosts)
+ {
+ InetAddressAndPort broadcastAddress = getBroadcastAddress(hostId);
+ if (broadcastAddress == null)
+ {
+ logger.warn("Could not resolve broadcast address from host id {} in ClusterMetadata can't accurately " +
+ "determine if this node is a replica of the local node.", hostId);
+ }
+ else
+ {
+ broadcastAddressMap.put(broadcastAddress, hostId);
+ }
+ }
+ return broadcastAddressMap;
+ }
+
+ /**
+ * @return Mapping of unique replication strategy to keyspaces using that strategy that we care about repairing.
+ */
+ private static Map<AbstractReplicationStrategy, List<String>> getReplicationStrategies()
+ {
+ // Collect all unique replication strategies among all keyspaces.
+ Map<AbstractReplicationStrategy, List<String>> replicationStrategies = new HashMap<>();
+ for (Keyspace keyspace : Keyspace.all())
+ {
+ if (AutoRepairUtils.shouldConsiderKeyspace(keyspace))
+ {
+ replicationStrategies.computeIfAbsent(keyspace.getReplicationStrategy(), k -> new ArrayList<>())
+ .add(keyspace.getName());
+ }
+ }
+ return replicationStrategies;
+ }
+
+ /**
+ * Collects all hosts being repaired among all active repair schedules and their schedule if
+ * {@link AutoRepairConfig#getAllowParallelReplicaRepairAcrossSchedules(RepairType)} is true for this repairType.
+ * Accepts the currently evaluated repairType's schedule as an optimization to avoid grabbing its repair status an
+ * additional time.
+ *
+ * @param myRepairType The repair type schedule being evaluated.
+ * @param myRepairStatus The repair status for that repair type.
+ * @return All hosts among active schedules currently being repaired.
+ */
+ private static Map<UUID, RepairType> getHostsBeingRepaired(RepairType myRepairType, CurrentRepairStatus myRepairStatus)
+ {
+ AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig();
+
+ Map<UUID, RepairType> hostsBeingRepaired = myRepairStatus.getAllHostsWithOngoingRepair().stream()
+ .collect(Collectors.toMap((h) -> h, (v) -> myRepairType));
+
+ // If we don't allow repairing across schedules, iterate over other enabled schedules and include hosts
+ // actively being repaired.
+ if (!config.getAllowParallelReplicaRepairAcrossSchedules(myRepairType))
+ {
+ for (RepairType repairType : RepairType.values())
+ {
+ if (myRepairType == repairType)
+ continue;
+
+ if (config.isAutoRepairEnabled(repairType))
+ {
+ CurrentRepairStatus repairStatus = getCurrentRepairStatus(repairType, getAutoRepairHistory(repairType), null);
+ if (repairStatus != null)
+ {
+ for (UUID hostId : repairStatus.getAllHostsWithOngoingRepair())
+ {
+ hostsBeingRepaired.putIfAbsent(hostId, repairType);
+ }
+ }
+ }
+ }
+ }
+ return hostsBeingRepaired;
+ }
+
+ /**
+ * Identifies the most eligible host to repair for nodes preceding or equal to this nodes' lastRepairFinishTime.
+ * The criteria for this is to find the node with the oldest last repair finish time of which none of its replicas
+ * are currently under repair.
+ *
+ * @return The most eligible host to repair or null if no candidates before and including this nodes' current repair status.
+ */
+ @VisibleForTesting
+ public static AutoRepairHistory getMostEligibleHostToRepair(RepairType repairType, CurrentRepairStatus currentRepairStatus, UUID myId)
+ {
+ // 0. If this repairType allows parallel replica repair, short circuit and return the host with the longest unrepair time
+ AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig();
+ if (config.getAllowParallelReplicaRepair(repairType))
+ {
+ return getHostWithLongestUnrepairTime(currentRepairStatus.historiesWithoutOnGoingRepair);
+ }
+
+ // 1. Sort repair histories from oldest completed to newest
+ Stream<AutoRepairHistory> finishedRepairHistories = currentRepairStatus.historiesWithoutOnGoingRepair
+ .stream()
+ .sorted(Comparator.comparingLong(h -> h.lastRepairFinishTime));
+
+ // 2. Optimization: Truncate repair histories after myId so we don't evaluate anything more recent as if we
+ // aren't interested in anything that isn't this node.
+ final AtomicBoolean myHistoryFound = new AtomicBoolean(false);
+ finishedRepairHistories = finishedRepairHistories
+ .filter(history -> {
+ if (myHistoryFound.get()) return false; // Stop processing after finding myId
+ myHistoryFound.set(history.hostId.equals(myId));
+ return true;
+ });
+
+ // If there are any hosts with ongoing repair, filter the repair histories to not include nodes whose replicas
+ // are ongoing repair.
+ Map<UUID, RepairType> hostsBeingRepairedToRepairType = getHostsBeingRepaired(repairType, currentRepairStatus);
+
+ // 3. If I am already actively being repaired in another schedule, defer submitting repairs; if already
+ // repairing for this type, return node so it can take its turn.
+ RepairType alreadyRepairingType = hostsBeingRepairedToRepairType.get(myId);
+ if (alreadyRepairingType != null)
+ {
+ if (repairType != alreadyRepairingType)
+ {
+ logger.info("Deferring repair because I am already actively repairing in schedule {}", hostsBeingRepairedToRepairType.get(myId));
+ AutoRepairMetricsManager.getMetrics(repairType).repairDelayedBySchedule.inc();
+ return null;
+ }
+ else if (currentRepairStatus.myRepairHistory != null)
+ {
+ // if the repair type matches this repair, assume the node was restarted while repairing, return node
+ // so it can take its turn.
+ logAlreadyMyTurn();
+ return currentRepairStatus.myRepairHistory;
+ }
+ }
+
+ if (!hostsBeingRepairedToRepairType.isEmpty())
+ {
+ // 4. Extract InetAddresses for each UUID as replicas are identified by their address.
+ Map<InetAddressAndPort, UUID> hostsBeingRepaired = getBroadcastAddressToHostIdMap(hostsBeingRepairedToRepairType.keySet());
+
+ // 5. Collect unique replication strategies and group them up with their keyspaces.
+ Map<AbstractReplicationStrategy, List<String>> replicationStrategies = getReplicationStrategies();
+
+ // 6. Filter out repair histories who have a replica being repaired, note that this is lazy, given the stream
+ // is completed using findFirst, it should stop as soon as the matching criteria is met.
+ finishedRepairHistories = finishedRepairHistories.filter((history) -> !hasReplicaWithOngoingRepair(history,
+ myId,
+ repairType,
+ hostsBeingRepaired,
+ hostsBeingRepairedToRepairType,
+ replicationStrategies));
+ }
+
+ // 7. Select the first (oldest lastRepairFinishTime) repair history without replicas being repaired
+ return finishedRepairHistories.findFirst().orElse(null);
+ }
+
+
+ /**
+ * @param eligibleHistory History of node to check
+ * @param myId Host id of this node, if the repair history is for this node, additional logging will take place.
+ * @param myRepairType repair type being evaluated
+ * @param hostsBeingRepaired Hosts being repaired.
+ * @param hostIdToRepairType mapping of hosts being repaired to the repair type its being repaired for.
+ * @param replicationStrategies Mapping of unique replication strategies to keyspaces having that strategy.
+ * @return Whether the host for the given eligibleRepairHistory has any replicas in hostsBeingRepaired.
+ */
+ private static boolean hasReplicaWithOngoingRepair(AutoRepairHistory eligibleHistory,
+ UUID myId,
+ RepairType myRepairType,
+ Map<InetAddressAndPort, UUID> hostsBeingRepaired,
+ Map<UUID, RepairType> hostIdToRepairType,
+ Map<AbstractReplicationStrategy, List<String>> replicationStrategies)
+ {
+ // If no broadcast address found for this host id in cluster metadata, just skip it, a node should always
+ // see itself in cluster metadata.
+ InetAddressAndPort eligibleBroadcastAddress = getBroadcastAddress(eligibleHistory.hostId);
+ if (eligibleBroadcastAddress == null)
+ {
+ return true;
+ }
+
+ // For each replication strategy, determine if host being repaired is a replica of the local node.
+ for (Map.Entry<AbstractReplicationStrategy, List<String>> entry : replicationStrategies.entrySet())
+ {
+ AbstractReplicationStrategy replicationStrategy = entry.getKey();
+ EndpointsByRange endpointsByRange = replicationStrategy.getRangeAddresses(StorageService.instance.getTokenMetadata());
+
+ // get ranges of the eligible address for the given replication strategy.
+ RangesAtEndpoint rangesAtEndpoint = StorageService.instance.getReplicas(replicationStrategy, eligibleBroadcastAddress);
+ for (Replica replica : rangesAtEndpoint)
+ {
+ // get the endpoints involved in this range.
+ EndpointsForRange endpointsForRange = endpointsByRange.get(replica.range());
+ // For each host in this range...
+ for (InetAddressAndPort inetAddressAndPort : endpointsForRange.endpoints())
+ {
+ // If the address of the node in the range belongs to a host being repaired, return true.
+ UUID hostId = hostsBeingRepaired.get(inetAddressAndPort);
+ if (hostId != null)
+ {
+ // log if the repair history matches the current running node.
+ InetAddressAndPort myBroadcastAddress = getBroadcastAddress(myId);
+ if (myBroadcastAddress != null && myBroadcastAddress.equals(eligibleBroadcastAddress))
+ {
+ logger.info("Deferring repair because replica {} ({}) with shared ranges for " +
+ "{} keyspace(s) (e.g. {}) is currently taking its turn for schedule {}",
+ hostId, inetAddressAndPort, entry.getValue().size(), entry.getValue().get(0),
+ hostIdToRepairType.get(hostId));
+ AutoRepairMetricsManager.getMetrics(myRepairType).repairDelayedByReplica.inc();
+ }
+ else if (logger.isDebugEnabled())
+ {
+ logger.debug("Not considering node {} ({}) for repair as it has replica {} ({}) with " +
+ "shared ranges for {} keyspace(s) (e.g. {}) which is currently taking its " +
+ "turn for schedule {}",
+ eligibleHistory.hostId, eligibleBroadcastAddress,
+ hostId, inetAddressAndPort, entry.getValue().size(), entry.getValue().get(0),
+ hostIdToRepairType.get(hostId));
+ }
+ return true;
+ }
+ }
+ }
+ }
+
+ // No replicas found of eligible host.
+ return false;
+ }
+
+ private static AutoRepairHistory getHostWithLongestUnrepairTime(List<AutoRepairHistory> autoRepairHistories)
+ {
+ if (autoRepairHistories == null)
+ {
+ return null;
+ }
+ AutoRepairHistory rst = null;
+ long oldestTimestamp = Long.MAX_VALUE;
+ for (AutoRepairHistory autoRepairHistory : autoRepairHistories)
+ {
+ if (autoRepairHistory.lastRepairFinishTime < oldestTimestamp)
+ {
+ rst = autoRepairHistory;
+ oldestTimestamp = autoRepairHistory.lastRepairFinishTime;
+ }
+ }
+ return rst;
+ }
+
+ public static int getMaxNumberOfNodeRunAutoRepair(RepairType repairType, int groupSize)
+ {
+ AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig();
+ if (groupSize == 0)
+ {
+ return Math.max(config.getParallelRepairCount(repairType), 1);
+ }
+ // we will use the max number from config between auto_repair_parallel_repair_count_in_group and auto_repair_parallel_repair_percentage_in_group
+ int value = Math.max(groupSize * config.getParallelRepairPercentage(repairType) / 100,
+ config.getParallelRepairCount(repairType));
+ // make sure at least one node getting repaired
+ return Math.max(1, value);
+ }
+
+ private static void logAlreadyMyTurn()
+ {
+ logger.warn("This node already was considered to having an ongoing repair for this repair type, must have " +
+ "been restarted, taking my turn back");
+ }
+
+ @VisibleForTesting
+ public static RepairTurn myTurnToRunRepair(RepairType repairType, UUID myId)
+ {
+ try
+ {
+ Set<InetAddressAndPort> allNodesInRing = StorageService.instance.getTokenMetadata().getAllEndpoints();
+ logger.info("Total nodes in ring {}", allNodesInRing.size());
+ TreeSet<UUID> hostIdsInCurrentRing = getHostIdsInCurrentRing(repairType, allNodesInRing);
+ logger.info("Total nodes qualified for repair {}", hostIdsInCurrentRing.size());
+
+ List<AutoRepairHistory> autoRepairHistories = getAutoRepairHistory(repairType);
+ Set<UUID> autoRepairHistoryIds = new HashSet<>();
+
+ // 1. Remove any node that is not part of group based on gossip info
+ if (autoRepairHistories != null)
+ {
+ for (AutoRepairHistory nodeHistory : autoRepairHistories)
+ {
+ autoRepairHistoryIds.add(nodeHistory.hostId);
+ // clear delete_hosts if the node's delete hosts is not growing for more than two hours
+ AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig();
+ if (!nodeHistory.deleteHosts.isEmpty()
+ && config.getAutoRepairHistoryClearDeleteHostsBufferInterval().toSeconds() < TimeUnit.MILLISECONDS.toSeconds(
+ currentTimeMillis() - nodeHistory.deleteHostsUpdateTime
+ ))
+ {
+ clearDeleteHosts(repairType, nodeHistory.hostId);
+ logger.info("Delete hosts for {} for repair type {} has not been updated for more than {} seconds. Delete hosts has been cleared. Delete hosts before clear {}"
+ , nodeHistory.hostId, repairType, config.getAutoRepairHistoryClearDeleteHostsBufferInterval(), nodeHistory.deleteHosts);
+ }
+ else if (!hostIdsInCurrentRing.contains(nodeHistory.hostId))
+ {
+ if (nodeHistory.deleteHosts.size() > Math.max(2, hostIdsInCurrentRing.size() * 0.5))
+ {
+ // More than half of the groups thinks the record should be deleted
+ logger.info("{} think {} is orphan node, will delete auto repair history for repair type {}.", nodeHistory.deleteHosts, nodeHistory.hostId, repairType);
+ deleteAutoRepairHistory(repairType, nodeHistory.hostId);
+ }
+ else
+ {
+ // I think this host should be deleted
+ logger.info("I({}) think {} is not part of ring, vote to delete it for repair type {}.", myId, nodeHistory.hostId, repairType);
+ addHostIdToDeleteHosts(repairType, myId, nodeHistory.hostId);
+ }
+ }
+ }
+ }
+
+ // 2. Add node to auto repair history table if a node is in gossip info
+ for (UUID hostId : hostIdsInCurrentRing)
+ {
+ if (!autoRepairHistoryIds.contains(hostId))
+ {
+ logger.info("{} for repair type {} doesn't exist in the auto repair history table, insert a new record.", repairType, hostId);
+ insertNewRepairHistory(repairType, hostId, currentTimeMillis(), currentTimeMillis());
+ }
+ }
+
+ // refresh auto repair histories
+ autoRepairHistories = getAutoRepairHistory(repairType);
+ if (autoRepairHistories == null)
+ {
+ logger.error("No record found");
+ return NOT_MY_TURN;
+ }
+
+ // get updated current repair status
+ CurrentRepairStatus currentRepairStatus = getCurrentRepairStatus(repairType, autoRepairHistories, myId);
+ if (logger.isDebugEnabled())
+ {
+ logger.debug("Latest repair status {}", currentRepairStatus);
+ }
+ //check if I am forced to run repair
+ for (AutoRepairHistory history : currentRepairStatus.historiesWithoutOnGoingRepair)
+ {
+ if (history.forceRepair && history.hostId.equals(myId))
+ {
+ return MY_TURN_FORCE_REPAIR;
+ }
+ }
+
+ // check if node was already indicated as having an ongoing repair, this may happen when a node restarts
+ // before finishing repairing.
+ if (currentRepairStatus.getAllHostsWithOngoingRepair().contains(myId))
+ {
+ logAlreadyMyTurn();
+
+ // use the previously chosen turn.
+ if (currentRepairStatus.myRepairHistory != null && currentRepairStatus.myRepairHistory.repairTurn != null)
+ {
+ return RepairTurn.valueOf(currentRepairStatus.myRepairHistory.repairTurn);
+ }
+ else
+ {
+ return MY_TURN;
+ }
+ }
+
+ int parallelRepairNumber = getMaxNumberOfNodeRunAutoRepair(repairType, autoRepairHistories.size());
+ logger.info("Will run repairs concurrently on {} node(s)", parallelRepairNumber);
+ if (parallelRepairNumber > currentRepairStatus.hostIdsWithOnGoingRepair.size())
+ {
+ UUID priorityHostId = null;
+ if (currentRepairStatus.priority != null)
+ {
+ for (UUID priorityID : currentRepairStatus.priority)
+ {
+ // remove ids doesn't belong to this ring
+ if (!hostIdsInCurrentRing.contains(priorityID))
+ {
+ logger.info("{} is not part of the current ring, will be removed from priority list.", priorityID);
+ removePriorityStatus(repairType, priorityID);
+ }
+ else
+ {
+ priorityHostId = priorityID;
+ break;
+ }
+ }
+ }
+
+ if (priorityHostId != null && !myId.equals(priorityHostId))
+ {
+ logger.info("Priority list is not empty and I'm not the first node in the list, not my turn." +
+ "First node in priority list is {}", getBroadcastAddress(priorityHostId));
+ return NOT_MY_TURN;
+ }
+
+ if (myId.equals(priorityHostId))
+ {
+ //I have a priority for repair hence its my turn now
+ return MY_TURN_DUE_TO_PRIORITY;
+ }
+
+ // Determine if this node is the most eligible host to repair.
+ AutoRepairHistory nodeToBeRepaired = getMostEligibleHostToRepair(repairType, currentRepairStatus, myId);
+ if (nodeToBeRepaired != null)
+ {
+ if (nodeToBeRepaired.hostId.equals(myId))
+ {
+ logger.info("This node is selected to be repaired for repair type {}", repairType);
+ return MY_TURN;
+ }
+
+ // log which node is next, which is helpful for debugging
+ logger.info("Next node to be repaired for repair type {}: {} ({})", repairType,
+ getBroadcastAddress(nodeToBeRepaired.hostId),
+ nodeToBeRepaired);
+ }
+
+ // If this node is not identified as most eligible, set the repair lag time.
+ if (currentRepairStatus.myRepairHistory != null)
+ {
+ AutoRepairMetricsManager.getMetrics(repairType)
+ .recordRepairStartLag(currentRepairStatus.myRepairHistory.lastRepairFinishTime);
+ }
+ }
+ else if (currentRepairStatus.hostIdsWithOnGoingForceRepair.contains(myId))
+ {
+ return MY_TURN_FORCE_REPAIR;
+ }
+ // for some reason I was not done with the repair hence resume (maybe node restart in-between, etc.)
+ return currentRepairStatus.hostIdsWithOnGoingRepair.contains(myId) ? MY_TURN : NOT_MY_TURN;
+ }
+ catch (Exception e)
+ {
+ logger.error("Exception while deciding node's turn:", e);
+ }
+ return NOT_MY_TURN;
+ }
+
+ static void deleteAutoRepairHistory(RepairType repairType, UUID hostId)
+ {
+ //delete the given hostId
+ delStatementRepairHistory.execute(QueryState.forInternalCalls(),
+ QueryOptions.forInternalCalls(internalQueryCL,
+ Lists.newArrayList(ByteBufferUtil.bytes(repairType.toString()),
+ ByteBufferUtil.bytes(hostId))), Dispatcher.RequestTime.forImmediateExecution());
+ }
+
+ static void updateStartAutoRepairHistory(RepairType repairType, UUID myId, long timestamp, RepairTurn turn)
+ {
+ recordStartRepairHistoryStatement.execute(QueryState.forInternalCalls(),
+ QueryOptions.forInternalCalls(internalQueryCL,
+ Lists.newArrayList(ByteBufferUtil.bytes(timestamp),
+ ByteBufferUtil.bytes(turn.name()),
+ ByteBufferUtil.bytes(repairType.toString()),
+ ByteBufferUtil.bytes(myId)
+ )), Dispatcher.RequestTime.forImmediateExecution());
+ }
+
+ static void updateFinishAutoRepairHistory(RepairType repairType, UUID myId, long timestamp)
+ {
+ recordFinishRepairHistoryStatement.execute(QueryState.forInternalCalls(),
+ QueryOptions.forInternalCalls(internalQueryCL,
+ Lists.newArrayList(ByteBufferUtil.bytes(timestamp),
+ ByteBufferUtil.bytes(repairType.toString()),
+ ByteBufferUtil.bytes(myId)
+ )), Dispatcher.RequestTime.forImmediateExecution());
+ logger.info("Auto repair finished for {}", myId);
+ }
+
+ public static void insertNewRepairHistory(RepairType repairType, UUID hostId, long startTime, long finishTime)
+ {
+ try
+ {
+ Keyspace autoRepairKS = Schema.instance.getKeyspaceInstance(SchemaConstants.DISTRIBUTED_KEYSPACE_NAME);
+ ConsistencyLevel cl = autoRepairKS.getReplicationStrategy().getClass() == NetworkTopologyStrategy.class ?
+ ConsistencyLevel.LOCAL_SERIAL : null;
+
+ UntypedResultSet resultSet;
+ ResultMessage.Rows resultMessage = (ResultMessage.Rows) insertNewRepairHistoryStatement.execute(
+ QueryState.forInternalCalls(), QueryOptions.create(internalQueryCL, Lists.newArrayList(
+ ByteBufferUtil.bytes(repairType.toString()),
+ ByteBufferUtil.bytes(hostId),
+ ByteBufferUtil.bytes(startTime),
+ ByteBufferUtil.bytes(finishTime),
+ ByteBufferUtil.bytes(currentTimeMillis())
+ ), false, -1, null, cl, ProtocolVersion.CURRENT, SchemaConstants.DISTRIBUTED_KEYSPACE_NAME),
+ Dispatcher.RequestTime.forImmediateExecution());
+ resultSet = UntypedResultSet.create(resultMessage.result);
+ boolean applied = resultSet.one().getBoolean(ModificationStatement.CAS_RESULT_COLUMN.toString());
+ if (applied)
+ {
+ logger.info("Successfully inserted a new auto repair history record for host id: {}", hostId);
+ }
+ else
+ {
+ logger.info("Record exists, no need to insert again for host id: {}", hostId);
+ }
+ }
+ catch (Exception e)
+ {
+ logger.error("Exception in inserting new repair history:", e);
+ }
+ }
+
+ public static void insertNewRepairHistory(RepairType repairType, long startTime, long finishTime)
+ {
+ UUID hostId = Gossiper.instance.getHostId(FBUtilities.getBroadcastAddressAndPort());
+ insertNewRepairHistory(repairType, hostId, startTime, finishTime);
+ }
+
+ public static void addHostIdToDeleteHosts(RepairType repairType, UUID myID, UUID hostToBeDeleted)
+ {
+ SetSerializer<UUID> serializer = SetSerializer.getInstance(UUIDSerializer.instance, UTF8Type.instance.comparatorSet);
+ addHostIDToDeleteHostsStatement.execute(QueryState.forInternalCalls(),
+ QueryOptions.forInternalCalls(internalQueryCL,
+ Lists.newArrayList(serializer.serialize(new HashSet<>(Arrays.asList(myID))),
+ ByteBufferUtil.bytes(currentTimeMillis()),
+ ByteBufferUtil.bytes(repairType.toString()),
+ ByteBufferUtil.bytes(hostToBeDeleted)
+ )), Dispatcher.RequestTime.forImmediateExecution());
+ }
+
+ public static void addPriorityHosts(RepairType repairType, Set<InetAddressAndPort> hosts)
+ {
+ Set<UUID> hostIds = new HashSet<>();
+ for (InetAddressAndPort host : hosts)
+ {
+ //find hostId from IP address
+ UUID hostId = StorageService.instance.getTokenMetadata().getHostId(host);
+ hostIds.add(hostId);
+ if (hostId != null)
+ {
+ logger.info("Add host {} to the priority list", hostId);
+ }
+ }
+ if (!hostIds.isEmpty())
+ {
+ SetSerializer<UUID> serializer = SetSerializer.getInstance(UUIDSerializer.instance, UTF8Type.instance.comparatorSet);
+ addPriorityHost.execute(QueryState.forInternalCalls(),
+ QueryOptions.forInternalCalls(internalQueryCL,
+ Lists.newArrayList(serializer.serialize(hostIds),
+ ByteBufferUtil.bytes(repairType.toString()))),
+ Dispatcher.RequestTime.forImmediateExecution());
+ }
+ }
+
+ static void removePriorityStatus(RepairType repairType, UUID hostId)
+ {
+ logger.info("Remove host {} from priority list", hostId);
+ delStatementPriorityStatus.execute(QueryState.forInternalCalls(),
+ QueryOptions.forInternalCalls(internalQueryCL,
+ Lists.newArrayList(ByteBufferUtil.bytes(hostId),
+ ByteBufferUtil.bytes(repairType.toString()))),
+ Dispatcher.RequestTime.forImmediateExecution());
+ }
+
+ public static Set<UUID> getPriorityHostIds(RepairType repairType)
+ {
+ UntypedResultSet repairPriorityResult;
+
+ ResultMessage.Rows repairPriorityRows = selectStatementRepairPriority.execute(QueryState.forInternalCalls(),
+ QueryOptions.forInternalCalls(internalQueryCL, Lists.newArrayList(ByteBufferUtil.bytes(repairType.toString()))), Dispatcher.RequestTime.forImmediateExecution());
+ repairPriorityResult = UntypedResultSet.create(repairPriorityRows.result);
+
+ Set<UUID> priorities = null;
+ if (!repairPriorityResult.isEmpty())
+ {
+ // there should be only one row
+ UntypedResultSet.Row row = repairPriorityResult.one();
+ priorities = row.getSet(COL_REPAIR_PRIORITY, UUIDType.instance);
+ }
+ if (priorities != null)
+ {
+ return priorities;
+ }
+ return Collections.emptySet();
+ }
+
+ public static Set<InetAddressAndPort> getPriorityHosts(RepairType repairType)
+ {
+ Set<InetAddressAndPort> hosts = new HashSet<>();
+ for (UUID hostId : getPriorityHostIds(repairType))
+ {
+ InetAddressAndPort broadcastAddress = getBroadcastAddress(hostId);
+ if (broadcastAddress == null)
+ {
+ logger.warn("Could not resolve broadcastAddress for {}, skipping considering it as a priority host", hostId);
+ continue;
+ }
+ hosts.add(broadcastAddress);
+ }
+ return hosts;
+ }
+
+ public static boolean shouldConsiderKeyspace(Keyspace ks)
+ {
+ AbstractReplicationStrategy replicationStrategy = ks.getReplicationStrategy();
+ boolean repair = true;
+ if (replicationStrategy instanceof NetworkTopologyStrategy)
+ {
+ Set<String> datacenters = ((NetworkTopologyStrategy) replicationStrategy).getDatacenters();
+ String localDC = DatabaseDescriptor.getEndpointSnitch().getDatacenter(FBUtilities.getBroadcastAddressAndPort());
+ if (!datacenters.contains(localDC))
+ {
+ repair = false;
+ }
+ }
+ if (replicationStrategy instanceof LocalStrategy)
+ {
+ repair = false;
+ }
+ if (ks.getName().equalsIgnoreCase(SchemaConstants.TRACE_KEYSPACE_NAME))
+ {
+ // by default, ignore the tables under system_traces as they do not have
+ // that much important data
+ repair = false;
+ }
+ return repair;
+ }
+
+ public static boolean tableMaxRepairTimeExceeded(RepairType repairType, long startTime)
+ {
+ long tableRepairTimeSoFar = TimeUnit.MILLISECONDS.toSeconds
+ (currentTimeMillis() - startTime);
+ return AutoRepairService.instance.getAutoRepairConfig().getAutoRepairTableMaxRepairTime(repairType).toSeconds() <
+ tableRepairTimeSoFar;
+ }
+
+ public static boolean keyspaceMaxRepairTimeExceeded(RepairType repairType, long startTime, int numOfTablesToBeRepaired)
+ {
+ long keyspaceRepairTimeSoFar = TimeUnit.MILLISECONDS.toSeconds((currentTimeMillis() - startTime));
+ return (long) AutoRepairService.instance.getAutoRepairConfig().getAutoRepairTableMaxRepairTime(repairType).toSeconds() *
+ numOfTablesToBeRepaired < keyspaceRepairTimeSoFar;
+ }
+
+ public static List<String> getAllMVs(RepairType repairType, Keyspace keyspace, TableMetadata tableMetadata)
+ {
+ List<String> allMvs = new ArrayList<>();
+ if (AutoRepairService.instance.getAutoRepairConfig().getMaterializedViewRepairEnabled(repairType) && keyspace.getMetadata().views != null)
+ {
+ Iterator<ViewMetadata> views = keyspace.getMetadata().views.forTable(tableMetadata.id).iterator();
+ while (views.hasNext())
+ {
+ String viewName = views.next().name();
+ logger.info("Adding MV to the list {}.{}.{}", keyspace.getName(), tableMetadata.name, viewName);
+ allMvs.add(viewName);
+ }
+ }
+ return allMvs;
+ }
+
+ public static Collection<Range<Token>> split(Range<Token> tokenRange, int numberOfSplits)
+ {
+ Collection<Range<Token>> ranges;
+ Optional<Splitter> splitter = DatabaseDescriptor.getPartitioner().splitter();
+ if (!splitter.isPresent())
+ {
+ NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 30, TimeUnit.MINUTES, "Partitioner {} does not support splitting, falling back to splitting by token range", DatabaseDescriptor.getPartitioner());
+ ranges = Collections.singleton(tokenRange);
+ }
+ else
+ {
+ ranges = splitter.get().split(Collections.singleton(tokenRange), numberOfSplits);
+ }
+ return ranges;
+ }
+
+ /**
+ * Finds a list of SSTables for a given {@code repairType},
+ * {@code keyspace}, {@code table}, and {@code tokenRange} and then it internally calls
+ * another API {@code AutoRepairUtils.getSizesForRangeOfSSTables}, which figures out the estimated data size.
+ *
+ * @param repairType the repair type (e.g., FULL, INCREMENTAL)
+ * @param keyspace the keyspace name
+ * @param table the table name
+ * @param tokenRange the token range to evaluate
+ * @return an estimate representing the number of partitions, size in range, and total size
+ */
+ static SizeEstimate getRangeSizeEstimate(RepairType repairType, String keyspace, String table, Range<Token> tokenRange)
+ {
+ logger.debug("Calculating size estimate for {}.{} for range {}", keyspace, table, tokenRange);
+ try (Refs<SSTableReader> refs = RepairTokenRangeSplitter.getSSTableReaderRefs(repairType, keyspace, table, tokenRange))
+ {
+ SizeEstimate estimate = getSizesForRangeOfSSTables(repairType, keyspace, table, tokenRange, refs);
+ logger.debug("Generated size estimate {}", estimate);
+ return estimate;
+ }
+ }
+ /**
+ * Calculates the size estimation qualified to be repaired for a given {@code repairType},
+ * {@code keyspace}, {@code table}, {@code tokenRange}, and {@code refs}.
+ * <p>
+ * If the compression is enabled, then the size will be an estimate, otherwise it will be accurate.
+ * </p>
+ *
+ * @param repairType
+ * @param keyspace
+ * @param table
+ * @param tokenRange
+ * @param refs
+ * @return an estimate representing the number of partitions, size in range, and total size
+ */
+ static SizeEstimate getSizesForRangeOfSSTables(RepairType repairType, String keyspace, String table,
+ Range<Token> tokenRange, Refs<SSTableReader> refs)
+ {
+ List<Range<Token>> singletonRange = Collections.singletonList(tokenRange);
+ ICardinality cardinality = new HyperLogLogPlus(13, 25);
+ long approxBytesInRange = 0L;
+ long totalBytes = 0L;
+
+ for (SSTableReader reader : refs)
+ {
+ try
+ {
+ if (reader.openReason == SSTableReader.OpenReason.EARLY)
+ continue;
+ CompactionMetadata metadata = (CompactionMetadata) reader.descriptor.getMetadataSerializer().deserialize(reader.descriptor, MetadataType.COMPACTION);
+ if (metadata != null)
+ cardinality = cardinality.merge(metadata.cardinalityEstimator);
+
+ // use onDiskLength, which is the actual size of the SSTable data file.
+ long sstableSize = reader.onDiskLength();
+ totalBytes += sstableSize;
+
+ // get the on disk size for the token range, note for compressed data this includes the full
+ // chunks the start and end ranges are found in.
+ long approximateRangeBytesInSSTable = reader.onDiskSizeForPartitionPositions(reader.getPositionsForRanges(singletonRange));
+ approxBytesInRange += Math.min(approximateRangeBytesInSSTable, sstableSize);
+ }
+ catch (IOException | CardinalityMergeException e)
+ {
+ logger.error("Error calculating size estimate for {}.{} for range {} on {}", keyspace, table, tokenRange, reader, e);
+ }
+ }
+
+ long partitions = 0L;
+ if (totalBytes > 0)
+ {
+ // use the ratio from size to estimate the partitions in the range as well
+ double ratio = approxBytesInRange / (double) totalBytes;
+ partitions = (long) Math.max(1, Math.ceil(cardinality.cardinality() * ratio));
+ }
+ return new SizeEstimate(repairType, keyspace, table, tokenRange, partitions, approxBytesInRange, totalBytes);
+ }
+
+ /**
+ * Calculates the token ranges owned by this node for a given keyspace.
+ *
+ * @param primaryRangeOnly whether to use only primary token ranges or include replicated ones
+ * @param keyspaceName the name of the keyspace
+ * @return one or more token ranges owned by this node
+ */
+ public static List<Range<Token>> getTokenRanges(boolean primaryRangeOnly, String keyspaceName)
+ {
+ // Collect all applicable token ranges
+ Collection<Range<Token>> wrappedRanges;
+ if (primaryRangeOnly)
+ {
+ wrappedRanges = StorageService.instance.getPrimaryRanges(keyspaceName);
+ }
+ else
+ {
+ wrappedRanges = StorageService.instance.getLocalRanges(keyspaceName);
+ }
+
+ // Unwrap each range as we need to account for ranges that overlap the ring
+ List<Range<Token>> ranges = new ArrayList<>();
+ for (Range<Token> wrappedRange : wrappedRanges)
+ {
+ ranges.addAll(wrappedRange.unwrap());
+ }
+ return ranges;
+ }
+
+ /**
+ * Calculates the total bytes to be repaired for a given keyspace and list of tables.
+ *
+ * @param repairType the repair type (e.g., FULL, INCREMENTAL)
+ * @param keyspaceName the name of the keyspace
+ * @param tableNames the list of tables
+ * @return a key-value map where the key is {@code keyspaceName.tableName} and the value is the number of bytes
+ * to be repaired.
+ */
+ public static Map<String, Map<Range<Token>, SizeEstimate>> calcTotalBytesToBeRepaired(RepairType repairType, String keyspaceName, List<String> tableNames, List<Range<Token>> tokenRanges)
+ {
+ Map<String, Map<Range<Token>, SizeEstimate>> ksTablesEstimatedBytes = new HashMap<>();
+ for (String tableName : tableNames)
+ {
+ String ksTable = getKeyspaceTableName(keyspaceName, tableName);
+ ksTablesEstimatedBytes.computeIfAbsent(ksTable, k -> new HashMap<>());
+ Map<Range<Token>, SizeEstimate> tokenToSize = ksTablesEstimatedBytes.get(ksTable);
+ for (Range<Token> tokenRange : tokenRanges)
+ {
+ SizeEstimate tableAssignments = getRangeSizeEstimate(repairType, keyspaceName, tableName, tokenRange);
+ tokenToSize.put(tokenRange, tableAssignments);
+ }
+ }
+ return ksTablesEstimatedBytes;
+ }
+
+ public static String getKeyspaceTableName(String keyspace, String table)
+ {
+ return keyspace + "." + table;
+ }
+
+ /**
+ * Represents a size estimate by both bytes and partition count for a given keyspace and table for a token range.
+ */
+ @VisibleForTesting
+ protected static class SizeEstimate
+ {
+ public final RepairType repairType;
+ public final String keyspace;
+ public final String table;
+ public final Range<Token> tokenRange;
+ public final long partitions;
+ public final long sizeInRange;
+ public final long totalSize;
+ /**
+ * Size to consider in the repair. For incremental repair, we want to consider the total size
+ * of the estimate as we have to factor in anticompacting the entire SSTable.
+ * For full repair, just use the size containing the range.
+ */
+ public final long sizeForRepair;
+
+ public SizeEstimate(RepairType repairType,
+ String keyspace, String table, Range<Token> tokenRange,
+ long partitions, long sizeInRange, long totalSize)
+ {
+ this.repairType = repairType;
+ this.keyspace = keyspace;
+ this.table = table;
+ this.tokenRange = tokenRange;
+ this.partitions = partitions;
+ this.sizeInRange = sizeInRange;
+ this.totalSize = totalSize;
+
+ this.sizeForRepair = repairType == RepairType.INCREMENTAL ? totalSize : sizeInRange;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "SizeEstimate{" +
+ "repairType=" + repairType +
+ ", keyspace='" + keyspace + '\'' +
+ ", table='" + table + '\'' +
+ ", tokenRange=" + tokenRange +
+ ", partitions=" + partitions +
+ ", sizeInRange=" + sizeInRange +
+ ", totalSize=" + totalSize +
+ ", sizeForRepair=" + sizeForRepair +
+ '}';
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java
new file mode 100644
index 0000000000..1b2c80b742
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/autorepair/FixedSplitTokenRangeSplitter.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.repair.autorepair;
+
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.service.AutoRepairService;
+
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+
+import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.split;
+
+/**
+ * An implementation that splits token ranges into a fixed number of subranges.
+ */
+public class FixedSplitTokenRangeSplitter implements IAutoRepairTokenRangeSplitter
+{
+ private static final Logger logger = LoggerFactory.getLogger(FixedSplitTokenRangeSplitter.class);
+
+ /**
+ * Selecting the default value is tricky. If we select a small number, individual repairs would be heavy.
+ * On the other hand, if we select a large number, too many repair sessions would be created.
+ * <p>
+ * If vnodes are configured using <code>num_tokens</code>, attempts to evenly subdivide subranges by each range
+ * using the following formula:
+ * <p>
+ * Math.max(1, numberOfSubranges / tokens.size())
+ * <p>
+ * To maintain balance, 32 serves as a good default that accommodates both vnodes and non-vnodes effectively.
+ */
+ public static final int DEFAULT_NUMBER_OF_SUBRANGES = 32;
+
+ /**
+ * Number of evenly split subranges to create for each node that repair runs for.
+ * <p>
+ * If vnodes are configured using <code>num_tokens</code>, attempts to evenly subdivide subranges by each range.
+ * For example, for <code>num_tokens: 16</code> and <code>number_of_subranges: 32</code>, <code>2 (32/16)</code>
+ * repair assignments will be created for each token range. At least one repair assignment will be
+ * created for each token range.
+ */
+ static final String NUMBER_OF_SUBRANGES = "number_of_subranges";
+
+ private final AutoRepairConfig.RepairType repairType;
+ private int numberOfSubranges;
+
+ public FixedSplitTokenRangeSplitter(AutoRepairConfig.RepairType repairType, Map<String, String> parameters)
+ {
+ this.repairType = repairType;
+
+ numberOfSubranges = Integer.parseInt(parameters.getOrDefault(NUMBER_OF_SUBRANGES, Integer.toString(DEFAULT_NUMBER_OF_SUBRANGES)));
+ }
+
+ @Override
+ public Iterator<KeyspaceRepairAssignments> getRepairAssignments(boolean primaryRangeOnly, List<PrioritizedRepairPlan> repairPlans)
+ {
+ return new RepairAssignmentIterator(repairPlans)
+ {
+ @Override
+ protected KeyspaceRepairAssignments next(int priority, KeyspaceRepairPlan repairPlan)
+ {
+ return getRepairAssignmentsForKeyspace(primaryRangeOnly, priority, repairPlan);
+ }
+ };
+ }
+
+ private KeyspaceRepairAssignments getRepairAssignmentsForKeyspace(boolean primaryRangeOnly, int priority, KeyspaceRepairPlan repairPlan)
+ {
+ AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig();
+ List<RepairAssignment> repairAssignments = new ArrayList<>();
+ String keyspaceName = repairPlan.getKeyspaceName();
+ List<String> tableNames = repairPlan.getTableNames();
+
+ Collection<Range<Token>> tokens = AutoRepairUtils.getTokenRanges(primaryRangeOnly, keyspaceName);
+ boolean byKeyspace = config.getRepairByKeyspace(repairType);
+ // collect all token ranges.
+ List<Range<Token>> allRanges = new ArrayList<>();
+ // this is done to avoid micro splits in the case of vnodes
+ int splitsPerRange = Math.max(1, numberOfSubranges / tokens.size());
+ for (Range<Token> token : tokens)
+ {
+ allRanges.addAll(split(token, splitsPerRange));
+ }
+
+ if (byKeyspace)
+ {
+
+ // This calculation is the best effort for the FixedSplitTokenRangeSplitter.
+ // In practice, this metric may not give you an accurate view in case of uneven data distribution.
+ long totalBytes = repairPlan.getEstimatedBytes();
+ long bytesPerRange = Math.max(1, totalBytes / splitsPerRange);
+ for (Range<Token> splitRange : allRanges)
+ {
+ // add repair assignment for each range entire keyspace's tables
+ repairAssignments.add(new RepairAssignment(splitRange, keyspaceName, tableNames, bytesPerRange));
+ }
+ }
+ else
+ {
+ // add repair assignment per table
+ for (String tableName : tableNames)
+ {
+ long totalBytes = repairPlan.getTableEstimatedBytes(AutoRepairUtils.getKeyspaceTableName(keyspaceName, tableName));
+ long bytesPerRange = Math.max(1, totalBytes / splitsPerRange);
+ for (Range<Token> splitRange : allRanges)
+ {
+ repairAssignments.add(new RepairAssignment(splitRange, keyspaceName, Collections.singletonList(tableName), bytesPerRange));
+ }
+ }
+ }
+ return new KeyspaceRepairAssignments(priority, keyspaceName, repairAssignments);
+ }
+
+ @Override
+ public void setParameter(String key, String value)
+ {
+ if (!key.equals(NUMBER_OF_SUBRANGES))
+ {
+ throw new IllegalArgumentException("Unexpected parameter '" + key + "', must be " + NUMBER_OF_SUBRANGES);
+ }
+ logger.info("Setting {} to {} for repair type {}", key, value, repairType);
+ this.numberOfSubranges = Integer.parseInt(value);
+ }
+
+ @Override
+ public Map<String, String> getParameters()
+ {
+ return Collections.singletonMap(NUMBER_OF_SUBRANGES, Integer.toString(numberOfSubranges));
+ }
+}
diff --git a/src/java/org/apache/cassandra/repair/autorepair/IAutoRepairTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/IAutoRepairTokenRangeSplitter.java
new file mode 100644
index 0000000000..8b82eac296
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/autorepair/IAutoRepairTokenRangeSplitter.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.repair.autorepair;
+
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.cassandra.config.ParameterizedClass;
+
+/**
+ * Interface that defines how to generate {@link KeyspaceRepairAssignments}.
+ * <p/>
+ * The default is {@link RepairTokenRangeSplitter} which aims to provide sensible defaults for all repair types.
+ * <p/>
+ * Custom implementations class should require a constructor accepting
+ * ({@link AutoRepairConfig.RepairType}, {@link java.util.Map}) with the {@link java.util.Map} parameter accepting
+ * custom configuration for your splitter. If such a constructor does not exist,
+ * {@link AutoRepairConfig#newAutoRepairTokenRangeSplitter(AutoRepairConfig.RepairType, ParameterizedClass)}
+ * will fall back on invoking a default zero argument constructor.
+ */
+public interface IAutoRepairTokenRangeSplitter
+{
+ /**
+ * Split the token range you wish to repair into multiple assignments.
+ * The autorepair framework will repair the assignments from returned subrange iterator in the sequence it's
+ * provided.
+ * @param primaryRangeOnly Whether to repair only this node's primary ranges or all of its ranges.
+ * @param repairPlans A list of ordered prioritized repair plans to generate assignments for in order.
+ * @return iterator of repair assignments, with each element representing a grouping of repair assignments for a given keyspace.
+ * The iterator is traversed lazily {@link KeyspaceRepairAssignments} at a time with the intent to try to get the
+ * most up-to-date representation of your data (e.g. how much data exists and is unrepaired at a given time).
+ */
+ Iterator<KeyspaceRepairAssignments> getRepairAssignments(boolean primaryRangeOnly, List<PrioritizedRepairPlan> repairPlans);
+
+ /**
+ * Update a configuration parameter. This is meant to be used by <code>nodetool setautorepairconfig</code> to
+ * update configuration dynamically.
+ * @param key parameter to update
+ * @param value The value to set to.
+ */
+ default void setParameter(String key, String value)
+ {
+ throw new IllegalArgumentException(this.getClass().getName() + " does not support custom configuration");
+ }
+
+ /**
+ * @return custom configuration. This is meant to be used by <code>nodetool getautorepairconfig</code> for
+ * retrieving the splitter configuration.
+ */
+ default Map<String, String> getParameters()
+ {
+ return Collections.emptyMap();
+ }
+}
diff --git a/src/java/org/apache/cassandra/repair/autorepair/KeyspaceRepairAssignments.java b/src/java/org/apache/cassandra/repair/autorepair/KeyspaceRepairAssignments.java
new file mode 100644
index 0000000000..3ea91e9922
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/autorepair/KeyspaceRepairAssignments.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.autorepair;
+
+import java.util.List;
+
+/**
+ * A grouping of repair assignments that were generated for a particular keyspace for a given priority.
+ */
+public class KeyspaceRepairAssignments
+{
+ private final int priority;
+ private final String keyspaceName;
+ private final List<RepairAssignment> repairAssignments;
+
+ public KeyspaceRepairAssignments(int priority, String keyspaceName, List<RepairAssignment> repairAssignments)
+ {
+ this.priority = priority;
+ this.keyspaceName = keyspaceName;
+ this.repairAssignments = repairAssignments;
+ }
+
+ public int getPriority()
+ {
+ return priority;
+ }
+
+ public String getKeyspaceName()
+ {
+ return keyspaceName;
+ }
+
+ public List<RepairAssignment> getRepairAssignments()
+ {
+ return repairAssignments;
+ }
+}
diff --git a/src/java/org/apache/cassandra/repair/autorepair/KeyspaceRepairPlan.java b/src/java/org/apache/cassandra/repair/autorepair/KeyspaceRepairPlan.java
new file mode 100644
index 0000000000..750e56e9d2
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/autorepair/KeyspaceRepairPlan.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.autorepair;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+
+/**
+ * Encapsulates an intent to repair the given keyspace's tables
+ */
+public class KeyspaceRepairPlan
+{
+ private final String keyspaceName;
+
+ private final List<String> tableNames;
+
+ @VisibleForTesting
+ public Map<String, Map<Range<Token>, AutoRepairUtils.SizeEstimate>> ksTablesEstimatedBytes;
+
+ public KeyspaceRepairPlan(String keyspaceName, List<String> tableNames, Map<String, Map<Range<Token>, AutoRepairUtils.SizeEstimate>> ksTablesEstimatedBytes)
+ {
+ this.keyspaceName = keyspaceName;
+ this.tableNames = tableNames;
+ this.ksTablesEstimatedBytes = ksTablesEstimatedBytes;
+ }
+
+ public String getKeyspaceName()
+ {
+ return keyspaceName;
+ }
+
+ public List<String> getTableNames()
+ {
+ return tableNames;
+ }
+
+ public long getEstimatedBytes()
+ {
+ return ksTablesEstimatedBytes.values().stream()
+ .flatMap(tableMap -> tableMap.values().stream())
+ .mapToLong(sizeEstimate -> sizeEstimate.sizeForRepair)
+ .sum();
+ }
+
+ public long getTableEstimatedBytes(String keyspaceTableName)
+ {
+ return ksTablesEstimatedBytes.getOrDefault(keyspaceTableName,
+ Collections.emptyMap()).values().stream().mapToLong(sizeEstimate -> sizeEstimate.sizeForRepair).sum();
+ }
+
+ public AutoRepairUtils.SizeEstimate getSizeEstimate(String keyspaceTableName, Range<Token> tokenRange)
+ {
+ return ksTablesEstimatedBytes == null ? null
+ : ksTablesEstimatedBytes.getOrDefault(keyspaceTableName, null) == null ? null
+ : ksTablesEstimatedBytes.get(keyspaceTableName).get(tokenRange);
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (o == null || getClass() != o.getClass()) return false;
+ KeyspaceRepairPlan that = (KeyspaceRepairPlan) o;
+ return Objects.equals(keyspaceName, that.keyspaceName) && Objects.equals(tableNames, that.tableNames)
+ && Objects.equals(ksTablesEstimatedBytes, that.ksTablesEstimatedBytes);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(keyspaceName, tableNames, ksTablesEstimatedBytes);
+ }
+
+ @Override
+ public String toString()
+ {
+ return "KeyspaceRepairPlan{" +
+ "keyspaceName='" + keyspaceName + '\'' +
+ ", tableNames=" + tableNames +
+ ", ksTablesEstimatedBytes=" + ksTablesEstimatedBytes +
+ '}';
+ }
+}
diff --git a/src/java/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlan.java b/src/java/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlan.java
new file mode 100644
index 0000000000..4457ccdd34
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/autorepair/PrioritizedRepairPlan.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.autorepair;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.TreeSet;
+import java.util.function.Consumer;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+
+/**
+ * Encapsulates a devised plan to repair tables, grouped by their keyspace and a given priority. This is used
+ * by {@link AutoRepair} to pass in an organized plan to
+ * {@link IAutoRepairTokenRangeSplitter#getRepairAssignments(boolean, List)} which
+ * can iterate over this plan in order to generate {@link RepairAssignment}s.
+ */
+public class PrioritizedRepairPlan
+{
+ private final int priority;
+
+ private final List<KeyspaceRepairPlan> keyspaceRepairPlans;
+
+ public PrioritizedRepairPlan(int priority, List<KeyspaceRepairPlan> keyspaceRepairPlans)
+ {
+ this.priority = priority;
+ this.keyspaceRepairPlans = keyspaceRepairPlans;
+ }
+
+ public int getPriority()
+ {
+ return priority;
+ }
+
+ public List<KeyspaceRepairPlan> getKeyspaceRepairPlans()
+ {
+ return keyspaceRepairPlans;
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (o == null || getClass() != o.getClass()) return false;
+ PrioritizedRepairPlan that = (PrioritizedRepairPlan) o;
+ return priority == that.priority && Objects.equals(keyspaceRepairPlans, that.keyspaceRepairPlans);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(priority, keyspaceRepairPlans);
+ }
+
+ @Override
+ public String toString()
+ {
+ return "PrioritizedRepairPlan{" +
+ "priority=" + priority +
+ ", keyspaceRepairPlans=" + keyspaceRepairPlans +
+ '}';
+ }
+
+ /**
+ * Builds a list of {@link PrioritizedRepairPlan}s for the given keyspace and table map, ordered by priority from
+ * highest to lowest, where priority is derived from table schema's defined priority for the given repair type.
+ * <p>
+ * If a keyspace has tables with differing priorities, those tables will be included in the PrioritizedRepairPlan
+ * for their given priority.
+ *
+ * @param keyspacesToTableNames A mapping keyspace to table names
+ * @param repairType The repair type that is being executed
+ * @param orderFunc A function to order keyspace and tables in the returned plan.
+ * @return Ordered list of plan's by table priorities.
+ */
+ public static List<PrioritizedRepairPlan> build(Map<String, List<String>> keyspacesToTableNames, AutoRepairConfig.RepairType repairType, Consumer<List<String>> orderFunc, boolean primaryRangeOnly)
+ {
+ // Build a map of priority -> (keyspace -> tables)
+ Map<Integer, Map<String, List<String>>> plans = new HashMap<>();
+ for (Map.Entry<String, List<String>> keyspaceToTableNames : keyspacesToTableNames.entrySet())
+ {
+ String keyspaceName = keyspaceToTableNames.getKey();
+ for (String tableName : keyspaceToTableNames.getValue())
+ {
+ int priority = getPriority(repairType, keyspaceName, tableName);
+ Map<String, List<String>> keyspacesForPriority = plans.computeIfAbsent(priority, p -> new HashMap<>());
+ List<String> tableNamesAtPriority = keyspacesForPriority.computeIfAbsent(keyspaceName, k -> new ArrayList<>());
+ tableNamesAtPriority.add(tableName);
+ }
+ }
+
+ // Extract map into a List<PrioritizedRepairPlan> ordered by priority from highest to lowest.
+ List<PrioritizedRepairPlan> planList = new ArrayList<>(plans.size());
+ TreeSet<Integer> priorities = new TreeSet<>(Comparator.reverseOrder());
+ priorities.addAll(plans.keySet());
+ for (int priority : priorities)
+ {
+ Map<String, List<String>> keyspacesAndTables = plans.get(priority);
+ List<KeyspaceRepairPlan> keyspaceRepairPlans = new ArrayList<>(keyspacesAndTables.size());
+ planList.add(new PrioritizedRepairPlan(priority, keyspaceRepairPlans));
+
+ // Order keyspace and table names based on the input function (typically, this would shuffle the keyspace
+ // and table names randomly).
+ List<String> keyspaceNames = new ArrayList<>(keyspacesAndTables.keySet());
+ orderFunc.accept(keyspaceNames);
+
+ for (String keyspaceName : keyspaceNames)
+ {
+ List<String> tableNames = keyspacesAndTables.get(keyspaceName);
+ orderFunc.accept(tableNames);
+ KeyspaceRepairPlan keyspaceRepairPlan =
+ new KeyspaceRepairPlan(keyspaceName, new ArrayList<>(tableNames),
+ AutoRepairUtils.calcTotalBytesToBeRepaired(repairType, keyspaceName, tableNames, AutoRepairUtils.getTokenRanges(primaryRangeOnly, keyspaceName)));
+ keyspaceRepairPlans.add(keyspaceRepairPlan);
+ }
+ }
+
+ return planList;
+ }
+
+ /**
+ * @return The priority of the given table if defined, otherwise 0.
+ */
+ private static int getPriority(AutoRepairConfig.RepairType repairType, String keyspaceName, String tableName)
+ {
+ ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(keyspaceName, tableName);
+ return cfs != null ? cfs.metadata().params.autoRepair.priority() : 0;
+ }
+}
diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairAssignment.java b/src/java/org/apache/cassandra/repair/autorepair/RepairAssignment.java
new file mode 100644
index 0000000000..6e07399aad
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/autorepair/RepairAssignment.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.autorepair;
+
+import java.util.List;
+import java.util.Objects;
+
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+
+/**
+ * Defines a repair assignment to be issued by the autorepair framework.
+ */
+public class RepairAssignment
+{
+ final Range<Token> tokenRange;
+
+ final String keyspaceName;
+
+ final List<String> tableNames;
+
+ protected final long estimatedBytes;
+
+ public RepairAssignment(Range<Token> tokenRange, String keyspaceName, List<String> tableNames, long estimatedBytes)
+ {
+ this.tokenRange = tokenRange;
+ this.keyspaceName = keyspaceName;
+ this.tableNames = tableNames;
+ this.estimatedBytes = estimatedBytes;
+ }
+
+ public Range<Token> getTokenRange()
+ {
+ return tokenRange;
+ }
+
+ public String getKeyspaceName()
+ {
+ return keyspaceName;
+ }
+
+ public List<String> getTableNames()
+ {
+ return tableNames;
+ }
+
+ public long getEstimatedBytes()
+ {
+ return estimatedBytes;
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ RepairAssignment that = (RepairAssignment) o;
+ return Objects.equals(tokenRange, that.tokenRange) && Objects.equals(keyspaceName, that.keyspaceName)
+ && Objects.equals(tableNames, that.tableNames) && Objects.equals(estimatedBytes, that.estimatedBytes);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(tokenRange, keyspaceName, tableNames, estimatedBytes);
+ }
+
+ @Override
+ public String toString()
+ {
+ return "RepairAssignment{" +
+ "tokenRange=" + tokenRange +
+ ", keyspaceName='" + keyspaceName + '\'' +
+ ", tableNames=" + tableNames +
+ ", estimatedBytes=" + estimatedBytes +
+ '}';
+ }
+}
diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairAssignmentIterator.java b/src/java/org/apache/cassandra/repair/autorepair/RepairAssignmentIterator.java
new file mode 100644
index 0000000000..44d9f5ef5e
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/autorepair/RepairAssignmentIterator.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.autorepair;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+/**
+ * Convenience {@link Iterator} implementation to assist implementations of
+ * {@link IAutoRepairTokenRangeSplitter#getRepairAssignments(boolean, List)} by passing {@link KeyspaceRepairPlan}
+ * to a custom {@link #next(int, KeyspaceRepairPlan)} method in priority order.
+ */
+public abstract class RepairAssignmentIterator implements Iterator<KeyspaceRepairAssignments>
+{
+ private final Iterator<PrioritizedRepairPlan> repairPlanIterator;
+
+ private Iterator<KeyspaceRepairPlan> currentIterator = null;
+ private PrioritizedRepairPlan currentPlan = null;
+
+ public RepairAssignmentIterator(List<PrioritizedRepairPlan> repairPlans)
+ {
+ this.repairPlanIterator = repairPlans.iterator();
+ }
+
+ private synchronized Iterator<KeyspaceRepairPlan> currentIterator()
+ {
+ if (currentIterator == null || !currentIterator.hasNext())
+ {
+ // Advance the repair plan iterator if the current repair plan is exhausted, but only
+ // if there are more repair plans.
+ if (repairPlanIterator.hasNext())
+ {
+ currentPlan = repairPlanIterator.next();
+ currentIterator = currentPlan.getKeyspaceRepairPlans().iterator();
+ }
+ }
+ return currentIterator;
+ }
+
+ @Override
+ public boolean hasNext()
+ {
+ Iterator<KeyspaceRepairPlan> iterator = currentIterator();
+ return (iterator != null && iterator.hasNext());
+ }
+
+ @Override
+ public KeyspaceRepairAssignments next()
+ {
+ if (!hasNext())
+ {
+ throw new NoSuchElementException("No remaining repair plans");
+ }
+
+ final KeyspaceRepairPlan repairPlan = currentIterator().next();
+ return next(currentPlan.getPriority(), repairPlan);
+ }
+
+ /**
+ * Invoked by {@link #next()} with the next {@link KeyspaceRepairPlan} for the given priority.
+ * @param priority current priority being processed.
+ * @param repairPlan the next keyspace repair plan to process
+ * @return assignments for the given keyspace at this priority. Should never return null, if one desires to
+ * short-circuit the iterator, override {@link #hasNext()}.
+ */
+ protected abstract KeyspaceRepairAssignments next(int priority, KeyspaceRepairPlan repairPlan);
+}
diff --git a/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java b/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java
new file mode 100644
index 0000000000..4e44b06208
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java
@@ -0,0 +1,810 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair.autorepair;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.DataStorageSpec;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.lifecycle.SSTableIntervalTree;
+import org.apache.cassandra.db.lifecycle.SSTableSet;
+import org.apache.cassandra.db.lifecycle.View;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.service.AutoRepairService;
+import org.apache.cassandra.utils.concurrent.Refs;
+
+import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.split;
+
+/**
+ * The default implementation of {@link IAutoRepairTokenRangeSplitter} that attempts to:
+ * <ol>
+ * <li>Create smaller, consistent repair times</li>
+ * <li>Minimize the impact on hosts</li>
+ * <li>Reduce overstreaming</li>
+ * <li>Reduce number of repairs</li>
+ * </ol>
+ * <p>
+ * To achieve these goals, this implementation inspects SSTable metadata to estimate the bytes and number of partitions
+ * within a range and splits it accordingly to bound the size of the token ranges used for repair assignments.
+ * </p>
+ * <p>
+ * Refer to
+ * <a href="https://cassandra.apache.org/doc/latest/cassandra/managing/operating/auto_repair.html#repair-token-range-splitter">Auto Repair documentation for this implementation</a>
+ * for a more thorough breakdown of this implementation.
+ * </p>
+ * <p>
+ * While this splitter has a lot of tuning parameters, the expectation is that the established default configuration
+ * shall be sensible for all {@link org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType}'s. The following
+ * configuration parameters are offered.
+ * </p>
+ *
+ * <p><b>Configuration parameters:</b></p>
+ * <ul>
+ * <li><b>bytes_per_assignment</b> – Target size (in compressed bytes) for each repair. Throttles incremental repair
+ * and anticompaction per schedule after incremental repairs are enabled.</li>
+ *
+ * <li><b>max_bytes_per_schedule</b> – Maximum data (in compressed bytes) to cover in a single schedule. Acts as a
+ * throttle for the repair cycle workload. Tune this up if writes are outpacing repair, or down if repairs are too
+ * disruptive. Alternatively, adjust {@code min_repair_interval}.</li>
+ *
+ * <li><b>partitions_per_assignment</b> – Maximum number of partitions per repair assignment. Limits the number of
+ * partitions in Merkle tree leaves to prevent overstreaming.</li>
+ *
+ * <li><b>max_tables_per_assignment</b> – Maximum number of tables to include in a single repair assignment.
+ * Especially useful for keyspaces with many tables. Prevents excessive batching of tables that exceed other
+ * parameters like {@code bytes_per_assignment} or {@code partitions_per_assignment}.</li>
+ * </ul>
+ */
+public class RepairTokenRangeSplitter implements IAutoRepairTokenRangeSplitter
+{
+ private static final Logger logger = LoggerFactory.getLogger(RepairTokenRangeSplitter.class);
+
+ // Default max bytes to 100TiB, which is much more readable than Long.MAX_VALUE
+ private static final DataStorageSpec.LongBytesBound MAX_BYTES = new DataStorageSpec.LongBytesBound(102_400, DataStorageSpec.DataStorageUnit.GIBIBYTES);
+
+ /**
+ * The target bytes that should be included in a repair assignment
+ */
+ static final String BYTES_PER_ASSIGNMENT = "bytes_per_assignment";
+
+ /**
+ * Maximum number of partitions to include in a repair assignment
+ */
+ static final String PARTITIONS_PER_ASSIGNMENT = "partitions_per_assignment";
+
+ /**
+ * Maximum number of tables to include in a repair assignment if {@link AutoRepairConfig.Options#repair_by_keyspace}
+ * is enabled
+ */
+ static final String MAX_TABLES_PER_ASSIGNMENT = "max_tables_per_assignment";
+
+ /**
+ * The maximum number of bytes to cover in an individual schedule
+ */
+ static final String MAX_BYTES_PER_SCHEDULE = "max_bytes_per_schedule";
+
+ static final List<String> PARAMETERS = Arrays.asList(BYTES_PER_ASSIGNMENT, PARTITIONS_PER_ASSIGNMENT, MAX_TABLES_PER_ASSIGNMENT, MAX_BYTES_PER_SCHEDULE);
+
+ private final AutoRepairConfig.RepairType repairType;
+
+ private final Map<String, String> givenParameters = new HashMap<>();
+
+ private DataStorageSpec.LongBytesBound bytesPerAssignment;
+ private long partitionsPerAssignment;
+ private int maxTablesPerAssignment;
+ private DataStorageSpec.LongBytesBound maxBytesPerSchedule;
+
+ /**
+ * Established default for each {@link org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType}, meant to
+ * choose sensible defaults for each.
+ * <p>
+ * Defaults if not specified for the given repair type:
+ * <li>
+ * <ul><b>bytes_per_assignment</b>: 50GiB</ul>
+ * <ul><b>partitions_per_assignment</b>: 1048576 (2^20)</ul>
+ * <ul><b>max_tables_per_assignment</b>: 64</ul>
+ * <ul><b>max_bytes_per_schedule</b>: 1000GiB</ul>
+ * </li>
+ * It's expected that these defaults should work well for everything except incremental, where we set
+ * max_bytes_per_schedule to 100GiB. This should strike a good balance between the amount of data that will be
+ * repaired during an initial migration to incremental repair and should move the entire repaired set from
+ * unrepaired to repaired at steady state, assuming not more the 100GiB of data is written to a node per
+ * min_repair_interval.
+ */
+ private static final Map<AutoRepairConfig.RepairType, RepairTypeDefaults> DEFAULTS_BY_REPAIR_TYPE = new EnumMap<AutoRepairConfig.RepairType, RepairTypeDefaults>(AutoRepairConfig.RepairType.class) {{
+ put(AutoRepairConfig.RepairType.FULL, RepairTypeDefaults.builder(AutoRepairConfig.RepairType.FULL)
+ .build());
+ // Restrict incremental repair to 100GiB max bytes per schedule to confine the amount of possible autocompaction.
+ put(AutoRepairConfig.RepairType.INCREMENTAL, RepairTypeDefaults.builder(AutoRepairConfig.RepairType.INCREMENTAL)
+ .withMaxBytesPerSchedule(new DataStorageSpec.LongBytesBound("100GiB"))
+ .build());
+ put(AutoRepairConfig.RepairType.PREVIEW_REPAIRED, RepairTypeDefaults.builder(AutoRepairConfig.RepairType.PREVIEW_REPAIRED)
+ .build());
+ }};
+
+ public RepairTokenRangeSplitter(AutoRepairConfig.RepairType repairType, Map<String, String> parameters)
+ {
+ this.repairType = repairType;
+ this.givenParameters.putAll(parameters);
+
+ reinitParameters();
+ }
+
+ private void reinitParameters()
+ {
+ RepairTypeDefaults defaults = DEFAULTS_BY_REPAIR_TYPE.get(repairType);
+
+ DataStorageSpec.LongBytesBound bytesPerAssignmentTmp = getPropertyOrDefault(BYTES_PER_ASSIGNMENT, DataStorageSpec.LongBytesBound::new, defaults.bytesPerAssignment);
+ DataStorageSpec.LongBytesBound maxBytesPerScheduleTmp = getPropertyOrDefault(MAX_BYTES_PER_SCHEDULE, DataStorageSpec.LongBytesBound::new, defaults.maxBytesPerSchedule);
+
+ // Validate that bytesPerAssignment <= maxBytesPerSchedule
+ if (bytesPerAssignmentTmp.toBytes() > maxBytesPerScheduleTmp.toBytes())
+ {
+ throw new IllegalArgumentException(String.format("%s='%s' cannot be greater than %s='%s' for %s",
+ BYTES_PER_ASSIGNMENT,
+ bytesPerAssignmentTmp,
+ MAX_BYTES_PER_SCHEDULE,
+ maxBytesPerScheduleTmp,
+ repairType.getConfigName()));
+ }
+
+ bytesPerAssignment = bytesPerAssignmentTmp;
+ maxBytesPerSchedule = maxBytesPerScheduleTmp;
+
+ partitionsPerAssignment = getPropertyOrDefault(PARTITIONS_PER_ASSIGNMENT, Long::parseLong, defaults.partitionsPerAssignment);
+ maxTablesPerAssignment = getPropertyOrDefault(MAX_TABLES_PER_ASSIGNMENT, Integer::parseInt, defaults.maxTablesPerAssignment);
+
+ logger.info("Configured {}[{}] with {}={}, {}={}, {}={}, {}={}", RepairTokenRangeSplitter.class.getName(),
+ repairType.getConfigName(),
+ BYTES_PER_ASSIGNMENT, bytesPerAssignment,
+ PARTITIONS_PER_ASSIGNMENT, partitionsPerAssignment,
+ MAX_TABLES_PER_ASSIGNMENT, maxTablesPerAssignment,
+ MAX_BYTES_PER_SCHEDULE, maxBytesPerSchedule);
+ }
+
+ private <T> T getPropertyOrDefault(String propertyName, Function<String, T> mapper, T defaultValue)
+ {
+ return Optional.ofNullable(this.givenParameters.get(propertyName)).map(mapper).orElse(defaultValue);
+ }
+
+ @Override
+ public Iterator<KeyspaceRepairAssignments> getRepairAssignments(boolean primaryRangeOnly, List<PrioritizedRepairPlan> repairPlans)
+ {
+ return new BytesBasedRepairAssignmentIterator(primaryRangeOnly, repairPlans);
+ }
+
+ /**
+ * A custom {@link RepairAssignmentIterator} that confines the number of repair assignments to
+ * <code>max_bytes_per_schedule</code>.
+ */
+ private class BytesBasedRepairAssignmentIterator extends RepairAssignmentIterator
+ {
+
+ private final boolean primaryRangeOnly;
+ private long bytesSoFar = 0;
+
+ BytesBasedRepairAssignmentIterator(boolean primaryRangeOnly, List<PrioritizedRepairPlan> repairPlans)
+ {
+ super(repairPlans);
+ this.primaryRangeOnly = primaryRangeOnly;
+ }
+
+ @Override
+ protected KeyspaceRepairAssignments next(int priority, KeyspaceRepairPlan repairPlan)
+ {
+ // short circuit if we've accumulated too many bytes by returning a KeyspaceRepairAssignments with
+ // no assignments. We do this rather than returning false in hasNext() because we want to signal
+ // to AutoRepair that a keyspace generated no assignments.
+ if (bytesSoFar >= maxBytesPerSchedule.toBytes())
+ {
+ return new KeyspaceRepairAssignments(priority, repairPlan.getKeyspaceName(), Collections.emptyList());
+ }
+
+ List<Range<Token>> tokenRanges = AutoRepairUtils.getTokenRanges(primaryRangeOnly, repairPlan.getKeyspaceName());
+ // shuffle token ranges to unbias selection of ranges
+ Collections.shuffle(tokenRanges);
+ List<SizedRepairAssignment> repairAssignments = new ArrayList<>();
+ // Generate assignments for each range speparately
+ for (Range<Token> tokenRange : tokenRanges)
+ {
+ repairAssignments.addAll(getRepairAssignmentsForKeyspace(repairType, repairPlan, tokenRange));
+ }
+
+ FilteredRepairAssignments filteredRepairAssignments = filterRepairAssignments(priority, repairPlan.getKeyspaceName(), repairAssignments, bytesSoFar);
+ bytesSoFar = filteredRepairAssignments.newBytesSoFar;
+ return new KeyspaceRepairAssignments(priority, repairPlan.getKeyspaceName(), filteredRepairAssignments.repairAssignments);
+ }
+ }
+
+ @VisibleForTesting
+ List<SizedRepairAssignment> getRepairAssignmentsForKeyspace(AutoRepairConfig.RepairType repairType, KeyspaceRepairPlan repairPlan, Range<Token> tokenRange)
+ {
+ List<SizedRepairAssignment> repairAssignments = new ArrayList<>();
+ // this is used for batching minimal single assignment tables together
+ List<SizedRepairAssignment> currentAssignments = new ArrayList<>();
+
+ AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig();
+
+ // If we can repair by keyspace, sort the tables by size so can batch the smallest ones together
+ boolean repairByKeyspace = config.getRepairByKeyspace(repairType);
+ List<String> tablesToProcess = repairPlan.getTableNames();
+ if (repairByKeyspace)
+ {
+ tablesToProcess = repairPlan.getTableNames().stream().sorted((t1, t2) -> {
+ ColumnFamilyStore cfs1 = ColumnFamilyStore.getIfExists(repairPlan.getKeyspaceName(), t1);
+ ColumnFamilyStore cfs2 = ColumnFamilyStore.getIfExists(repairPlan.getKeyspaceName(), t2);
+ // If for whatever reason the CFS is not retrievable, we can assume it has been deleted, so give the
+ // other cfs precedence.
+ if (cfs1 == null)
+ {
+ // cfs1 is lesser than because its null
+ return -1;
+ }
+ else if (cfs2 == null)
+ {
+ // cfs1 is greather than because cfs2 is null
+ return 1;
+ }
+ return Long.compare(cfs1.metric.totalDiskSpaceUsed.getCount(), cfs2.metric.totalDiskSpaceUsed.getCount());
+ }).collect(Collectors.toList());
+ }
+
+ for (String tableName : tablesToProcess)
+ {
+ List<SizedRepairAssignment> tableAssignments = getRepairAssignmentsForTable(repairPlan, tableName, tokenRange);
+
+ if (tableAssignments.isEmpty())
+ continue;
+
+ // if not repairing by keyspace don't attempt to batch them with others.
+ if (!repairByKeyspace)
+ {
+ repairAssignments.addAll(tableAssignments);
+ }
+ // If the table assignments are for the same token range, and we have room to add more tables to the current assignment
+ else if (tableAssignments.size() == 1 &&
+ currentAssignments.size() < maxTablesPerAssignment &&
+ (currentAssignments.isEmpty() || currentAssignments.get(0).getTokenRange().equals(tableAssignments.get(0).getTokenRange())))
+ {
+ long currentAssignmentsBytes = getEstimatedBytes(currentAssignments);
+ long tableAssignmentsBytes = getEstimatedBytes(tableAssignments);
+ // only add assignments together if they don't exceed max bytes per schedule.
+ if (currentAssignmentsBytes + tableAssignmentsBytes < maxBytesPerSchedule.toBytes()) {
+ currentAssignments.addAll(tableAssignments);
+ }
+ else
+ {
+ // add table assignments by themselves
+ repairAssignments.addAll(tableAssignments);
+ }
+ }
+ else
+ {
+ if (!currentAssignments.isEmpty())
+ {
+ repairAssignments.add(merge(currentAssignments));
+ currentAssignments.clear();
+ }
+ repairAssignments.addAll(tableAssignments);
+ }
+ }
+
+ if (!currentAssignments.isEmpty())
+ repairAssignments.add(merge(currentAssignments));
+
+ return repairAssignments;
+ }
+
+ /**
+ * Given a repair type and map of sized-based repair assignments, confine them by <code>maxBytesPerSchedule</code>.
+ *
+ * @param repairAssignments the assignments to filter.
+ * @param bytesSoFar repair assignment bytes accumulated so far.
+ * @return A list of repair assignments confined by <code>maxBytesPerSchedule</code>.
+ */
+ @VisibleForTesting
+ FilteredRepairAssignments filterRepairAssignments(int priority, String keyspaceName, List<SizedRepairAssignment> repairAssignments, long bytesSoFar)
+ {
+ // Confine repair assignments by maxBytesPerSchedule.
+ long bytesSoFarThisIteration = 0L;
+ long bytesNotRepaired = 0L;
+ int assignmentsNotRepaired = 0;
+ int assignmentsToRepair = 0;
+ int totalAssignments = 0;
+
+ List<RepairAssignment> assignmentsToReturn = new ArrayList<>(repairAssignments.size());
+ for (SizedRepairAssignment repairAssignment : repairAssignments)
+ {
+ totalAssignments++;
+ // skip any repair assignments that would accumulate us past the maxBytesPerSchedule
+ if (bytesSoFar + repairAssignment.getEstimatedBytes() > maxBytesPerSchedule.toBytes())
+ {
+ // log that repair assignment was skipped.
+ bytesNotRepaired += repairAssignment.getEstimatedBytes();
+ assignmentsNotRepaired++;
+ logger.warn("Skipping {} because it would increase total repair bytes to {}",
+ repairAssignment,
+ getBytesOfMaxBytesPerSchedule(bytesSoFar + repairAssignment.getEstimatedBytes()));
+ }
+ else
+ {
+ bytesSoFar += repairAssignment.getEstimatedBytes();
+ bytesSoFarThisIteration += repairAssignment.getEstimatedBytes();
+ assignmentsToRepair++;
+ logger.info("Adding {}, increasing repair bytes to {}",
+ repairAssignment,
+ getBytesOfMaxBytesPerSchedule(bytesSoFar));
+ assignmentsToReturn.add(repairAssignment);
+ }
+ }
+
+ String message = "Returning {} assignment(s) for priorityBucket {} and keyspace {}, totaling {} ({} overall)";
+ if (assignmentsNotRepaired != 0)
+ {
+ message += ". Skipping {} of {} assignment(s), totaling {}";
+ if (repairType != AutoRepairConfig.RepairType.INCREMENTAL)
+ {
+ message += ". The entire primary range will not be repaired this schedule. " +
+ "Consider increasing maxBytesPerSchedule, reducing node density or monitoring to ensure " +
+ "all ranges do get repaired within gc_grace_seconds";
+ logger.warn(message, assignmentsToRepair, priority, keyspaceName,
+ FileUtils.stringifyFileSize(bytesSoFarThisIteration),
+ getBytesOfMaxBytesPerSchedule(bytesSoFar),
+ assignmentsNotRepaired, totalAssignments,
+ FileUtils.stringifyFileSize(bytesNotRepaired));
+ }
+ else
+ {
+ logger.info(message, assignmentsToRepair, priority, keyspaceName,
+ FileUtils.stringifyFileSize(bytesSoFarThisIteration),
+ getBytesOfMaxBytesPerSchedule(bytesSoFar),
+ assignmentsNotRepaired, totalAssignments,
+ FileUtils.stringifyFileSize(bytesNotRepaired));
+ }
+ }
+ else
+ {
+ logger.info(message, assignmentsToRepair, priority, keyspaceName,
+ FileUtils.stringifyFileSize(bytesSoFarThisIteration),
+ getBytesOfMaxBytesPerSchedule(bytesSoFar));
+ }
+
+ return new FilteredRepairAssignments(assignmentsToReturn, bytesSoFar);
+ }
+
+ @VisibleForTesting
+ static class FilteredRepairAssignments
+ {
+ final List<RepairAssignment> repairAssignments;
+ final long newBytesSoFar;
+
+ private FilteredRepairAssignments(List<RepairAssignment> repairAssignments, long newBytesSoFar)
+ {
+ this.repairAssignments = repairAssignments;
+ this.newBytesSoFar = newBytesSoFar;
+ }
+ }
+
+ private String getBytesOfMaxBytesPerSchedule(long bytes)
+ {
+ if (maxBytesPerSchedule.equals(MAX_BYTES))
+ return FileUtils.stringifyFileSize(bytes);
+ else
+ return String.format("%s of %s", FileUtils.stringifyFileSize(bytes), maxBytesPerSchedule);
+ }
+
+ /**
+ * @param repairAssignments The assignments to sum
+ * @return The sum of {@link SizedRepairAssignment#getEstimatedBytes()} of all given
+ * repairAssignments.
+ */
+ @VisibleForTesting
+ protected static long getEstimatedBytes(List<SizedRepairAssignment> repairAssignments)
+ {
+ return repairAssignments
+ .stream()
+ .mapToLong(SizedRepairAssignment::getEstimatedBytes)
+ .sum();
+ }
+
+ @VisibleForTesting
+ static SizedRepairAssignment merge(List<SizedRepairAssignment> assignments)
+ {
+ if (assignments.isEmpty())
+ throw new IllegalStateException("Cannot merge empty assignments");
+
+ Set<String> mergedTableNames = new HashSet<>();
+ Range<Token> referenceTokenRange = assignments.get(0).getTokenRange();
+ String referenceKeyspaceName = assignments.get(0).getKeyspaceName();
+
+ for (SizedRepairAssignment assignment : assignments)
+ {
+ // These checks _should_ be unnecessary but are here to ensure that the assignments are consistent
+ if (!assignment.getTokenRange().equals(referenceTokenRange))
+ throw new IllegalStateException("All assignments must have the same token range");
+ if (!assignment.getKeyspaceName().equals(referenceKeyspaceName))
+ throw new IllegalStateException("All assignments must have the same keyspace name");
+
+ mergedTableNames.addAll(assignment.getTableNames());
+ }
+
+ long sizeForAssignment = getEstimatedBytes(assignments);
+ return new SizedRepairAssignment(referenceTokenRange, referenceKeyspaceName, new ArrayList<>(mergedTableNames),
+ "full primary range for " + mergedTableNames.size() + " tables", sizeForAssignment);
+ }
+
+ @VisibleForTesting
+ protected List<SizedRepairAssignment> getRepairAssignmentsForTable(KeyspaceRepairPlan repairPlan, String tableName, Range<Token> tokenRange)
+ {
+ AutoRepairUtils.SizeEstimate sizeEstimate = repairPlan.getSizeEstimate(AutoRepairUtils.getKeyspaceTableName(repairPlan.getKeyspaceName(), tableName), tokenRange);
+ if (sizeEstimate == null)
+ {
+ // Ideally, it should have been cached already inside the KeyspaceRepairPlan, but incase it was not,
+ // then recalculating it. It is a bit expensive, but necessary for the repair
+ logger.warn("The size estimate for {}.{} range {} was not pre-calculated, calculating on-demand",
+ repairPlan.getKeyspaceName(), tableName, tokenRange);
+ sizeEstimate = AutoRepairUtils.getRangeSizeEstimate(repairType, repairPlan.getKeyspaceName(), tableName, tokenRange);
+ }
+ return getRepairAssignments(sizeEstimate);
+ }
+
+ private static void logSkippingTable(String keyspaceName, String tableName)
+ {
+ logger.warn("Could not resolve table data for {}.{} assuming it has since been deleted, skipping", keyspaceName, tableName);
+ }
+
+ @VisibleForTesting
+ protected List<SizedRepairAssignment> getRepairAssignments(AutoRepairUtils.SizeEstimate estimate)
+ {
+ List<SizedRepairAssignment> repairAssignments = new ArrayList<>();
+
+ // since its possible for us to hit maxBytesPerSchedule before seeing all ranges, shuffle so there is chance
+ // at least of hitting all the ranges _eventually_ for the worst case scenarios
+ int totalExpectedSubRanges = 0;
+ if (estimate.sizeForRepair != 0)
+ {
+ boolean needsSplitting = estimate.sizeForRepair > bytesPerAssignment.toBytes() || estimate.partitions > partitionsPerAssignment;
+ if (needsSplitting)
+ {
+ totalExpectedSubRanges += calculateNumberOfSplits(estimate);
+ }
+ }
+ if (estimate.sizeForRepair == 0)
+ {
+ ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(estimate.keyspace, estimate.table);
+
+ if (cfs == null)
+ {
+ logSkippingTable(estimate.keyspace, estimate.table);
+ return Collections.emptyList();
+ }
+
+ long memtableSize = cfs.getTracker().getView().getCurrentMemtable().getLiveDataSize();
+ if (memtableSize > 0L)
+ {
+ logger.debug("Included {}.{} range {}, had no unrepaired SSTables, but memtableSize={}, adding single repair assignment", estimate.keyspace, estimate.table, estimate.tokenRange, memtableSize);
+ SizedRepairAssignment assignment = new SizedRepairAssignment(estimate.tokenRange, estimate.keyspace, Collections.singletonList(estimate.table), "full primary rangee for table with memtable only detected", memtableSize);
+ repairAssignments.add(assignment);
+ }
+ else
+ {
+ logger.debug("Included {}.{} range {}, has no SSTables or memtable data, but adding single repair assignment for entire range in case writes were missed", estimate.keyspace, estimate.table, estimate.tokenRange);
+ SizedRepairAssignment assignment = new SizedRepairAssignment(estimate.tokenRange, estimate.keyspace, Collections.singletonList(estimate.table), "full primary range for table with no data detected", 0L);
+ repairAssignments.add(assignment);
+ }
+ }
+ else
+ {
+ // Check if the estimate needs splitting based on the criteria
+ boolean needsSplitting = estimate.sizeForRepair > bytesPerAssignment.toBytes() || estimate.partitions > partitionsPerAssignment;
+ if (needsSplitting)
+ {
+ int numberOfSplits = calculateNumberOfSplits(estimate);
+ long approximateBytesPerSplit = estimate.sizeForRepair / numberOfSplits;
+ Collection<Range<Token>> subranges = split(estimate.tokenRange, numberOfSplits);
+ for (Range<Token> subrange : subranges)
+ {
+ SizedRepairAssignment assignment = new SizedRepairAssignment(subrange, estimate.keyspace, Collections.singletonList(estimate.table),
+ String.format("subrange %d of %d", repairAssignments.size() + 1, totalExpectedSubRanges),
+ approximateBytesPerSplit);
+ repairAssignments.add(assignment);
+ }
+ }
+ else
+ {
+ // No splitting needed, repair the entire range as-is
+ SizedRepairAssignment assignment = new SizedRepairAssignment(estimate.tokenRange, estimate.keyspace,
+ Collections.singletonList(estimate.table),
+ "full primary range for table", estimate.sizeForRepair);
+ repairAssignments.add(assignment);
+ }
+ }
+ return repairAssignments;
+ }
+
+ private int calculateNumberOfSplits(AutoRepairUtils.SizeEstimate estimate)
+ {
+ // Calculate the number of splits needed for size and partitions
+ int splitsForSize = (int) Math.ceil((double) estimate.sizeForRepair / bytesPerAssignment.toBytes());
+ int splitsForPartitions = (int) Math.ceil((double) estimate.partitions / partitionsPerAssignment);
+
+ // Split the token range into subranges based on whichever (partitions, bytes) would generate the most splits.
+ boolean splitBySize = splitsForSize > splitsForPartitions;
+ int splits = splitBySize ? splitsForSize : splitsForPartitions;
+
+ // calculate approximation for logging purposes
+ long approximateBytesPerSplit = estimate.sizeForRepair / splits;
+ long approximatePartitionsPerSplit = estimate.partitions / splits;
+
+ logger.info("Splitting {}.{} for range {} into {} sub ranges by {} (splitsForSize={}, splitsForPartitions={}, " +
+ "approximateBytesInRange={}, approximatePartitionsInRange={}, " +
+ "approximateBytesPerSplit={}, approximatePartitionsPerSplit={})",
+ estimate.keyspace, estimate.table, estimate.tokenRange,
+ splits, splitBySize ? "size" : "partitions",
+ splitsForSize, splitsForPartitions,
+ FileUtils.stringifyFileSize(estimate.sizeForRepair), estimate.partitions,
+ FileUtils.stringifyFileSize(approximateBytesPerSplit), approximatePartitionsPerSplit
+ );
+ return splits;
+ }
+
+ @VisibleForTesting
+ static Refs<SSTableReader> getSSTableReaderRefs(AutoRepairConfig.RepairType repairType, String keyspaceName, String tableName, Range<Token> tokenRange)
+ {
+ final ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(keyspaceName, tableName);
+ if (cfs == null)
+ {
+ logSkippingTable(keyspaceName, tableName);
+ return Refs.ref(Collections.<SSTableReader>emptyList());
+ }
+
+ Refs<SSTableReader> refs = null;
+ while (refs == null)
+ {
+ Iterable<SSTableReader> sstables = cfs.getTracker().getView().select(SSTableSet.CANONICAL);
+ SSTableIntervalTree tree = SSTableIntervalTree.buildSSTableIntervalTree(Lists.newArrayList(sstables));
+ Range<PartitionPosition> r = Range.makeRowRange(tokenRange);
+ List<SSTableReader> canonicalSSTables = View.sstablesInBounds(r.left, r.right, tree);
+ if (repairType == AutoRepairConfig.RepairType.INCREMENTAL)
+ {
+ canonicalSSTables = canonicalSSTables.stream().filter((sstable) -> !sstable.isRepaired()).collect(Collectors.toList());
+ }
+ refs = Refs.tryRef(canonicalSSTables);
+ }
+ return refs;
+ }
+
+ @Override
+ public void setParameter(String key, String value)
+ {
+ if (!PARAMETERS.contains(key))
+ {
+ throw new IllegalArgumentException("Unexpected parameter '" + key + "', must be one of " + PARAMETERS);
+ }
+
+ logger.info("Setting {} to {} for repair type {}", key, value, repairType);
+ givenParameters.put(key, value);
+ reinitParameters();
+ }
+
+ @Override
+ public Map<String, String> getParameters()
+ {
+ final Map<String, String> parameters = new LinkedHashMap<>();
+ for (String parameter : PARAMETERS)
+ {
+ // Use the parameter as provided if present.
+ if (givenParameters.containsKey(parameter))
+ {
+ parameters.put(parameter, givenParameters.get(parameter));
+ continue;
+ }
+
+ switch (parameter)
+ {
+ case BYTES_PER_ASSIGNMENT:
+ parameters.put(parameter, bytesPerAssignment.toString());
+ continue;
+ case PARTITIONS_PER_ASSIGNMENT:
+ parameters.put(parameter, Long.toString(partitionsPerAssignment));
+ continue;
+ case MAX_TABLES_PER_ASSIGNMENT:
+ parameters.put(parameter, Integer.toString(maxTablesPerAssignment));
+ continue;
+ case MAX_BYTES_PER_SCHEDULE:
+ parameters.put(parameter, maxBytesPerSchedule.toString());
+ continue;
+ default:
+ // not expected
+ parameters.put(parameter, "");
+ }
+ }
+ return Collections.unmodifiableMap(parameters);
+ }
+
+ /**
+ * Implementation of RepairAssignment that also assigns an estimation of bytes involved
+ * in the repair.
+ */
+ @VisibleForTesting
+ protected static class SizedRepairAssignment extends RepairAssignment
+ {
+
+ final String description;
+
+ public SizedRepairAssignment(Range<Token> tokenRange, String keyspaceName, List<String> tableNames)
+ {
+ this(tokenRange, keyspaceName, tableNames, "", 0L);
+ }
+
+ public SizedRepairAssignment(Range<Token> tokenRange, String keyspaceName, List<String> tableNames,
+ String description,
+ long estimatedBytes)
+ {
+ super(tokenRange, keyspaceName, tableNames, estimatedBytes);
+ this.description = description;
+ }
+
+ /**
+ * @return Additional metadata about the repair assignment.
+ */
+ public String getDescription() {
+ return description;
+ }
+
+ /**
+ * Estimated bytes involved in the assignment. Typically Derived from {@link AutoRepairUtils.SizeEstimate#sizeForRepair}.
+ *
+ * @return estimated bytes involved in the assignment.
+ */
+ public long getEstimatedBytes()
+ {
+ return estimatedBytes;
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ if (!super.equals(o)) return false;
+ SizedRepairAssignment that = (SizedRepairAssignment) o;
+ return estimatedBytes == that.estimatedBytes && Objects.equals(description, that.description);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(super.hashCode(), description, estimatedBytes);
+ }
+
+ @Override
+ public String toString()
+ {
+ return "SizedRepairAssignment{" +
+ "description='" + description + '\'' +
+ ", tokenRange=" + tokenRange +
+ ", keyspaceName='" + keyspaceName + '\'' +
+ ", tableNames=" + tableNames +
+ ", estimatedBytes=" + FileUtils.stringifyFileSize(estimatedBytes) +
+ '}';
+ }
+ }
+
+ /**
+ * Conveinence builder for establishing defaults by repair type.
+ */
+ protected static class RepairTypeDefaults
+ {
+ final AutoRepairConfig.RepairType repairType;
+ final DataStorageSpec.LongBytesBound bytesPerAssignment;
+ final long partitionsPerAssignment;
+ final int maxTablesPerAssignment;
+ final DataStorageSpec.LongBytesBound maxBytesPerSchedule;
+
+ public RepairTypeDefaults(AutoRepairConfig.RepairType repairType,
+ DataStorageSpec.LongBytesBound bytesPerAssignment,
+ long partitionsPerAssignment,
+ int maxTablesPerAssignment,
+ DataStorageSpec.LongBytesBound maxBytesPerSchedule)
+ {
+ this.repairType = repairType;
+ this.bytesPerAssignment = bytesPerAssignment;
+ this.partitionsPerAssignment = partitionsPerAssignment;
+ this.maxTablesPerAssignment = maxTablesPerAssignment;
+ this.maxBytesPerSchedule = maxBytesPerSchedule;
+ }
+
+ static RepairTypeDefaultsBuilder builder(AutoRepairConfig.RepairType repairType)
+ {
+ return new RepairTypeDefaultsBuilder(repairType);
+ }
+
+ static class RepairTypeDefaultsBuilder
+ {
+ private final AutoRepairConfig.RepairType repairType;
+ private DataStorageSpec.LongBytesBound bytesPerAssignment = new DataStorageSpec.LongBytesBound("50GiB");
+ // Aims to target at most 1 partitions per leaf assuming a merkle tree of depth 20 (2^20 = 1,048,576)
+ private long partitionsPerAssignment = 1_048_576;
+ private int maxTablesPerAssignment = 64;
+ private DataStorageSpec.LongBytesBound maxBytesPerSchedule = MAX_BYTES;
+
+ private RepairTypeDefaultsBuilder(AutoRepairConfig.RepairType repairType)
+ {
+ this.repairType = repairType;
+ }
+
+ @SuppressWarnings("unused")
+ public RepairTypeDefaultsBuilder withBytesPerAssignment(DataStorageSpec.LongBytesBound bytesPerAssignment)
+ {
+ this.bytesPerAssignment = bytesPerAssignment;
+ return this;
+ }
+
+ @SuppressWarnings("unused")
+ public RepairTypeDefaultsBuilder withPartitionsPerAssignment(long partitionsPerAssignment)
+ {
+ this.partitionsPerAssignment = partitionsPerAssignment;
+ return this;
+ }
+
+ @SuppressWarnings("unused")
+ public RepairTypeDefaultsBuilder withMaxTablesPerAssignment(int maxTablesPerAssignment)
+ {
+ this.maxTablesPerAssignment = maxTablesPerAssignment;
+ return this;
+ }
+
+ public RepairTypeDefaultsBuilder withMaxBytesPerSchedule(DataStorageSpec.LongBytesBound maxBytesPerSchedule)
+ {
+ this.maxBytesPerSchedule = maxBytesPerSchedule;
+ return this;
+ }
+
+ public RepairTokenRangeSplitter.RepairTypeDefaults build()
+ {
+ return new RepairTypeDefaults(repairType, bytesPerAssignment, partitionsPerAssignment, maxTablesPerAssignment, maxBytesPerSchedule);
+ }
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/repair/consistent/SyncStatSummary.java b/src/java/org/apache/cassandra/repair/consistent/SyncStatSummary.java
index 855ad4bad3..820c6b011b 100644
--- a/src/java/org/apache/cassandra/repair/consistent/SyncStatSummary.java
+++ b/src/java/org/apache/cassandra/repair/consistent/SyncStatSummary.java
@@ -81,7 +81,7 @@ public class SyncStatSummary
}
}
- private static class Table
+ public static class Table
{
final String keyspace;
@@ -94,7 +94,7 @@ public class SyncStatSummary
final Map<Pair<InetSocketAddress, InetSocketAddress>, Session> sessions = new HashMap<>();
- Table(String keyspace, String table)
+ public Table(String keyspace, String table)
{
this.keyspace = keyspace;
this.table = table;
@@ -138,7 +138,7 @@ public class SyncStatSummary
totalsCalculated = true;
}
- boolean isCounter()
+ public boolean isCounter()
{
TableMetadata tmd = Schema.instance.getTableMetadata(keyspace, table);
return tmd != null && tmd.isCounter();
@@ -174,6 +174,16 @@ public class SyncStatSummary
}
return output.toString();
}
+
+ public long getBytes()
+ {
+ return this.bytes;
+ }
+
+ public long getRanges()
+ {
+ return this.ranges.size();
+ }
}
private final Map<Pair<String, String>, Table> summaries = new HashMap<>();
@@ -233,6 +243,12 @@ public class SyncStatSummary
totalsCalculated = true;
}
+ public Map<Pair<String, String>, Table> getTotals()
+ {
+ calculateTotals();
+ return summaries;
+ }
+
public String toString()
{
List<Pair<String, String>> tables = Lists.newArrayList(summaries.keySet());
diff --git a/src/java/org/apache/cassandra/schema/AutoRepairParams.java b/src/java/org/apache/cassandra/schema/AutoRepairParams.java
new file mode 100644
index 0000000000..5f05edab9d
--- /dev/null
+++ b/src/java/org/apache/cassandra/schema/AutoRepairParams.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.schema;
+
+import java.util.Arrays;
+import java.util.Map;
+import java.util.Objects;
+import java.util.TreeMap;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.collect.ImmutableMap;
+import org.apache.commons.lang3.StringUtils;
+
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.repair.autorepair.AutoRepairConfig;
+
+import static java.lang.String.format;
+
+/**
+ * AutoRepair table parameters - used to define the auto-repair configuration for a table.
+ */
+public final class AutoRepairParams
+{
+ public enum Option
+ {
+ FULL_ENABLED,
+ INCREMENTAL_ENABLED,
+ PREVIEW_REPAIRED_ENABLED,
+ PRIORITY;
+
+ @Override
+ public String toString()
+ {
+ return name().toLowerCase();
+ }
+ }
+
+ private final ImmutableMap<String, String> options;
+
+ public static final Map<String, String> DEFAULT_OPTIONS = ImmutableMap.of(
+ Option.FULL_ENABLED.name().toLowerCase(), Boolean.toString(true),
+ Option.INCREMENTAL_ENABLED.name().toLowerCase(), Boolean.toString(true),
+ Option.PREVIEW_REPAIRED_ENABLED.name().toLowerCase(), Boolean.toString(true),
+ Option.PRIORITY.toString(), "0"
+ );
+
+ AutoRepairParams(Map<String, String> options)
+ {
+ this.options = ImmutableMap.copyOf(options);
+ }
+
+ public static final AutoRepairParams DEFAULT =
+ new AutoRepairParams(DEFAULT_OPTIONS);
+
+ public static AutoRepairParams create(Map<String, String> options)
+ {
+ Map<String, String> optionsMap = new TreeMap<>(DEFAULT_OPTIONS);
+ if (options != null)
+ {
+ for (Map.Entry<String, String> entry : options.entrySet())
+ {
+ if (Arrays.stream(Option.values()).noneMatch(option -> option.toString().equalsIgnoreCase(entry.getKey())))
+ {
+ throw new ConfigurationException(format("Unknown property '%s'", entry.getKey()));
+ }
+ optionsMap.put(entry.getKey(), entry.getValue());
+ }
+ }
+ return new AutoRepairParams(optionsMap);
+ }
+
+ public boolean repairEnabled(AutoRepairConfig.RepairType type)
+ {
+ String option = type.toString().toLowerCase() + "_enabled";
+ String enabled = options.getOrDefault(option, DEFAULT_OPTIONS.get(option));
+ return Boolean.parseBoolean(enabled);
+ }
+
+ public int priority()
+ {
+ String priority = options.getOrDefault(Option.PRIORITY.toString(), DEFAULT_OPTIONS.get(Option.PRIORITY.toString()));
+ return Integer.parseInt(priority);
+ }
+
+ public void validate()
+ {
+ for (Option option : Option.values())
+ {
+ if (!options.containsKey(option.toString().toLowerCase()))
+ {
+ throw new ConfigurationException(format("Missing repair sub-option '%s'", option));
+ }
+ }
+ if (options.get(Option.FULL_ENABLED.toString().toLowerCase()) != null && !isValidBoolean(options.get(Option.FULL_ENABLED.toString().toLowerCase())))
+ {
+ throw new ConfigurationException(format("Invalid value %s for '%s' repair sub-option - must be a boolean",
+ options.get(Option.FULL_ENABLED.toString().toLowerCase()),
+ Option.FULL_ENABLED));
+ }
+ if (options.get(Option.INCREMENTAL_ENABLED.toString().toLowerCase()) != null && !isValidBoolean(options.get(Option.INCREMENTAL_ENABLED.toString().toLowerCase())))
+ {
+ throw new ConfigurationException(format("Invalid value %s for '%s' repair sub-option - must be a boolean",
+ options.get(Option.INCREMENTAL_ENABLED.toString().toLowerCase()),
+ Option.INCREMENTAL_ENABLED));
+ }
+ if (options.get(Option.PREVIEW_REPAIRED_ENABLED.toString().toLowerCase()) != null && !isValidBoolean(options.get(Option.PREVIEW_REPAIRED_ENABLED.toString().toLowerCase())))
+ {
+ throw new ConfigurationException(format("Invalid value %s for '%s' repair sub-option - must be a boolean",
+ options.get(Option.PREVIEW_REPAIRED_ENABLED.toString().toLowerCase()),
+ Option.PREVIEW_REPAIRED_ENABLED));
+ }
+ if (options.get(Option.PRIORITY.toString().toLowerCase()) != null && !isValidInt(options.get(Option.PRIORITY.toString().toLowerCase())))
+ {
+ throw new ConfigurationException(format("Invalid value %s for '%s' repair sub-option - must be an integer",
+ options.get(Option.PRIORITY.toString().toLowerCase()),
+ Option.PRIORITY));
+ }
+ }
+
+ public static boolean isValidBoolean(String value)
+ {
+ return StringUtils.equalsIgnoreCase(value, "true") || StringUtils.equalsIgnoreCase(value, "false");
+ }
+
+ public static boolean isValidInt(String value)
+ {
+ return StringUtils.isNumeric(value);
+ }
+
+ public Map<String, String> options()
+ {
+ return options;
+ }
+
+ public static AutoRepairParams fromMap(Map<String, String> map)
+ {
+ return create(map);
+ }
+
+ public Map<String, String> asMap()
+ {
+ return options;
+ }
+
+ @Override
+ public String toString()
+ {
+ return MoreObjects.toStringHelper(this)
+ .add("options", options)
+ .toString();
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o)
+ return true;
+
+ if (!(o instanceof AutoRepairParams))
+ return false;
+
+ AutoRepairParams cp = (AutoRepairParams) o;
+
+ return options.equals(cp.options);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(options);
+ }
+}
diff --git a/src/java/org/apache/cassandra/schema/SchemaConstants.java b/src/java/org/apache/cassandra/schema/SchemaConstants.java
index ff990f7cbb..614ba99092 100644
--- a/src/java/org/apache/cassandra/schema/SchemaConstants.java
+++ b/src/java/org/apache/cassandra/schema/SchemaConstants.java
@@ -186,7 +186,7 @@ public final class SchemaConstants
.addAll(SchemaKeyspaceTables.ALL)
.addAll(TraceKeyspace.TABLE_NAMES)
.addAll(AuthKeyspace.TABLE_NAMES)
- .addAll(SystemDistributedKeyspace.TABLE_NAMES)
+ .addAll(SystemDistributedKeyspace.getTableNames())
.build();
}
}
diff --git a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java
index f7044b55f7..6811a85316 100644
--- a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java
+++ b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java
@@ -96,6 +96,8 @@ public final class SchemaKeyspace
+ "replication frozen<map<text, text>>,"
+ "PRIMARY KEY ((keyspace_name)))");
+ // auto_repair column is only included if AUTOREPAIR_ENABLE is true to avoid schema disagreement
+ // with pre-5.0.8 nodes that don't have this column in their system_schema.tables definition
private static final TableMetadata Tables =
parse(TABLES,
"table definitions",
@@ -125,6 +127,7 @@ public final class SchemaKeyspace
+ "additional_write_policy text,"
+ "cdc boolean,"
+ "read_repair text,"
+ + (CassandraRelevantProperties.AUTOREPAIR_ENABLE.getBoolean() ? "auto_repair frozen<map<text, text>>," : "")
+ "PRIMARY KEY ((keyspace_name), table_name))");
private static final TableMetadata Columns =
@@ -177,6 +180,8 @@ public final class SchemaKeyspace
+ "options frozen<map<text, text>>,"
+ "PRIMARY KEY ((keyspace_name), table_name, trigger_name))");
+ // auto_repair column is only included if AUTOREPAIR_ENABLE is true to avoid schema disagreement
+ // with pre-5.0.8 nodes that don't have this column in their system_schema.views definition
private static final TableMetadata Views =
parse(VIEWS,
"view definitions",
@@ -209,6 +214,7 @@ public final class SchemaKeyspace
+ "additional_write_policy text,"
+ "cdc boolean,"
+ "read_repair text,"
+ + (CassandraRelevantProperties.AUTOREPAIR_ENABLE.getBoolean() ? "auto_repair frozen<map<text, text>>," : "")
+ "PRIMARY KEY ((keyspace_name), view_name))");
private static final TableMetadata Indexes =
@@ -563,7 +569,12 @@ public final class SchemaKeyspace
}
}
- private static void addTableParamsToRowBuilder(TableParams params, Row.SimpleBuilder builder)
+ public static void addTableParamsToRowBuilder(TableParams params, Row.SimpleBuilder builder)
+ {
+ addTableParamsToRowBuilder(params, Tables, builder);
+ }
+
+ private static void addTableParamsToRowBuilder(TableParams params, TableMetadata schemaTable, Row.SimpleBuilder builder)
{
builder.add("bloom_filter_fp_chance", params.bloomFilterFpChance)
.add("comment", params.comment)
@@ -602,6 +613,17 @@ public final class SchemaKeyspace
// incremental_backups is enabled, to avoid RTE in pre-4.2 versioned node during upgrades
if (!params.incrementalBackups)
builder.add("incremental_backups", false);
+
+ // Only add auto_repair column if:
+ // 1. The column exists in the schema (depends on AUTOREPAIR_ENABLE at class load time)
+ // 2. The scheduler is enabled (which includes AUTOREPAIR_ENABLE check)
+ // to avoid RTE in pre-5.1 versioned node during upgrades
+ if (schemaTable.getColumn(ByteBufferUtil.bytes("auto_repair")) != null
+ && DatabaseDescriptor.getRawConfig() != null
+ && DatabaseDescriptor.getAutoRepairConfig().isAutoRepairSchedulingEnabled())
+ {
+ builder.add("auto_repair", params.autoRepair.asMap());
+ }
}
private static void addAlterTableToSchemaMutation(TableMetadata oldTable, TableMetadata newTable, Mutation.SimpleBuilder builder)
@@ -814,7 +836,7 @@ public final class SchemaKeyspace
.add("where_clause", view.whereClause.toCQLString())
.add("id", table.id.asUUID());
- addTableParamsToRowBuilder(table.params, rowBuilder);
+ addTableParamsToRowBuilder(table.params, Views, rowBuilder);
if (includeColumns)
{
@@ -1070,6 +1092,12 @@ public final class SchemaKeyspace
if (row.has("incremental_backups"))
builder.incrementalBackups(row.getBoolean("incremental_backups"));
+ // auto_repair column was introduced in 5.0.8
+ if (row.has("auto_repair"))
+ {
+ builder.automatedRepair(AutoRepairParams.fromMap(row.getFrozenTextMap("auto_repair")));
+ }
+
return builder.build();
}
diff --git a/src/java/org/apache/cassandra/schema/SystemDistributedKeyspace.java b/src/java/org/apache/cassandra/schema/SystemDistributedKeyspace.java
index b36ec64874..b44278ffae 100644
--- a/src/java/org/apache/cassandra/schema/SystemDistributedKeyspace.java
+++ b/src/java/org/apache/cassandra/schema/SystemDistributedKeyspace.java
@@ -83,8 +83,9 @@ public final class SystemDistributedKeyspace
* gen 4: compression chunk length reduced to 16KiB, memtable_flush_period_in_ms now unset on all tables in 4.0
* gen 5: add ttl and TWCS to repair_history tables
* gen 6: add denylist table
+ * gen 7: add auto_repair_history and auto_repair_priority tables for AutoRepair feature
*/
- public static final long GENERATION = 6;
+ public static final long GENERATION = CassandraRelevantProperties.AUTOREPAIR_ENABLE.getBoolean() ? 7 : 6;
public static final String REPAIR_HISTORY = "repair_history";
@@ -94,8 +95,28 @@ public final class SystemDistributedKeyspace
public static final String PARTITION_DENYLIST_TABLE = "partition_denylist";
- public static final Set<String> TABLE_NAMES = ImmutableSet.of(REPAIR_HISTORY, PARENT_REPAIR_HISTORY, VIEW_BUILD_STATUS, PARTITION_DENYLIST_TABLE);
-
+ public static final String AUTO_REPAIR_HISTORY = "auto_repair_history";
+
+ public static final String AUTO_REPAIR_PRIORITY = "auto_repair_priority";
+
+ private static final Set<String> TABLE_NAMES = ImmutableSet.of(REPAIR_HISTORY, PARENT_REPAIR_HISTORY,
+ VIEW_BUILD_STATUS, PARTITION_DENYLIST_TABLE);
+ private static final Set<String> TABLE_NAMES_WITH_AUTO_REPAIR = ImmutableSet.of(REPAIR_HISTORY, PARENT_REPAIR_HISTORY,
+ VIEW_BUILD_STATUS, PARTITION_DENYLIST_TABLE,
+ AUTO_REPAIR_HISTORY, AUTO_REPAIR_PRIORITY);
+
+ /**
+ * Returns the set of table names for the system_distributed keyspace.
+ * When AUTOREPAIR_ENABLE is false, auto-repair tables are not included.
+ */
+ public static Set<String> getTableNames()
+ {
+ return CassandraRelevantProperties.AUTOREPAIR_ENABLE.getBoolean()
+ ? TABLE_NAMES_WITH_AUTO_REPAIR
+ : TABLE_NAMES;
+ }
+
+
private static final TableMetadata RepairHistory =
parse(REPAIR_HISTORY,
"Repair history",
@@ -161,6 +182,27 @@ public final class SystemDistributedKeyspace
+ "PRIMARY KEY ((ks_name, table_name), key))")
.build();
+ public static final TableMetadata AutoRepairHistory =
+ parse(AUTO_REPAIR_HISTORY,
+ "Auto repair history for each node",
+ "CREATE TABLE %s ("
+ + "host_id uuid,"
+ + "repair_type text,"
+ + "repair_turn text,"
+ + "repair_start_ts timestamp,"
+ + "repair_finish_ts timestamp,"
+ + "delete_hosts set<uuid>,"
+ + "delete_hosts_update_time timestamp,"
+ + "force_repair boolean,"
+ + "PRIMARY KEY (repair_type, host_id))").build();
+ public static final TableMetadata AutoRepairPriority =
+ parse(AUTO_REPAIR_PRIORITY,
+ "Auto repair priority for each group",
+ "CREATE TABLE %s ("
+ + "repair_type text,"
+ + "repair_priority set<uuid>,"
+ + "PRIMARY KEY (repair_type))").build();
+
private static TableMetadata.Builder parse(String table, String description, String cql)
{
return CreateTableStatement.parse(format(cql, table), SchemaConstants.DISTRIBUTED_KEYSPACE_NAME)
@@ -170,7 +212,20 @@ public final class SystemDistributedKeyspace
public static KeyspaceMetadata metadata()
{
- return KeyspaceMetadata.create(SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, KeyspaceParams.simple(Math.max(DEFAULT_RF, DatabaseDescriptor.getDefaultKeyspaceRF())), Tables.of(RepairHistory, ParentRepairHistory, ViewBuildStatus, PartitionDenylistTable));
+ Tables tables;
+ if (CassandraRelevantProperties.AUTOREPAIR_ENABLE.getBoolean())
+ {
+ tables = Tables.of(RepairHistory, ParentRepairHistory, ViewBuildStatus,
+ PartitionDenylistTable, AutoRepairHistory, AutoRepairPriority);
+ }
+ else
+ {
+ tables = Tables.of(RepairHistory, ParentRepairHistory, ViewBuildStatus,
+ PartitionDenylistTable);
+ }
+ return KeyspaceMetadata.create(SchemaConstants.DISTRIBUTED_KEYSPACE_NAME,
+ KeyspaceParams.simple(Math.max(DEFAULT_RF, DatabaseDescriptor.getDefaultKeyspaceRF())),
+ tables);
}
public static void startParentRepair(TimeUUID parent_id, String keyspaceName, String[] cfnames, RepairOption options)
diff --git a/src/java/org/apache/cassandra/schema/TableParams.java b/src/java/org/apache/cassandra/schema/TableParams.java
index 8f883f8f47..9b39d662f2 100644
--- a/src/java/org/apache/cassandra/schema/TableParams.java
+++ b/src/java/org/apache/cassandra/schema/TableParams.java
@@ -25,6 +25,7 @@ import com.google.common.base.MoreObjects;
import com.google.common.base.Objects;
import com.google.common.collect.ImmutableMap;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.Attributes;
import org.apache.cassandra.cql3.CqlBuilder;
import org.apache.cassandra.exceptions.ConfigurationException;
@@ -60,7 +61,8 @@ public final class TableParams
ADDITIONAL_WRITE_POLICY,
CRC_CHECK_CHANCE,
CDC,
- READ_REPAIR;
+ READ_REPAIR,
+ AUTO_REPAIR;
@Override
public String toString()
@@ -88,6 +90,7 @@ public final class TableParams
public final ImmutableMap<String, ByteBuffer> extensions;
public final boolean cdc;
public final ReadRepairStrategy readRepair;
+ public final AutoRepairParams autoRepair;
private TableParams(Builder builder)
{
@@ -112,6 +115,7 @@ public final class TableParams
extensions = builder.extensions;
cdc = builder.cdc;
readRepair = builder.readRepair;
+ autoRepair = builder.autoRepair;
}
public static Builder builder()
@@ -139,7 +143,8 @@ public final class TableParams
.additionalWritePolicy(params.additionalWritePolicy)
.extensions(params.extensions)
.cdc(params.cdc)
- .readRepair(params.readRepair);
+ .readRepair(params.readRepair)
+ .automatedRepair(params.autoRepair);
}
public Builder unbuild()
@@ -153,7 +158,7 @@ public final class TableParams
compression.validate();
double minBloomFilterFpChanceValue = BloomCalculations.minSupportedBloomFilterFpChance();
- if (bloomFilterFpChance <= minBloomFilterFpChanceValue || bloomFilterFpChance > 1)
+ if (bloomFilterFpChance <= minBloomFilterFpChanceValue || bloomFilterFpChance > 1)
{
fail("%s must be larger than %s and less than or equal to 1.0 (got %s)",
BLOOM_FILTER_FP_CHANCE,
@@ -194,6 +199,8 @@ public final class TableParams
if (cdc && memtable.factory().writesShouldSkipCommitLog())
fail("CDC cannot work if writes skip the commit log. Check your memtable configuration.");
+
+ autoRepair.validate();
}
private static void fail(String format, Object... args)
@@ -217,7 +224,7 @@ public final class TableParams
&& allowAutoSnapshot == p.allowAutoSnapshot
&& bloomFilterFpChance == p.bloomFilterFpChance
&& crcCheckChance == p.crcCheckChance
- && gcGraceSeconds == p.gcGraceSeconds
+ && gcGraceSeconds == p.gcGraceSeconds
&& incrementalBackups == p.incrementalBackups
&& defaultTimeToLive == p.defaultTimeToLive
&& memtableFlushPeriodInMs == p.memtableFlushPeriodInMs
@@ -230,7 +237,8 @@ public final class TableParams
&& memtable.equals(p.memtable)
&& extensions.equals(p.extensions)
&& cdc == p.cdc
- && readRepair == p.readRepair;
+ && readRepair == p.readRepair
+ && autoRepair.equals(p.autoRepair);
}
@Override
@@ -254,7 +262,8 @@ public final class TableParams
memtable,
extensions,
cdc,
- readRepair);
+ readRepair,
+ autoRepair);
}
@Override
@@ -280,6 +289,7 @@ public final class TableParams
.add(EXTENSIONS.toString(), extensions)
.add(CDC.toString(), cdc)
.add(READ_REPAIR.toString(), readRepair)
+ .add(AUTO_REPAIR.toString(), autoRepair)
.toString();
}
@@ -332,6 +342,13 @@ public final class TableParams
.append("AND read_repair = ").appendWithSingleQuotes(readRepair.toString())
.newLine()
.append("AND speculative_retry = ").appendWithSingleQuotes(speculativeRetry.toString());
+
+ if (DatabaseDescriptor.getRawConfig() != null
+ && DatabaseDescriptor.getAutoRepairConfig().isAutoRepairSchedulingEnabled())
+ {
+ builder.newLine()
+ .append("AND auto_repair = ").append(autoRepair.asMap());
+ }
}
public static final class Builder
@@ -356,6 +373,7 @@ public final class TableParams
private boolean cdc;
private ReadRepairStrategy readRepair = ReadRepairStrategy.BLOCKING;
+ private AutoRepairParams autoRepair = AutoRepairParams.DEFAULT;
public Builder()
{
}
@@ -478,5 +496,11 @@ public final class TableParams
extensions = ImmutableMap.copyOf(val);
return this;
}
+
+ public Builder automatedRepair(AutoRepairParams val)
+ {
+ autoRepair = val;
+ return this;
+ }
}
}
diff --git a/src/java/org/apache/cassandra/service/ActiveRepairService.java b/src/java/org/apache/cassandra/service/ActiveRepairService.java
index e120122c08..5c8ca677c0 100644
--- a/src/java/org/apache/cassandra/service/ActiveRepairService.java
+++ b/src/java/org/apache/cassandra/service/ActiveRepairService.java
@@ -41,6 +41,9 @@ import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.cassandra.concurrent.ExecutorPlus;
import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DurationSpec;
@@ -50,6 +53,7 @@ import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.locator.AbstractReplicationStrategy;
import org.apache.cassandra.locator.EndpointsByRange;
import org.apache.cassandra.locator.EndpointsForRange;
+import org.apache.cassandra.service.disk.usage.DiskUsageMonitor;
import org.apache.cassandra.utils.ExecutorUtils;
import org.apache.cassandra.repair.state.CoordinatorState;
import org.apache.cassandra.repair.state.ParticipateState;
@@ -61,8 +65,6 @@ import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.utils.concurrent.AsyncPromise;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
@@ -645,8 +647,26 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
return true;
}
+ public boolean verifyDiskHeadroomThreshold(TimeUUID parentRepairSession, PreviewKind previewKind)
+ {
+ double diskUsage = DiskUsageMonitor.instance.getDiskUsage();
+ double rejectRatio = getIncrementalRepairDiskHeadroomRejectRatio();
+
+ if (diskUsage + rejectRatio > 1)
+ {
+ logger.error("[{}] Rejecting incoming repair, disk usage ({}%) above threshold ({}%)",
+ previewKind.logPrefix(parentRepairSession), String.format("%.2f", diskUsage * 100), String.format("%.2f", (1 - rejectRatio) * 100));
+ return false;
+ }
+
+ return true;
+ }
+
public Future<?> prepareForRepair(TimeUUID parentRepairSession, InetAddressAndPort coordinator, Set<InetAddressAndPort> endpoints, RepairOption options, boolean isForcedRepair, List<ColumnFamilyStore> columnFamilyStores)
{
+ if (!verifyDiskHeadroomThreshold(parentRepairSession, options.getPreviewKind()))
+ failRepair(parentRepairSession, "Rejecting incoming repair, disk usage above threshold"); // failRepair throws exception
+
if (!verifyCompactionsPendingThreshold(parentRepairSession, options.getPreviewKind()))
failRepair(parentRepairSession, "Rejecting incoming repair, pending compactions above threshold"); // failRepair throws exception
@@ -1059,6 +1079,16 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
DatabaseDescriptor.setRepairPendingCompactionRejectThreshold(value);
}
+ public double getIncrementalRepairDiskHeadroomRejectRatio()
+ {
+ return DatabaseDescriptor.getRepairDiskHeadroomRejectRatio();
+ }
+
+ public void setIncrementalRepairDiskHeadroomRejectRatio(double value)
+ {
+ DatabaseDescriptor.setRepairDiskHeadroomRejectRatio(value);
+ }
+
/**
* Remove any parent repair sessions matching predicate
*/
diff --git a/src/java/org/apache/cassandra/service/ActiveRepairServiceMBean.java b/src/java/org/apache/cassandra/service/ActiveRepairServiceMBean.java
index 851dc6c802..c739b048d6 100644
--- a/src/java/org/apache/cassandra/service/ActiveRepairServiceMBean.java
+++ b/src/java/org/apache/cassandra/service/ActiveRepairServiceMBean.java
@@ -74,4 +74,8 @@ public interface ActiveRepairServiceMBean
int parentRepairSessionsCount();
public int getPaxosRepairParallelism();
public void setPaxosRepairParallelism(int v);
+
+ public double getIncrementalRepairDiskHeadroomRejectRatio();
+
+ public void setIncrementalRepairDiskHeadroomRejectRatio(double value);
}
diff --git a/src/java/org/apache/cassandra/service/AutoRepairService.java b/src/java/org/apache/cassandra/service/AutoRepairService.java
new file mode 100644
index 0000000000..52cd9b3940
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/AutoRepairService.java
@@ -0,0 +1,349 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.service;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.ParameterizedClass;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.repair.autorepair.AutoRepairConfig;
+import org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType;
+import org.apache.cassandra.repair.autorepair.AutoRepairUtils;
+import org.apache.cassandra.utils.MBeanWrapper;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Implement all the MBeans for AutoRepair.
+ */
+public class AutoRepairService implements AutoRepairServiceMBean
+{
+ private static final Logger logger = LoggerFactory.getLogger(AutoRepairService.class);
+
+ public static final String MBEAN_NAME = "org.apache.cassandra.db:type=AutoRepairService";
+
+ @VisibleForTesting
+ protected AutoRepairConfig config;
+
+ public static final AutoRepairService instance = new AutoRepairService();
+
+ @VisibleForTesting
+ protected AutoRepairService()
+ {
+ }
+
+ public static void setup()
+ {
+ if (!CassandraRelevantProperties.AUTOREPAIR_ENABLE.getBoolean())
+ {
+ logger.info("Auto-repair service is disabled via JVM property");
+ return;
+ }
+ instance.config = DatabaseDescriptor.getAutoRepairConfig();
+ }
+
+ static
+ {
+ if (CassandraRelevantProperties.AUTOREPAIR_ENABLE.getBoolean())
+ {
+ MBeanWrapper.instance.registerMBean(instance, MBEAN_NAME);
+ }
+ }
+
+ public void checkCanRun(String repairType)
+ {
+ checkCanRun(RepairType.parse(repairType));
+ }
+
+ public void checkCanRun(RepairType repairType)
+ {
+ if (!config.isAutoRepairSchedulingEnabled())
+ throw new ConfigurationException("Auto-repair scheduler is disabled.");
+
+ if (repairType != RepairType.INCREMENTAL)
+ return;
+
+ if (config.getMaterializedViewRepairEnabled(repairType) && DatabaseDescriptor.isMaterializedViewsOnRepairEnabled())
+ throw new ConfigurationException("Cannot run incremental repair while materialized view replay is enabled. Set materialized_views_on_repair_enabled to false.");
+
+ if (DatabaseDescriptor.isCDCEnabled() && DatabaseDescriptor.isCDCOnRepairEnabled())
+ throw new ConfigurationException("Cannot run incremental repair while CDC replay is enabled. Set cdc_on_repair_enabled to false.");
+ }
+
+ public AutoRepairConfig getAutoRepairConfig()
+ {
+ return config;
+ }
+
+ @Override
+ public boolean isAutoRepairDisabled()
+ {
+ return !CassandraRelevantProperties.AUTOREPAIR_ENABLE.getBoolean()
+ || config == null
+ || !config.isAutoRepairSchedulingEnabled();
+ }
+
+ @Override
+ public String getAutoRepairConfiguration()
+ {
+ StringBuilder sb = new StringBuilder();
+ sb.append("repair scheduler configuration:");
+ appendConfig(sb, "repair_check_interval", config.getRepairCheckInterval());
+ appendConfig(sb, "repair_task_min_duration", config.getRepairTaskMinDuration());
+ appendConfig(sb, "history_clear_delete_hosts_buffer_interval", config.getAutoRepairHistoryClearDeleteHostsBufferInterval());
+ appendConfig(sb, "mixed_major_version_repair_enabled", config.getMixedMajorVersionRepairEnabled());
+ for (RepairType repairType : RepairType.values())
+ {
+ sb.append(formatRepairTypeConfig(repairType, config));
+ }
+ return sb.toString();
+ }
+
+ @Override
+ public void setAutoRepairEnabled(String repairType, boolean enabled)
+ {
+ checkCanRun(repairType);
+ config.setAutoRepairEnabled(RepairType.parse(repairType), enabled);
+ }
+
+ @Override
+ public void setRepairThreads(String repairType, int repairThreads)
+ {
+ config.setRepairThreads(RepairType.parse(repairType), repairThreads);
+ }
+
+ @Override
+ public void setRepairPriorityForHosts(String repairType, String commaSeparatedHostSet)
+ {
+ Set<InetAddressAndPort> hosts = InetAddressAndPort.parseHosts(commaSeparatedHostSet, false);
+ if (!hosts.isEmpty())
+ {
+ AutoRepairUtils.addPriorityHosts(RepairType.parse(repairType), hosts);
+ }
+ }
+
+ @Override
+ public void setForceRepairForHosts(String repairType, String commaSeparatedHostSet)
+ {
+ Set<InetAddressAndPort> hosts = InetAddressAndPort.parseHosts(commaSeparatedHostSet, false);
+ if (!hosts.isEmpty())
+ {
+ AutoRepairUtils.setForceRepair(RepairType.parse(repairType), hosts);
+ }
+ }
+
+ @Override
+ public void setRepairMinInterval(String repairType, String minRepairInterval)
+ {
+ config.setRepairMinInterval(RepairType.parse(repairType), minRepairInterval);
+ }
+
+ @Override
+ public void startScheduler()
+ {
+ config.startScheduler();
+ }
+
+ @Override
+ public void setAutoRepairHistoryClearDeleteHostsBufferDuration(String duration)
+ {
+ config.setAutoRepairHistoryClearDeleteHostsBufferInterval(duration);
+ }
+
+ @Override
+ public void setAutoRepairMinRepairTaskDuration(String duration)
+ {
+ config.setRepairTaskMinDuration(duration);
+ }
+
+ @Override
+ public void setRepairSSTableCountHigherThreshold(String repairType, int sstableHigherThreshold)
+ {
+ config.setRepairSSTableCountHigherThreshold(RepairType.parse(repairType), sstableHigherThreshold);
+ }
+
+ @Override
+ public void setAutoRepairTableMaxRepairTime(String repairType, String autoRepairTableMaxRepairTime)
+ {
+ config.setAutoRepairTableMaxRepairTime(RepairType.parse(repairType), autoRepairTableMaxRepairTime);
+ }
+
+ @Override
+ public void setIgnoreDCs(String repairType, Set<String> ignoreDCs)
+ {
+ config.setIgnoreDCs(RepairType.parse(repairType), ignoreDCs);
+ }
+
+ @Override
+ public void setPrimaryTokenRangeOnly(String repairType, boolean primaryTokenRangeOnly)
+ {
+ config.setRepairPrimaryTokenRangeOnly(RepairType.parse(repairType), primaryTokenRangeOnly);
+ }
+
+ @Override
+ public void setParallelRepairPercentage(String repairType, int percentage)
+ {
+ config.setParallelRepairPercentage(RepairType.parse(repairType), percentage);
+ }
+
+ @Override
+ public void setParallelRepairCount(String repairType, int count)
+ {
+ config.setParallelRepairCount(RepairType.parse(repairType), count);
+ }
+
+ @Override
+ public void setAllowParallelReplicaRepair(String repairType, boolean enabled)
+ {
+ config.setAllowParallelReplicaRepair(RepairType.parse(repairType), enabled);
+ }
+
+ @Override
+ public void setAllowParallelReplicaRepairAcrossSchedules(String repairType, boolean enabled)
+ {
+ config.setAllowParallelReplicaRepairAcrossSchedules(RepairType.parse(repairType), enabled);
+ }
+
+ @Override
+ public void setMVRepairEnabled(String repairType, boolean enabled)
+ {
+ config.setMaterializedViewRepairEnabled(RepairType.parse(repairType), enabled);
+ }
+
+ @Override
+ public void setRepairSessionTimeout(String repairType, String timeout)
+ {
+ config.setRepairSessionTimeout(RepairType.parse(repairType), timeout);
+ }
+
+ @Override
+ public Set<String> getOnGoingRepairHostIds(String repairType)
+ {
+ List<AutoRepairUtils.AutoRepairHistory> histories = AutoRepairUtils.getAutoRepairHistory(RepairType.parse(repairType));
+ if (histories == null)
+ {
+ return Collections.emptySet();
+ }
+ Set<String> hostIds = new HashSet<>();
+ AutoRepairUtils.CurrentRepairStatus currentRepairStatus = new AutoRepairUtils.CurrentRepairStatus(histories, AutoRepairUtils.getPriorityHostIds(RepairType.parse(repairType)), null);
+ for (UUID id : currentRepairStatus.hostIdsWithOnGoingRepair)
+ {
+ hostIds.add(id.toString());
+ }
+ for (UUID id : currentRepairStatus.hostIdsWithOnGoingForceRepair)
+ {
+ hostIds.add(id.toString());
+ }
+ return Collections.unmodifiableSet(hostIds);
+ }
+
+ @Override
+ public void setAutoRepairTokenRangeSplitterParameter(String repairType, String key, String value)
+ {
+ config.getTokenRangeSplitterInstance(RepairType.parse(repairType)).setParameter(key, value);
+ }
+
+ @Override
+ public void setRepairByKeyspace(String repairType, boolean repairByKeyspace)
+ {
+ config.setRepairByKeyspace(RepairType.parse(repairType), repairByKeyspace);
+ }
+
+ @Override
+ public void setAutoRepairMaxRetriesCount(String repairType, int retries)
+ {
+ config.setRepairMaxRetries(RepairType.parse(repairType), retries);
+ }
+
+ @Override
+ public void setAutoRepairRetryBackoff(String repairType, String interval)
+ {
+ config.setRepairRetryBackoff(RepairType.parse(repairType), interval);
+ }
+
+ @Override
+ public void setMixedMajorVersionRepairEnabled(boolean enabled)
+ {
+ config.setMixedMajorVersionRepairEnabled(enabled);
+ }
+
+ private String formatRepairTypeConfig(RepairType repairType, AutoRepairConfig config)
+ {
+ StringBuilder sb = new StringBuilder();
+ sb.append("\nconfiguration for repair_type: ").append(repairType.getConfigName());
+ sb.append("\n\tenabled: ").append(config.isAutoRepairEnabled(repairType));
+ // Only show configuration if enabled
+ if (config.isAutoRepairEnabled(repairType))
+ {
+ Set<InetAddressAndPort> priorityHosts = AutoRepairUtils.getPriorityHosts(repairType);
+ if (!priorityHosts.isEmpty())
+ {
+ appendConfig(sb, "priority_hosts", Joiner.on(',').skipNulls().join(priorityHosts));
+ }
+
+ appendConfig(sb, "min_repair_interval", config.getRepairMinInterval(repairType));
+ appendConfig(sb, "repair_by_keyspace", config.getRepairByKeyspace(repairType));
+ appendConfig(sb, "number_of_repair_threads", config.getRepairThreads(repairType));
+ appendConfig(sb, "sstable_upper_threshold", config.getRepairSSTableCountHigherThreshold(repairType));
+ appendConfig(sb, "table_max_repair_time", config.getAutoRepairTableMaxRepairTime(repairType));
+ appendConfig(sb, "ignore_dcs", config.getIgnoreDCs(repairType));
+ appendConfig(sb, "repair_primary_token_range_only", config.getRepairPrimaryTokenRangeOnly(repairType));
+ appendConfig(sb, "parallel_repair_count", config.getParallelRepairCount(repairType));
+ appendConfig(sb, "parallel_repair_percentage", config.getParallelRepairPercentage(repairType));
+ appendConfig(sb, "allow_parallel_replica_repair", config.getAllowParallelReplicaRepair(repairType));
+ appendConfig(sb, "allow_parallel_replica_repair_across_schedules", config.getAllowParallelReplicaRepairAcrossSchedules(repairType));
+ appendConfig(sb, "materialized_view_repair_enabled", config.getMaterializedViewRepairEnabled(repairType));
+ appendConfig(sb, "initial_scheduler_delay", config.getInitialSchedulerDelay(repairType));
+ appendConfig(sb, "repair_session_timeout", config.getRepairSessionTimeout(repairType));
+ appendConfig(sb, "force_repair_new_node", config.getForceRepairNewNode(repairType));
+ appendConfig(sb, "repair_max_retries", config.getRepairMaxRetries(repairType));
+ appendConfig(sb, "repair_retry_backoff", config.getRepairRetryBackoff(repairType));
+
+ final ParameterizedClass splitterClass = config.getTokenRangeSplitter(repairType);
+ final String splitterClassName = splitterClass.class_name != null ? splitterClass.class_name : AutoRepairConfig.DEFAULT_SPLITTER.getName();
+ appendConfig(sb, "token_range_splitter", splitterClassName);
+ Map<String, String> tokenRangeSplitterParameters = config.getTokenRangeSplitterInstance(repairType).getParameters();
+ if (!tokenRangeSplitterParameters.isEmpty())
+ {
+ for (Map.Entry<String, String> param : tokenRangeSplitterParameters.entrySet())
+ {
+ appendConfig(sb, String.format("token_range_splitter.%s", param.getKey()), param.getValue());
+ }
+ }
+ }
+
+ return sb.toString();
+ }
+
+ private <T> void appendConfig(StringBuilder sb, String config, T value)
+ {
+ sb.append(String.format("%s%s: %s", "\n\t", config, value));
+ }
+}
diff --git a/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java b/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java
new file mode 100644
index 0000000000..e4d554dd98
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.service;
+
+
+import java.util.Set;
+
+/**
+ * Defines all the MBeans exposed for AutoRepair.
+ */
+public interface AutoRepairServiceMBean
+{
+ public void setAutoRepairEnabled(String repairType, boolean enabled);
+
+ public void setRepairThreads(String repairType, int repairThreads);
+
+ public void setRepairPriorityForHosts(String repairType, String commaSeparatedHostSet);
+
+ public void setForceRepairForHosts(String repairType, String commaSeparatedHostSet);
+
+ public void setRepairMinInterval(String repairType, String minRepairInterval);
+
+ void startScheduler();
+
+ public void setAutoRepairHistoryClearDeleteHostsBufferDuration(String duration);
+
+ public void setAutoRepairMinRepairTaskDuration(String duration);
+
+ public void setRepairSSTableCountHigherThreshold(String repairType, int ssTableHigherThreshold);
+
+ public void setAutoRepairTableMaxRepairTime(String repairType, String autoRepairTableMaxRepairTime);
+
+ public void setIgnoreDCs(String repairType, Set<String> ignorDCs);
+
+ public void setPrimaryTokenRangeOnly(String repairType, boolean primaryTokenRangeOnly);
+
+ public void setParallelRepairPercentage(String repairType, int percentage);
+
+ public void setParallelRepairCount(String repairType, int count);
+
+ public void setAllowParallelReplicaRepair(String repairType, boolean enabled);
+
+ public void setAllowParallelReplicaRepairAcrossSchedules(String repairType, boolean enabled);
+
+ public void setMVRepairEnabled(String repairType, boolean enabled);
+
+ public boolean isAutoRepairDisabled();
+
+ public String getAutoRepairConfiguration();
+
+ public void setRepairSessionTimeout(String repairType, String timeout);
+
+ public Set<String> getOnGoingRepairHostIds(String repairType);
+
+ public void setAutoRepairTokenRangeSplitterParameter(String repairType, String key, String value);
+
+ public void setRepairByKeyspace(String repairType, boolean repairByKeyspace);
+
+ public void setAutoRepairMaxRetriesCount(String repairType, int retries);
+
+ public void setAutoRepairRetryBackoff(String repairType, String interval);
+
+ public void setMixedMajorVersionRepairEnabled(boolean enabled);
+}
diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java
index 0f178b00f0..22ef542f2a 100644
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@ -88,6 +88,10 @@ import com.google.common.collect.Sets;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.RateLimiter;
import com.google.common.util.concurrent.Uninterruptibles;
+
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.repair.autorepair.AutoRepairConfig;
+import org.apache.cassandra.repair.autorepair.AutoRepair;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -183,6 +187,7 @@ import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.repair.RepairCoordinator;
import org.apache.cassandra.repair.SharedContext;
+import org.apache.cassandra.repair.autorepair.AutoRepairUtils;
import org.apache.cassandra.repair.messages.RepairOption;
import org.apache.cassandra.schema.CompactionParams.TombstoneOption;
import org.apache.cassandra.schema.KeyspaceMetadata;
@@ -267,6 +272,7 @@ import static org.apache.cassandra.index.SecondaryIndexManager.getIndexName;
import static org.apache.cassandra.index.SecondaryIndexManager.isIndexColumnFamily;
import static org.apache.cassandra.net.NoPayload.noPayload;
import static org.apache.cassandra.net.Verb.REPLICATION_DONE_REQ;
+import static org.apache.cassandra.locator.InetAddressAndPort.stringify;
import static org.apache.cassandra.service.ActiveRepairService.ParentRepairStatus;
import static org.apache.cassandra.service.ActiveRepairService.repairCommandExecutor;
import static org.apache.cassandra.service.StorageService.Mode.DECOMMISSIONED;
@@ -408,7 +414,12 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
public RangesAtEndpoint getReplicas(String keyspaceName, InetAddressAndPort endpoint)
{
- return Keyspace.open(keyspaceName).getReplicationStrategy().getAddressReplicas(endpoint);
+ return getReplicas(Keyspace.open(keyspaceName).getReplicationStrategy(), endpoint);
+ }
+
+ public RangesAtEndpoint getReplicas(AbstractReplicationStrategy replicationStrategy, InetAddressAndPort endpoint)
+ {
+ return replicationStrategy.getAddressReplicas(tokenMetadata.cloneOnlyTokenMap(), endpoint);
}
public List<Range<Token>> getLocalRanges(String ks)
@@ -482,7 +493,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
/* Used for tracking drain progress */
private volatile int totalCFs, remainingCFs;
- private static final AtomicInteger nextRepairCommand = new AtomicInteger();
+ public static final AtomicInteger nextRepairCommand = new AtomicInteger();
private final List<IEndpointLifecycleSubscriber> lifecycleSubscribers = new CopyOnWriteArrayList<>();
@@ -1313,6 +1324,14 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
if (dataAvailable)
{
finishJoiningRing(shouldBootstrap, bootstrapTokens);
+ AutoRepairConfig repairConfig = DatabaseDescriptor.getAutoRepairConfig();
+ // this node might have just bootstrapped; check if we should run repair immediately
+ if (shouldBootstrap && repairConfig.isAutoRepairSchedulingEnabled())
+ {
+ for (AutoRepairConfig.RepairType rType : AutoRepairConfig.RepairType.values())
+ if (repairConfig.isAutoRepairEnabled(rType) && repairConfig.getForceRepairNewNode(rType))
+ AutoRepairUtils.setForceRepairNewNode(rType);
+ }
// remove the existing info about the replaced node.
if (!current.isEmpty())
{
@@ -1415,6 +1434,8 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
setTokens(tokens);
assert tokenMetadata.sortedTokens().size() > 0;
+
+ doAutoRepairSetup();
}
@VisibleForTesting
@@ -1438,6 +1459,24 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
}
}
+ public void doAutoRepairSetup()
+ {
+ if (!CassandraRelevantProperties.AUTOREPAIR_ENABLE.getBoolean())
+ {
+ logger.info("Auto-repair service is disabled via JVM property, skipping setup");
+ return;
+ }
+
+ AutoRepairService.setup();
+ if (DatabaseDescriptor.getAutoRepairConfig().isAutoRepairSchedulingEnabled())
+ {
+ logger.info("Enabling auto-repair scheduling");
+ AutoRepair.instance.setup();
+ logger.info("AutoRepair setup complete!");
+ }
+ }
+
+
public boolean isAuthSetupComplete()
{
return authSetupComplete;
@@ -4102,16 +4141,6 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
return FileUtils.getCanonicalPath(DatabaseDescriptor.getSavedCachesLocation());
}
- private List<String> stringify(Iterable<InetAddressAndPort> endpoints, boolean withPort)
- {
- List<String> stringEndpoints = new ArrayList<>();
- for (InetAddressAndPort ep : endpoints)
- {
- stringEndpoints.add(ep.getHostAddress(withPort));
- }
- return stringEndpoints;
- }
-
public int getCurrentGenerationNumber()
{
return Gossiper.instance.getCurrentGenerationNumber(FBUtilities.getBroadcastAddressAndPort());
@@ -5935,6 +5964,8 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
CommitLog.instance.shutdownBlocking();
+ AutoRepair.instance.shutdownBlocking();
+
// wait for miscellaneous tasks like sstable and commitlog segment deletion
ColumnFamilyStore.shutdownPostFlushExecutor();
@@ -7698,4 +7729,41 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
{
return DatabaseDescriptor.getPaxosRepairRaceWait();
}
+
+ @Override
+ public List<String> getTablesForKeyspace(String keyspace)
+ {
+ return Keyspace.open(keyspace).getColumnFamilyStores().stream().map(cfs -> cfs.name).collect(Collectors.toList());
+ }
+
+ @Override
+ public List<String> mutateSSTableRepairedState(boolean repaired, boolean preview, String keyspace, List<String> tableNames)
+ {
+ Map<String, ColumnFamilyStore> tables = Keyspace.open(keyspace).getColumnFamilyStores()
+ .stream().collect(Collectors.toMap(c -> c.name, c -> c));
+ for (String tableName : tableNames)
+ {
+ if (!tables.containsKey(tableName))
+ throw new RuntimeException("Table " + tableName + " does not exist in keyspace " + keyspace);
+ }
+
+ // only select SSTables that are unrepaired when repaired is true and vice versa
+ Predicate<SSTableReader> predicate = sst -> repaired != sst.isRepaired();
+
+ // mutate SSTables
+ long repairedAt = !repaired ? 0 : currentTimeMillis();
+ List<String> sstablesTouched = new ArrayList<>();
+ for (String tableName : tableNames)
+ {
+ ColumnFamilyStore table = tables.get(tableName);
+ Set<SSTableReader> result = table.runWithCompactionsDisabled(() -> {
+ Set<SSTableReader> sstables = table.getLiveSSTables().stream().filter(predicate).collect(Collectors.toSet());
+ if (!preview)
+ table.getCompactionStrategyManager().mutateRepaired(sstables, repairedAt, null, false);
+ return sstables;
+ }, predicate, OperationType.ANTICOMPACTION, true, false, true);
+ sstablesTouched.addAll(result.stream().map(sst -> sst.descriptor.baseFile().name()).collect(Collectors.toList()));
+ }
+ return sstablesTouched;
+ }
}
diff --git a/src/java/org/apache/cassandra/service/StorageServiceMBean.java b/src/java/org/apache/cassandra/service/StorageServiceMBean.java
index 875df36942..43edcceb36 100644
--- a/src/java/org/apache/cassandra/service/StorageServiceMBean.java
+++ b/src/java/org/apache/cassandra/service/StorageServiceMBean.java
@@ -1330,4 +1330,10 @@ public interface StorageServiceMBean extends NotificationEmitter
boolean getPaxosRepairRaceWait();
public void dropPreparedStatements(boolean memoryOnly);
+
+ /** Gets the names of all tables for the given keyspace */
+ public List<String> getTablesForKeyspace(String keyspace);
+
+ /** Mutates the repaired state of all SSTables for the given SSTables */
+ public List<String> mutateSSTableRepairedState(boolean repaired, boolean preview, String keyspace, List<String> tables);
}
diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java
index b72098523a..7e7934a960 100644
--- a/src/java/org/apache/cassandra/tools/NodeProbe.java
+++ b/src/java/org/apache/cassandra/tools/NodeProbe.java
@@ -105,6 +105,8 @@ import org.apache.cassandra.metrics.ThreadPoolMetrics;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.net.MessagingServiceMBean;
import org.apache.cassandra.service.ActiveRepairServiceMBean;
+import org.apache.cassandra.service.AutoRepairService;
+import org.apache.cassandra.service.AutoRepairServiceMBean;
import org.apache.cassandra.service.CacheService;
import org.apache.cassandra.service.CacheServiceMBean;
import org.apache.cassandra.service.GCInspector;
@@ -173,6 +175,7 @@ public class NodeProbe implements AutoCloseable
protected PermissionsCacheMBean pcProxy;
protected RolesCacheMBean rcProxy;
protected GuardrailsMBean grProxy;
+ protected AutoRepairServiceMBean autoRepairProxy;
protected Output output;
private boolean failed;
@@ -314,6 +317,8 @@ public class NodeProbe implements AutoCloseable
name = new ObjectName(Guardrails.MBEAN_NAME);
grProxy = JMX.newMBeanProxy(mbeanServerConn, name, GuardrailsMBean.class);
+ name = new ObjectName(AutoRepairService.MBEAN_NAME);
+ autoRepairProxy = JMX.newMBeanProxy(mbeanServerConn, name, AutoRepairServiceMBean.class);
}
catch (MalformedObjectNameException e)
{
@@ -2443,6 +2448,146 @@ public class NodeProbe implements AutoCloseable
{
return grProxy;
}
+
+ public boolean isAutoRepairDisabled()
+ {
+ return autoRepairProxy.isAutoRepairDisabled();
+ }
+
+ public String autoRepairConfiguration()
+ {
+ return autoRepairProxy.getAutoRepairConfiguration();
+ }
+
+ public void setAutoRepairTokenRangeSplitterParameter(String repairType, String key, String value)
+ {
+ autoRepairProxy.setAutoRepairTokenRangeSplitterParameter(repairType, key, value);
+ }
+
+ public void setAutoRepairEnabled(String repairType, boolean enabled)
+ {
+ autoRepairProxy.setAutoRepairEnabled(repairType, enabled);
+ }
+
+ public void setAutoRepairThreads(String repairType, int repairThreads)
+ {
+ autoRepairProxy.setRepairThreads(repairType, repairThreads);
+ }
+
+ public void setAutoRepairPriorityForHosts(String repairType, String commaSeparatedHostSet)
+ {
+ autoRepairProxy.setRepairPriorityForHosts(repairType, commaSeparatedHostSet);
+ }
+
+ public void setAutoRepairForceRepairForHosts(String repairType, String commaSeparatedHostSet)
+ {
+ autoRepairProxy.setForceRepairForHosts(repairType, commaSeparatedHostSet);
+ }
+
+ public void setAutoRepairMinInterval(String repairType, String minRepairInterval)
+ {
+ autoRepairProxy.setRepairMinInterval(repairType, minRepairInterval);
+ }
+
+ public void setAutoRepairHistoryClearDeleteHostsBufferDuration(String duration)
+ {
+ autoRepairProxy.setAutoRepairHistoryClearDeleteHostsBufferDuration(duration);
+ }
+
+ public void startAutoRepairScheduler()
+ {
+ autoRepairProxy.startScheduler();
+ }
+
+ public void setAutoRepairMinRepairTaskDuration(String duration)
+ {
+ autoRepairProxy.setAutoRepairMinRepairTaskDuration(duration);
+ }
+
+ public void setAutoRepairSSTableCountHigherThreshold(String repairType, int ssTableHigherThreshold)
+ {
+ autoRepairProxy.setRepairSSTableCountHigherThreshold(repairType, ssTableHigherThreshold);
+ }
+
+ public void setAutoRepairTableMaxRepairTime(String repairType, String autoRepairTableMaxRepairTime)
+ {
+ autoRepairProxy.setAutoRepairTableMaxRepairTime(repairType, autoRepairTableMaxRepairTime);
+ }
+
+ public void setAutoRepairIgnoreDCs(String repairType, Set<String> ignoreDCs)
+ {
+ autoRepairProxy.setIgnoreDCs(repairType, ignoreDCs);
+ }
+
+ public void setAutoRepairParallelRepairPercentage(String repairType, int percentage)
+ {
+ autoRepairProxy.setParallelRepairPercentage(repairType, percentage);
+ }
+
+ public void setAutoRepairParallelRepairCount(String repairType, int count)
+ {
+ autoRepairProxy.setParallelRepairCount(repairType, count);
+ }
+
+ public void setAutoRepairAllowParallelReplicaRepair(String repairType, boolean enabled)
+ {
+ autoRepairProxy.setAllowParallelReplicaRepair(repairType, enabled);
+ }
+
+ public void setAutoRepairAllowParallelReplicaRepairAcrossSchedules(String repairType, boolean enabled)
+ {
+ autoRepairProxy.setAllowParallelReplicaRepairAcrossSchedules(repairType, enabled);
+ }
+
+ public void setAutoRepairPrimaryTokenRangeOnly(String repairType, boolean primaryTokenRangeOnly)
+ {
+ autoRepairProxy.setPrimaryTokenRangeOnly(repairType, primaryTokenRangeOnly);
+ }
+
+ public void setAutoRepairMaterializedViewRepairEnabled(String repairType, boolean enabled)
+ {
+ autoRepairProxy.setMVRepairEnabled(repairType, enabled);
+ }
+
+ public List<String> mutateSSTableRepairedState(boolean repair, boolean preview, String keyspace, List<String> tables)
+ {
+ return ssProxy.mutateSSTableRepairedState(repair, preview, keyspace, tables);
+ }
+
+ public List<String> getAutoRepairTablesForKeyspace(String keyspace)
+ {
+ return ssProxy.getTablesForKeyspace(keyspace);
+ }
+
+ public void setAutoRepairSessionTimeout(String repairType, String timeout)
+ {
+ autoRepairProxy.setRepairSessionTimeout(repairType, timeout);
+ }
+
+ public Set<String> getAutoRepairOnGoingRepairHostIds(String repairType)
+ {
+ return autoRepairProxy.getOnGoingRepairHostIds(repairType);
+ }
+
+ public void setAutoRepairRepairByKeyspace(String repairType, boolean enabled)
+ {
+ autoRepairProxy.setRepairByKeyspace(repairType, enabled);
+ }
+
+ public void setAutoRepairMaxRetriesCount(String repairType, int retries)
+ {
+ autoRepairProxy.setAutoRepairMaxRetriesCount(repairType, retries);
+ }
+
+ public void setAutoRepairRetryBackoff(String repairType, String interval)
+ {
+ autoRepairProxy.setAutoRepairRetryBackoff(repairType, interval);
+ }
+
+ public void setMixedMajorVersionRepairEnabled(boolean enabled)
+ {
+ autoRepairProxy.setMixedMajorVersionRepairEnabled(enabled);
+ }
}
class ColumnFamilyStoreMBeanIterator implements Iterator<Map.Entry<String, ColumnFamilyStoreMBean>>
diff --git a/src/java/org/apache/cassandra/tools/NodeTool.java b/src/java/org/apache/cassandra/tools/NodeTool.java
index aa66719474..019319fcbc 100644
--- a/src/java/org/apache/cassandra/tools/NodeTool.java
+++ b/src/java/org/apache/cassandra/tools/NodeTool.java
@@ -96,6 +96,7 @@ public class NodeTool
public int execute(String... args)
{
List<Class<? extends NodeToolCmdRunnable>> commands = newArrayList(
+ AutoRepairStatus.class,
Assassinate.class,
CassHelp.class,
CIDRFilteringStats.class,
@@ -136,6 +137,7 @@ public class NodeTool
GcStats.class,
GetAuditLog.class,
GetAuthCacheConfig.class,
+ GetAutoRepairConfig.class,
GetBatchlogReplayTrottle.class,
GetCIDRGroupsOfIP.class,
GetColumnIndexSize.class,
@@ -199,6 +201,7 @@ public class NodeTool
Ring.class,
Scrub.class,
SetAuthCacheConfig.class,
+ SetAutoRepairConfig.class,
SetBatchlogReplayThrottle.class,
SetCacheCapacity.class,
SetCacheKeysToSave.class,
@@ -220,6 +223,7 @@ public class NodeTool
SetTraceProbability.class,
Sjk.class,
Snapshot.class,
+ SSTableRepairedSet.class,
Status.class,
StatusAutoCompaction.class,
StatusBackup.class,
diff --git a/src/java/org/apache/cassandra/tools/nodetool/AutoRepairStatus.java b/src/java/org/apache/cassandra/tools/nodetool/AutoRepairStatus.java
new file mode 100644
index 0000000000..bb594a010f
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/AutoRepairStatus.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools.nodetool;
+
+import java.io.PrintStream;
+import java.util.Set;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import io.airlift.airline.Command;
+import io.airlift.airline.Option;
+import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool;
+import org.apache.cassandra.tools.nodetool.formatter.TableBuilder;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+/**
+ * Provides currently running auto-repair tasks.
+ */
+@Command(name = "autorepairstatus", description = "Print autorepair status")
+public class AutoRepairStatus extends NodeTool.NodeToolCmd
+{
+ @VisibleForTesting
+ @Option(title = "repair type", name = { "-t", "--repair-type" }, description = "Repair type")
+ protected String repairType;
+
+ @Override
+ public void execute(NodeProbe probe)
+ {
+ checkArgument(repairType != null, "--repair-type is required.");
+ PrintStream out = probe.output().out;
+
+ if (probe.isAutoRepairDisabled())
+ {
+ out.println("Auto-repair is not enabled");
+ return;
+ }
+
+ TableBuilder table = new TableBuilder();
+ table.add("Active Repairs");
+ Set<String> ongoingRepairHostIds = probe.getAutoRepairOnGoingRepairHostIds(repairType);
+ table.add(getSetString(ongoingRepairHostIds));
+ table.printTo(out);
+ }
+
+ private String getSetString(Set<String> hostIds)
+ {
+ if (hostIds.isEmpty())
+ {
+ return "NONE";
+ }
+ StringBuilder sb = new StringBuilder();
+ for (String id : hostIds)
+ {
+ sb.append(id);
+ sb.append(",");
+ }
+ // remove last ","
+ sb.setLength(Math.max(sb.length() - 1, 0));
+ return sb.toString();
+ }
+}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java b/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java
new file mode 100644
index 0000000000..9744498de7
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/GetAutoRepairConfig.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.tools.nodetool;
+
+import java.io.PrintStream;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import io.airlift.airline.Command;
+import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
+
+/**
+ * Prints all the configurations for AutoRepair through nodetool.
+ */
+@Command(name = "getautorepairconfig", description = "Print autorepair configurations")
+public class GetAutoRepairConfig extends NodeToolCmd
+{
+ @VisibleForTesting
+ protected static PrintStream out = System.out;
+
+ @Override
+ public void execute(NodeProbe probe)
+ {
+ if (probe.isAutoRepairDisabled())
+ out.println("Auto-repair is not enabled");
+ else
+ out.println(probe.autoRepairConfiguration());
+ }
+}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/SSTableRepairedSet.java b/src/java/org/apache/cassandra/tools/nodetool/SSTableRepairedSet.java
new file mode 100644
index 0000000000..2a7b56732a
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/SSTableRepairedSet.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools.nodetool;
+
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
+import io.airlift.airline.Option;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool;
+
+/**
+ * Provides a way to set the repaired state of SSTables without any downtime through nodetool.
+ */
+@Command(name = "sstablerepairedset", description = "Set the repaired state of SSTables for given keyspace/tables")
+public class SSTableRepairedSet extends NodeTool.NodeToolCmd
+{
+ @Arguments(usage = "[<keyspace> <table...>]", description = "Optional keyspace followed by zero or more tables")
+ protected List<String> args = new ArrayList<>();
+
+ @Option(title = "really-set",
+ name = { "--really-set" },
+ description = "Really set the repaired state of SSTables. If not set, only print SSTables that would be affected.")
+ protected boolean reallySet = false;
+
+ @Option(title = "is-repaired",
+ name = { "--is-repaired" },
+ description = "Set SSTables to repaired state.")
+ protected boolean isRepaired = false;
+
+ @Option(title = "is-unrepaired",
+ name = { "--is-unrepaired" },
+ description = "Set SSTables to unrepaired state.")
+ protected boolean isUnrepaired = false;
+
+ @Override
+ public void execute(NodeProbe probe)
+ {
+ PrintStream out = probe.output().out;
+
+ if (isRepaired == isUnrepaired)
+ {
+ out.println("Exactly one of --is-repaired or --is-unrepaired must be provided.");
+ return;
+ }
+
+ String message;
+ if (reallySet)
+ message = "Mutating repaired state of SSTables for";
+ else
+ message = "Previewing repaired state mutation of SSTables for";
+
+ List<String> keyspaces = parseOptionalKeyspace(args, probe, KeyspaceSet.NON_LOCAL_STRATEGY);
+ List<String> tables = new ArrayList<>(Arrays.asList(parseOptionalTables(args)));
+
+ if (args.isEmpty())
+ message += " all keyspaces";
+ else
+ message += tables.isEmpty() ? " all tables" : " tables " + String.join(", ", tables)
+ + " in keyspace " + keyspaces.get(0);
+ message += " to " + (isRepaired ? "repaired" : "unrepaired");
+ out.println(message);
+
+ List<String> sstableList = new ArrayList<>();
+ for (String keyspace : keyspaces)
+ {
+ try
+ {
+ sstableList.addAll(probe.mutateSSTableRepairedState(isRepaired, !reallySet, keyspace,
+ tables.isEmpty()
+ ? probe.getAutoRepairTablesForKeyspace(keyspace) // mutate all tables
+ : tables)); // mutate specific tables
+ }
+ catch (InvalidRequestException e)
+ {
+ out.println(e.getMessage());
+ }
+ }
+ if (!reallySet)
+ out.println("The following SSTables would be mutated:");
+ else
+ out.println("The following SSTables were mutated:");
+ for (String sstable : sstableList)
+ out.println(sstable);
+ }
+}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java b/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java
new file mode 100644
index 0000000000..bc0e5d8803
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/SetAutoRepairConfig.java
@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.tools.nodetool;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Splitter;
+import com.google.common.base.Throwables;
+
+import io.airlift.airline.Arguments;
+import io.airlift.airline.Command;
+import io.airlift.airline.Option;
+import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
+
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+
+import javax.management.InstanceNotFoundException;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+/**
+ * Allows to set AutoRepair configuration through nodetool.
+ */
+@Command(name = "setautorepairconfig", description = "sets the autorepair configuration")
+public class SetAutoRepairConfig extends NodeToolCmd
+{
+ @VisibleForTesting
+ @Arguments(title = "<autorepairparam> <value>", usage = "<autorepairparam> <value>",
+ description = "autorepair param and value.\nPossible autorepair parameters are as following: " +
+ "[start_scheduler|number_of_repair_threads|min_repair_interval|sstable_upper_threshold" +
+ "|enabled|table_max_repair_time|priority_hosts|forcerepair_hosts|ignore_dcs" +
+ "|history_clear_delete_hosts_buffer_interval|repair_primary_token_range_only" +
+ "|parallel_repair_count|parallel_repair_percentage" +
+ "|allow_parallel_replica_repair|allow_parallel_repair_across_schedules" +
+ "|materialized_view_repair_enabled|repair_max_retries" +
+ "|repair_retry_backoff|repair_session_timeout|min_repair_task_duration" +
+ "|repair_by_keyspace|mixed_major_version_repair_enabled|token_range_splitter.<property>]",
+ required = true)
+ protected List<String> args = new ArrayList<>();
+
+ @VisibleForTesting
+ @Option(title = "repair type", name = { "-t", "--repair-type" }, description = "Repair type")
+ protected String repairTypeStr;
+
+ @VisibleForTesting
+ protected PrintStream out = System.out;
+
+ private static final String TOKEN_RANGE_SPLITTER_PROPERTY_PREFIX = "token_range_splitter.";
+
+ @Override
+ public void execute(NodeProbe probe)
+ {
+ checkArgument(args.size() == 2, "setautorepairconfig requires param-type, and value args.");
+ String paramType = args.get(0);
+ String paramVal = args.get(1);
+
+ try
+ {
+ probe.isAutoRepairDisabled();
+ }
+ catch (Throwable e)
+ {
+ if (Throwables.getRootCause(e) instanceof InstanceNotFoundException)
+ {
+ out.println("Auto-repair is not supported or not enabled via -Dcassandra.autorepair.enable=true on the server.");
+ return;
+ }
+ else
+ {
+ throw e;
+ }
+ }
+
+ if (probe.isAutoRepairDisabled() && !paramType.equalsIgnoreCase("start_scheduler"))
+ {
+ out.println("Auto-repair is not enabled");
+ return;
+ }
+
+ // options that do not require --repair-type option
+ switch (paramType)
+ {
+ case "start_scheduler":
+ if (Boolean.parseBoolean(paramVal))
+ {
+ probe.startAutoRepairScheduler();
+ }
+ return;
+ case "history_clear_delete_hosts_buffer_interval":
+ probe.setAutoRepairHistoryClearDeleteHostsBufferDuration(paramVal);
+ return;
+ case "min_repair_task_duration":
+ probe.setAutoRepairMinRepairTaskDuration(paramVal);
+ return;
+ case "mixed_major_version_repair_enabled":
+ probe.setMixedMajorVersionRepairEnabled(Boolean.parseBoolean(paramVal));
+ return;
+ default:
+ // proceed to options that require --repair-type option
+ break;
+ }
+
+ // options below require --repair-type option
+ Objects.requireNonNull(repairTypeStr, "--repair-type is required for this parameter.");
+
+ if(paramType.startsWith(TOKEN_RANGE_SPLITTER_PROPERTY_PREFIX))
+ {
+ final String key = paramType.replace(TOKEN_RANGE_SPLITTER_PROPERTY_PREFIX, "");
+ probe.setAutoRepairTokenRangeSplitterParameter(repairTypeStr, key, paramVal);
+ return;
+ }
+
+ switch (paramType)
+ {
+ case "enabled":
+ probe.setAutoRepairEnabled(repairTypeStr, Boolean.parseBoolean(paramVal));
+ break;
+ case "number_of_repair_threads":
+ probe.setAutoRepairThreads(repairTypeStr, Integer.parseInt(paramVal));
+ break;
+ case "min_repair_interval":
+ probe.setAutoRepairMinInterval(repairTypeStr, paramVal);
+ break;
+ case "sstable_upper_threshold":
+ probe.setAutoRepairSSTableCountHigherThreshold(repairTypeStr, Integer.parseInt(paramVal));
+ break;
+ case "table_max_repair_time":
+ probe.setAutoRepairTableMaxRepairTime(repairTypeStr, paramVal);
+ break;
+ case "priority_hosts":
+ if (paramVal!= null && !paramVal.isEmpty())
+ {
+ probe.setAutoRepairPriorityForHosts(repairTypeStr, paramVal);
+ }
+ break;
+ case "forcerepair_hosts":
+ probe.setAutoRepairForceRepairForHosts(repairTypeStr, paramVal);
+ break;
+ case "ignore_dcs":
+ Set<String> ignoreDCs = new HashSet<>();
+ for (String dc : Splitter.on(',').split(paramVal))
+ {
+ ignoreDCs.add(dc);
+ }
+ probe.setAutoRepairIgnoreDCs(repairTypeStr, ignoreDCs);
+ break;
+ case "repair_primary_token_range_only":
+ probe.setAutoRepairPrimaryTokenRangeOnly(repairTypeStr, Boolean.parseBoolean(paramVal));
+ break;
+ case "parallel_repair_count":
+ probe.setAutoRepairParallelRepairCount(repairTypeStr, Integer.parseInt(paramVal));
+ break;
+ case "parallel_repair_percentage":
+ probe.setAutoRepairParallelRepairPercentage(repairTypeStr, Integer.parseInt(paramVal));
+ break;
+ case "allow_parallel_replica_repair":
+ probe.setAutoRepairAllowParallelReplicaRepair(repairTypeStr, Boolean.parseBoolean(paramVal));
+ break;
+ case "allow_parallel_replica_repair_across_schedules":
+ probe.setAutoRepairAllowParallelReplicaRepairAcrossSchedules(repairTypeStr, Boolean.parseBoolean(paramVal));
+ break;
+ case "materialized_view_repair_enabled":
+ probe.setAutoRepairMaterializedViewRepairEnabled(repairTypeStr, Boolean.parseBoolean(paramVal));
+ break;
+ case "repair_session_timeout":
+ probe.setAutoRepairSessionTimeout(repairTypeStr, paramVal);
+ break;
+ case "repair_by_keyspace":
+ probe.setAutoRepairRepairByKeyspace(repairTypeStr, Boolean.parseBoolean(paramVal));
+ break;
+ case "repair_max_retries":
+ probe.setAutoRepairMaxRetriesCount(repairTypeStr, Integer.parseInt(paramVal));
+ break;
+ case "repair_retry_backoff":
+ probe.setAutoRepairRetryBackoff(repairTypeStr, paramVal);
+ break;
+ default:
+ throw new IllegalArgumentException("Unknown parameter: " + paramType);
+ }
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java
index 89d7a9c348..b1d05a8a22 100644
--- a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java
+++ b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java
@@ -115,6 +115,7 @@ import org.apache.cassandra.net.Verb;
import org.apache.cassandra.schema.MigrationCoordinator;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.repair.autorepair.AutoRepair;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.service.CassandraDaemon;
import org.apache.cassandra.service.ClientState;
@@ -902,6 +903,7 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance
() -> SSTableReader.shutdownBlocking(1L, MINUTES),
() -> shutdownAndWait(Collections.singletonList(ActiveRepairService.repairCommandExecutor())),
() -> ActiveRepairService.instance().shutdownNowAndWait(1L, MINUTES),
+ () -> AutoRepair.instance.shutdownBlocking(),
() -> SnapshotManager.shutdownAndWait(1L, MINUTES)
);
diff --git a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairFlagToggleTest.java b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairFlagToggleTest.java
new file mode 100644
index 0000000000..35ea0f3ee6
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairFlagToggleTest.java
@@ -0,0 +1,232 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.repair;
+
+import org.junit.Test;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.api.Feature;
+import org.apache.cassandra.distributed.test.TestBaseImpl;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests enabling and disabling the AUTOREPAIR_ENABLE flag.
+ * <p>
+ * This test verifies that:
+ * 1. When started with AUTOREPAIR_ENABLE=false, the auto_repair column does not appear in DESCRIBE TABLE
+ * 2. After restart with AUTOREPAIR_ENABLE=true, the auto_repair column appears in DESCRIBE TABLE
+ * 3. The system_distributed auto-repair tables are created after enabling
+ * 4. Restarting with AUTOREPAIR_ENABLE=false after it was previously enabled fails with a deserialization error
+ * <p>
+ * Note: The reverse (enabled → disabled) is NOT supported and will cause the node to fail during
+ * initialization due to schema incompatibility (the persisted schema contains the auto_repair column
+ * that is not recognized when the property is disabled).
+ */
+public class AutoRepairFlagToggleTest extends TestBaseImpl
+{
+ @Test
+ public void testDescribeWithAutoRepairSchedulerToggle() throws Exception
+ {
+ // Start with AUTOREPAIR_ENABLE=true
+ CassandraRelevantProperties.AUTOREPAIR_ENABLE.setBoolean(true);
+
+ try (Cluster cluster = Cluster.build(1)
+ .withConfig(config -> config
+ .with(Feature.GOSSIP, Feature.NETWORK)
+ .set("enable_materialized_views", true))
+ .start())
+ {
+ // Create a test keyspace and table
+ cluster.schemaChange("CREATE KEYSPACE test_ks WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}");
+ cluster.schemaChange("CREATE TABLE test_ks.test_tbl (pk int PRIMARY KEY, v int)");
+ cluster.schemaChange("CREATE MATERIALIZED VIEW test_ks.test_mv AS SELECT * FROM test_ks.test_tbl WHERE pk IS NOT NULL PRIMARY KEY (pk)");
+
+ // Phase 1: With auto-repair scheduling disabled
+ cluster.get(1).runOnInstance(() -> {
+ org.apache.cassandra.config.DatabaseDescriptor.getAutoRepairConfig().setAutoRepairSchedulingEnabled(false);
+ });
+
+ Object[][] describeTableDisabled = cluster.coordinator(1).execute(
+ "DESCRIBE TABLE test_ks.test_tbl",
+ ConsistencyLevel.LOCAL_ONE);
+ String tableStatementDisabled = (String) describeTableDisabled[0][3];
+ assertFalse("DESCRIBE TABLE should NOT include auto_repair when scheduler is disabled",
+ tableStatementDisabled.contains("auto_repair"));
+
+ Object[][] describeMvDisabled = cluster.coordinator(1).execute(
+ "DESCRIBE MATERIALIZED VIEW test_ks.test_mv",
+ ConsistencyLevel.LOCAL_ONE);
+ String mvStatementDisabled = (String) describeMvDisabled[0][3];
+ assertFalse("DESCRIBE MATERIALIZED VIEW should NOT include auto_repair when scheduler is disabled",
+ mvStatementDisabled.contains("auto_repair"));
+
+ // Phase 2: Enable auto-repair scheduling
+ cluster.get(1).runOnInstance(() -> {
+ org.apache.cassandra.config.DatabaseDescriptor.getAutoRepairConfig().setAutoRepairSchedulingEnabled(true);
+ });
+
+ Object[][] describeTableEnabled = cluster.coordinator(1).execute(
+ "DESCRIBE TABLE test_ks.test_tbl",
+ ConsistencyLevel.LOCAL_ONE);
+ String tableStatementEnabled = (String) describeTableEnabled[0][3];
+ assertTrue("DESCRIBE TABLE should include auto_repair when scheduler is enabled",
+ tableStatementEnabled.contains("auto_repair"));
+
+ Object[][] describeMvEnabled = cluster.coordinator(1).execute(
+ "DESCRIBE MATERIALIZED VIEW test_ks.test_mv",
+ ConsistencyLevel.LOCAL_ONE);
+ String mvStatementEnabled = (String) describeMvEnabled[0][3];
+ assertTrue("DESCRIBE MATERIALIZED VIEW should include auto_repair when scheduler is enabled",
+ mvStatementEnabled.contains("auto_repair"));
+
+ // Phase 3: Disable auto-repair scheduling again
+ cluster.get(1).runOnInstance(() -> {
+ org.apache.cassandra.config.DatabaseDescriptor.getAutoRepairConfig().setAutoRepairSchedulingEnabled(false);
+ });
+
+ Object[][] describeTableDisabledAgain = cluster.coordinator(1).execute(
+ "DESCRIBE TABLE test_ks.test_tbl",
+ ConsistencyLevel.LOCAL_ONE);
+ String tableStatementDisabledAgain = (String) describeTableDisabledAgain[0][3];
+ assertFalse("DESCRIBE TABLE should NOT include auto_repair after disabling scheduler",
+ tableStatementDisabledAgain.contains("auto_repair"));
+
+ Object[][] describeMvDisabledAgain = cluster.coordinator(1).execute(
+ "DESCRIBE MATERIALIZED VIEW test_ks.test_mv",
+ ConsistencyLevel.LOCAL_ONE);
+ String mvStatementDisabledAgain = (String) describeMvDisabledAgain[0][3];
+ assertFalse("DESCRIBE MATERIALIZED VIEW should NOT include auto_repair after disabling scheduler",
+ mvStatementDisabledAgain.contains("auto_repair"));
+
+ // Verify table and MV are still accessible throughout
+ cluster.coordinator(1).execute("INSERT INTO test_ks.test_tbl (pk, v) VALUES (1, 100)", ConsistencyLevel.ONE);
+ Object[][] tableData = cluster.coordinator(1).execute("SELECT * FROM test_ks.test_tbl WHERE pk = 1", ConsistencyLevel.ONE);
+ assertTrue("Table should still be accessible after scheduler toggles", tableData.length == 1);
+
+ Object[][] mvData = cluster.coordinator(1).execute("SELECT * FROM test_ks.test_mv WHERE pk = 1", ConsistencyLevel.ONE);
+ assertTrue("Materialized view should still be accessible after scheduler toggles", mvData.length == 1);
+ }
+ }
+
+ @Test
+ public void testEnablingAutoRepairFlag() throws Exception
+ {
+ // Phase 1: Start with AUTOREPAIR_ENABLE=false
+ CassandraRelevantProperties.AUTOREPAIR_ENABLE.setBoolean(false);
+
+ try (Cluster cluster = Cluster.build(1)
+ .withConfig(config -> config
+ .with(Feature.GOSSIP, Feature.NETWORK))
+ .start())
+ {
+ // Create a test keyspace and table
+ cluster.schemaChange("CREATE KEYSPACE test_ks WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}");
+ cluster.schemaChange("CREATE TABLE test_ks.test_tbl (pk int PRIMARY KEY, v int)");
+
+ // Verify DESCRIBE TABLE does NOT show auto_repair when flag is disabled
+ // DESCRIBE TABLE result columns: keyspace_name (0), type (1), name (2), create_statement (3)
+ Object[][] describeResultBefore = cluster.coordinator(1).execute(
+ "DESCRIBE TABLE test_ks.test_tbl",
+ ConsistencyLevel.LOCAL_ONE);
+ String createStatementBefore = (String) describeResultBefore[0][3];
+ assertFalse("DESCRIBE TABLE should NOT include auto_repair when flag is disabled",
+ createStatementBefore.contains("auto_repair"));
+
+ // Verify system_distributed auto-repair tables do NOT exist when flag is disabled
+ Object[][] historyResultBefore = cluster.coordinator(1).execute(
+ "SELECT table_name FROM system_schema.tables WHERE keyspace_name = 'system_distributed' AND table_name = 'auto_repair_history'",
+ ConsistencyLevel.LOCAL_ONE);
+ assertTrue("auto_repair_history table should NOT exist when flag is disabled", historyResultBefore.length == 0);
+
+ Object[][] priorityResultBefore = cluster.coordinator(1).execute(
+ "SELECT table_name FROM system_schema.tables WHERE keyspace_name = 'system_distributed' AND table_name = 'auto_repair_priority'",
+ ConsistencyLevel.LOCAL_ONE);
+ assertTrue("auto_repair_priority table should NOT exist when flag is disabled", priorityResultBefore.length == 0);
+
+ // Phase 2: Restart with AUTOREPAIR_ENABLE=true
+ cluster.get(1).nodetoolResult("drain").asserts().success();
+ cluster.get(1).shutdown().get();
+ CassandraRelevantProperties.AUTOREPAIR_ENABLE.setBoolean(true);
+ cluster.get(1).startup();
+
+ // Enable auto-repair scheduling so DESCRIBE TABLE will show the auto_repair property
+ cluster.get(1).runOnInstance(() -> {
+ org.apache.cassandra.config.DatabaseDescriptor.getAutoRepairConfig().setAutoRepairSchedulingEnabled(true);
+ });
+
+ // Verify DESCRIBE TABLE now shows auto_repair when flag is enabled
+ Object[][] describeResultAfter = cluster.coordinator(1).execute(
+ "DESCRIBE TABLE test_ks.test_tbl",
+ ConsistencyLevel.LOCAL_ONE);
+ String createStatementAfter = (String) describeResultAfter[0][3];
+ assertTrue("DESCRIBE TABLE should include auto_repair after enabling flag",
+ createStatementAfter.contains("auto_repair"));
+
+ // Verify the test table is still accessible
+ cluster.coordinator(1).execute("INSERT INTO test_ks.test_tbl (pk, v) VALUES (1, 100)", ConsistencyLevel.ONE);
+ Object[][] data = cluster.coordinator(1).execute("SELECT * FROM test_ks.test_tbl WHERE pk = 1", ConsistencyLevel.ONE);
+ assertTrue("Table should still be accessible after enabling flag", data.length == 1);
+ }
+ }
+
+ /**
+ * Tests that restarting with AUTOREPAIR_ENABLE=false after it was previously true causes
+ * "Unknown column auto_repair during deserialization" because the persisted system_schema.tables
+ * SSTable references the auto_repair column which is absent from the metadata when the flag
+ * is disabled. In production (disk_failure_policy=stop), this kills the JVM.
+ */
+ @Test
+ public void testDisablingAutoRepairFlagFails() throws Exception
+ {
+ CassandraRelevantProperties.AUTOREPAIR_ENABLE.setBoolean(true);
+
+ try (Cluster cluster = Cluster.build(1)
+ .withConfig(config -> config
+ .with(Feature.GOSSIP, Feature.NETWORK))
+ .start())
+ {
+ // Enable scheduling before creating tables so auto_repair data is written to system_schema
+ cluster.get(1).runOnInstance(() -> {
+ org.apache.cassandra.config.DatabaseDescriptor.getAutoRepairConfig().setAutoRepairSchedulingEnabled(true);
+ });
+
+ cluster.schemaChange("CREATE KEYSPACE test_ks WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}");
+ cluster.schemaChange("CREATE TABLE test_ks.test_tbl (pk int PRIMARY KEY, v int)");
+
+ cluster.get(1).flush("system_schema");
+ cluster.get(1).nodetoolResult("drain").asserts().success();
+ cluster.get(1).shutdown().get();
+
+ CassandraRelevantProperties.AUTOREPAIR_ENABLE.setBoolean(false);
+ cluster.get(1).startup();
+
+ assertTrue("Expected 'Unknown column auto_repair during deserialization' in logs",
+ cluster.get(1).logs().grep("Unknown column auto_repair during deserialization")
+ .getResult().size() > 0);
+
+ assertTrue("Expected user-friendly message instructing to set cassandra.autorepair.enable=true",
+ cluster.get(1).logs().grep("Set -Dcassandra.autorepair.enable=true and restart")
+ .getResult().size() > 0);
+ }
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerDisallowParallelReplicaRepairAcrossSchedulesTest.java b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerDisallowParallelReplicaRepairAcrossSchedulesTest.java
new file mode 100644
index 0000000000..bbbd003b23
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerDisallowParallelReplicaRepairAcrossSchedulesTest.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.repair;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.collect.ImmutableMap;
+
+import org.apache.cassandra.Util;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.metrics.AutoRepairMetrics;
+import org.apache.cassandra.metrics.AutoRepairMetricsManager;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.test.TestBaseImpl;
+import org.apache.cassandra.repair.autorepair.AutoRepair;
+import org.apache.cassandra.repair.autorepair.AutoRepairConfig;
+import org.apache.cassandra.service.AutoRepairService;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.hamcrest.Matchers.greaterThan;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Distributed tests for {@link org.apache.cassandra.repair.autorepair.AutoRepair} scheduler's
+ * allow_parallel_replica_repair_across_schedules feature.
+ */
+public class AutoRepairSchedulerDisallowParallelReplicaRepairAcrossSchedulesTest extends TestBaseImpl
+{
+ private static Cluster cluster;
+
+ @BeforeClass
+ public static void init() throws IOException
+ {
+ CassandraRelevantProperties.AUTOREPAIR_ENABLE.setBoolean(true);
+
+ // Configure a cluster with preview and incremental repair enabled in a way that preview repair can be
+ // run on all three nodes concurrently, but incremental repair can only be run when there are no parallel
+ // repairs. We should detect contention in the incremental repair scheduler but not preview repaired
+ // scheduler as a result.
+ cluster = Cluster.build(3)
+ .withConfig(config -> config
+ .set("auto_repair",
+ ImmutableMap.of(
+ "repair_type_overrides",
+ ImmutableMap.of(AutoRepairConfig.RepairType.PREVIEW_REPAIRED.getConfigName(),
+ ImmutableMap.of(
+ // Configure preview repair to run frequently to
+ // provoke contention with incremental scheduler.
+ "initial_scheduler_delay", "5s",
+ "enabled", "true",
+ "parallel_repair_count", "3",
+ "allow_parallel_replica_repair", "true",
+ "min_repair_interval", "5s"),
+ AutoRepairConfig.RepairType.INCREMENTAL.getConfigName(),
+ ImmutableMap.of(
+ "initial_scheduler_delay", "5s",
+ "enabled", "true",
+ "parallel_repair_count", "3",
+ // Don't allow parallel replica repair across
+ // schedules
+ "allow_parallel_replica_repair", "false",
+ "allow_parallel_replica_repair_across_schedules", "false",
+ "min_repair_interval", "5s"))))
+ .set("auto_repair.enabled", "true")
+ .set("auto_repair.global_settings.repair_retry_backoff", "5s")
+ .set("auto_repair.repair_task_min_duration", "0s")
+ .set("auto_repair.repair_check_interval", "5s"))
+ .start();
+
+ cluster.schemaChange("CREATE KEYSPACE IF NOT EXISTS " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3};");
+ cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (pk int, ck text, v1 int, v2 int, PRIMARY KEY (pk, ck)) WITH read_repair='NONE'"));
+ }
+
+ @AfterClass
+ public static void tearDown()
+ {
+ cluster.close();
+ }
+
+ @Test
+ public void testScheduler()
+ {
+ cluster.forEach(i -> i.runOnInstance(() -> {
+ try
+ {
+ AutoRepairService.setup();
+ AutoRepair.instance.setup();
+ }
+ catch (Exception e)
+ {
+ throw new RuntimeException(e);
+ }
+ }));
+
+ // validate that the repair ran on all nodes
+ cluster.forEach(i -> i.runOnInstance(() -> {
+ // Expect contention on incremental repair across schedules
+ AutoRepairMetrics incrementalMetrics = AutoRepairMetricsManager.getMetrics(AutoRepairConfig.RepairType.INCREMENTAL);
+ Util.spinAssert(String.format("%s: AutoRepair has not observed any replica contention in INCREMENTAL repair",
+ FBUtilities.getJustBroadcastAddress().toString()),
+ greaterThan(0L),
+ incrementalMetrics.repairDelayedBySchedule::getCount,
+ 5,
+ TimeUnit.MINUTES);
+
+ // No repair contention should be observed for preview repaired since allow_parallel_replica_repair was true
+ AutoRepairMetrics previewMetrics = AutoRepairMetricsManager.getMetrics(AutoRepairConfig.RepairType.PREVIEW_REPAIRED);
+ assertEquals(0L, previewMetrics.repairDelayedByReplica.getCount());
+ assertEquals(0L, previewMetrics.repairDelayedBySchedule.getCount());
+ }));
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerStatsHelper.java b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerStatsHelper.java
new file mode 100644
index 0000000000..4e1d64e6dd
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerStatsHelper.java
@@ -0,0 +1,262 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.repair;
+
+import java.io.IOException;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+
+import com.google.common.collect.ImmutableMap;
+import org.junit.Assert;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.config.DurationSpec;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.lifecycle.SSTableSet;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.api.TokenSupplier;
+import org.apache.cassandra.distributed.test.TestBaseImpl;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.metrics.AutoRepairMetrics;
+import org.apache.cassandra.metrics.AutoRepairMetricsManager;
+import org.apache.cassandra.repair.autorepair.AutoRepair;
+import org.apache.cassandra.repair.autorepair.AutoRepairConfig;
+import org.apache.cassandra.repair.autorepair.AutoRepairUtils;
+import org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitterTest;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.SystemDistributedKeyspace;
+import org.apache.cassandra.service.AutoRepairService;
+
+import static org.apache.cassandra.schema.SchemaConstants.DISTRIBUTED_KEYSPACE_NAME;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+/**
+ * Helper class to test {@code totalBytesToRepair}, {@code bytesAlreadyRepaired}, {@code totalKeyspaceRepairPlansToRepair},
+ * and {@code keyspaceRepairPlansAlreadyRepaired}
+ * for {@link org.apache.cassandra.repair.autorepair.AutoRepairState} scheduler
+ */
+public class AutoRepairSchedulerStatsHelper extends TestBaseImpl
+{
+ static final Logger logger = LoggerFactory.getLogger(AutoRepairSchedulerStatsHelper.class);
+
+ private static Cluster cluster;
+ static SimpleDateFormat sdf;
+ private static final String KEYSPACE1 = "ks1";
+ private static final String KEYSPACE2 = "ks2";
+ private static final String TABLE1 = "tbl1";
+ private static final String TABLE2 = "tbl2";
+
+ public static void init(int numTokens) throws IOException
+ {
+ CassandraRelevantProperties.AUTOREPAIR_ENABLE.setBoolean(true);
+
+ // Define the expected date format pattern
+ String pattern = "EEE MMM dd HH:mm:ss z yyyy";
+ // Create SimpleDateFormat object with the given pattern
+ sdf = new SimpleDateFormat(pattern);
+ sdf.setLenient(false);
+ CassandraRelevantProperties.SYSTEM_DISTRIBUTED_DEFAULT_RF.setInt(1);
+ cluster = Cluster.build(1)
+ .withTokenCount(numTokens)
+ .withTokenSupplier(TokenSupplier.evenlyDistributedTokens(1, numTokens))
+ .withConfig(config -> config
+ .set("num_tokens", numTokens)
+ .set("auto_repair",
+ ImmutableMap.of(
+ "repair_type_overrides",
+ ImmutableMap.of(AutoRepairConfig.RepairType.FULL.getConfigName(),
+ ImmutableMap.of(
+ "initial_scheduler_delay", "5s",
+ "enabled", "true",
+ "parallel_repair_count", "1",
+ // Allow parallel replica repair to allow replicas
+ // to execute full repair at same time.
+ "allow_parallel_replica_repair", "true",
+ // Set min_repair_interval to a higher number to
+ // run only one round of AutoRepair
+ "min_repair_interval", "48h"))))
+ .set("auto_repair.enabled", "true")
+ .set("auto_repair.global_settings.repair_retry_backoff", "5s")
+ .set("auto_repair.repair_task_min_duration", "0s")
+ .set("auto_repair.repair_check_interval", "5s"))
+ .start();
+
+ cluster.schemaChange("CREATE KEYSPACE IF NOT EXISTS " + KEYSPACE1 + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};");
+ cluster.schemaChange("CREATE KEYSPACE IF NOT EXISTS " + KEYSPACE2 + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};");
+ // disable the compression to calculate an accurate expected repair bytes because with compression enabled,
+ // we only get estimated bytes, which hinders the ability to do actual vs. expected checks in the test case
+ cluster.schemaChange(String.format("CREATE TABLE %s.%s (pk int, ck int, v int, PRIMARY KEY (pk, ck)) WITH compression = { 'enabled' : false }", KEYSPACE1, TABLE1));
+ cluster.schemaChange(String.format("CREATE TABLE %s.%s (pk int, ck int, v int, PRIMARY KEY (pk, ck)) WITH compression = { 'enabled' : false }", KEYSPACE1, TABLE2));
+ cluster.schemaChange(String.format("CREATE TABLE %s.%s (pk int, ck int, v int, PRIMARY KEY (pk, ck)) WITH compression = { 'enabled' : false }", KEYSPACE2, TABLE1));
+ cluster.schemaChange(String.format("CREATE TABLE %s.%s (pk int, ck int, v int, PRIMARY KEY (pk, ck)) WITH compression = { 'enabled' : false }", KEYSPACE2, TABLE2));
+ }
+
+ public static void tearDown()
+ {
+ cluster.close();
+ }
+
+ public static void testSchedulerStats() throws ParseException
+ {
+ // ensure there was no history of previous repair runs through the scheduler
+ Object[][] rows = cluster.coordinator(1).execute(String.format("SELECT repair_type, host_id, repair_start_ts, repair_finish_ts, repair_turn FROM %s.%s", DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY), ConsistencyLevel.QUORUM);
+ assertEquals(0, rows.length);
+
+ // disabling AutoRepair for system_distributed and system_auth tables to avoid
+ // interfering with the repaired bytes/plans calculation
+ disableAutoRepair(SystemDistributedKeyspace.NAME, new HashSet<>(RepairTokenRangeSplitterTest.SYSTEM_DISTRIBUTED_TABLE_NAMES));
+ disableAutoRepair(SchemaConstants.AUTH_KEYSPACE_NAME, new HashSet<>(RepairTokenRangeSplitterTest.AUTH_TABLE_NAMES));
+
+ insertData();
+
+ cluster.get(1).runOnInstance(() -> {
+ try
+ {
+ AutoRepairService.setup();
+ AutoRepair.instance.setup();
+ }
+ catch (Exception e)
+ {
+ throw new RuntimeException(e);
+ }
+ });
+
+ cluster.forEach(i -> i.runOnInstance(() -> {
+ AutoRepair.SLEEP_IF_REPAIR_FINISHES_QUICKLY = new DurationSpec.IntSecondsBound("2s");
+
+ AutoRepairMetrics fullMetrics = AutoRepairMetricsManager.getMetrics(AutoRepairConfig.RepairType.FULL);
+ // Since the AutoRepair sleeps up to SLEEP_IF_REPAIR_FINISHES_QUICKLY if the repair finishes quickly,
+ // so the "nodeRepairTimeInSec" metric should at least be greater than or equal to
+ // SLEEP_IF_REPAIR_FINISHES_QUICKLY
+ while (fullMetrics.nodeRepairTimeInSec.getValue().longValue() <= 1)
+ {
+ try
+ {
+ Thread.sleep(1000);
+ }
+ catch (InterruptedException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+ logger.info("AutoRepair has completed one FULL repair cycle");
+
+ long expectedRepairBytes = calculateExpectedBytes(Arrays.asList(KEYSPACE1, KEYSPACE2));
+ assertEquals(fullMetrics.totalKeyspaceRepairPlansToRepair.getValue(), fullMetrics.keyspaceRepairPlansAlreadyRepaired.getValue());
+ // AutoRepair creates a repair plan per keyspace;
+ // Since there are two separate keyspaces, KEYSPACE1 and KEYSPACE2, the total expected plans should be "2"
+ assertEquals(2, fullMetrics.totalKeyspaceRepairPlansToRepair.getValue().intValue());
+ assertEquals(fullMetrics.totalBytesToRepair.getValue().longValue(), fullMetrics.bytesAlreadyRepaired.getValue().longValue());
+ assertEquals(expectedRepairBytes, fullMetrics.bytesAlreadyRepaired.getValue().longValue());
+ }));
+ validate(AutoRepairConfig.RepairType.FULL.toString());
+ }
+
+ private static long calculateExpectedBytes(List<String> keyspaces)
+ {
+ long totalBytes = 0;
+ for (String keyspace : keyspaces)
+ {
+ for (String table : Arrays.asList(TABLE1, TABLE2))
+ {
+ ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(keyspace, table);
+ assertNotNull(cfs);
+ Iterable<SSTableReader> sstables = cfs.getTracker().getView().select(SSTableSet.CANONICAL);
+ List<Range<Token>> tokenRanges = AutoRepairUtils.getTokenRanges(true, keyspace);
+ for (SSTableReader sstable : sstables)
+ {
+ long sstableSize = sstable.bytesOnDisk();
+ for (Range<Token> tokenRange : tokenRanges)
+ {
+ // Use SSTableReader API to get positions and size for the range
+ // This matches what AutoRepairUtils.getSizesForRangeOfSSTables() does
+ List<SSTableReader.PartitionPositionBounds> positionBounds = sstable.getPositionsForRanges(Collections.singleton(tokenRange));
+ if (!positionBounds.isEmpty())
+ {
+ long approximateRangeBytesInSSTable = sstable.onDiskSizeForPartitionPositions(positionBounds);
+ totalBytes += Math.min(approximateRangeBytesInSSTable, sstableSize);
+ }
+ }
+ }
+ }
+ }
+ return totalBytes;
+ }
+
+ private static void insertData()
+ {
+ for (int i = 0; i < 100; i++)
+ {
+ cluster.coordinator(1).execute(String.format("INSERT INTO %s.%s (pk, ck, v) VALUES (?,?,?)", KEYSPACE1, TABLE1),
+ ConsistencyLevel.ONE, i, i, i);
+ cluster.coordinator(1).execute(String.format("INSERT INTO %s.%s (pk, ck, v) VALUES (?,?,?)", KEYSPACE1, TABLE2),
+ ConsistencyLevel.ONE, i, i, i);
+ cluster.coordinator(1).execute(String.format("INSERT INTO %s.%s (pk, ck, v) VALUES (?,?,?)", KEYSPACE2, TABLE1),
+ ConsistencyLevel.ONE, i, i, i);
+ cluster.coordinator(1).execute(String.format("INSERT INTO %s.%s (pk, ck, v) VALUES (?,?,?)", KEYSPACE2, TABLE2),
+ ConsistencyLevel.ONE, i, i, i);
+ }
+ cluster.get(1).nodetool("flush", KEYSPACE1, TABLE1);
+ cluster.get(1).nodetool("flush", KEYSPACE1, TABLE2);
+ cluster.get(1).nodetool("flush", KEYSPACE2, TABLE1);
+ cluster.get(1).nodetool("flush", KEYSPACE2, TABLE2);
+ }
+
+ private static void disableAutoRepair(String keyspaceName, Set<String> distributedSystemTables)
+ {
+ for (String tableName : distributedSystemTables)
+ {
+ cluster.coordinator(1).execute(String.format("ALTER TABLE %s.%s WITH auto_repair = {'full_enabled': 'false'}", keyspaceName, tableName),
+ ConsistencyLevel.ONE);
+ }
+ }
+
+ private static void validate(String repairType) throws ParseException
+ {
+ Object[][] rows = cluster.coordinator(1).execute(String.format("SELECT repair_type, host_id, repair_start_ts, repair_finish_ts, repair_turn FROM %s.%s where repair_type='%s'", DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, repairType), ConsistencyLevel.QUORUM);
+ assertEquals(1, rows.length);
+ for (int node = 0; node < rows.length; node++)
+ {
+ Object[] row = rows[node];
+ // repair_type
+ Assert.assertEquals(repairType, row[0].toString());
+ // host_id
+ Assert.assertNotNull(UUID.fromString(row[1].toString()));
+ // ensure there is a legit repair_start_ts and repair_finish_ts
+ sdf.parse(row[2].toString());
+ sdf.parse(row[3].toString());
+ // the reason why the repair was scheduled
+ Assert.assertNotNull(row[4]);
+ Assert.assertEquals("MY_TURN", row[4].toString());
+ }
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerStatsNoVNodesTest.java b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerStatsNoVNodesTest.java
new file mode 100644
index 0000000000..96376095b9
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerStatsNoVNodesTest.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.repair;
+
+import java.io.IOException;
+import java.text.ParseException;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.distributed.test.TestBaseImpl;
+
+/**
+ * Test {@code totalBytesToRepair}, {@code bytesAlreadyRepaired}, {@code totalKeyspaceRepairPlansToRepair},
+ * and {@code keyspaceRepairPlansAlreadyRepaired}
+ * for {@link org.apache.cassandra.repair.autorepair.AutoRepairState} scheduler without v-nodes
+ */
+public class AutoRepairSchedulerStatsNoVNodesTest extends TestBaseImpl
+{
+ @BeforeClass
+ public static void init() throws IOException
+ {
+ AutoRepairSchedulerStatsHelper.init(1);
+ }
+
+ @AfterClass
+ public static void tearDown()
+ {
+ AutoRepairSchedulerStatsHelper.tearDown();
+ }
+
+ @Test
+ public void testSchedulerStats() throws ParseException
+ {
+ AutoRepairSchedulerStatsHelper.testSchedulerStats();
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerStatsVNodesTest.java b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerStatsVNodesTest.java
new file mode 100644
index 0000000000..8040a30328
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerStatsVNodesTest.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.repair;
+
+import java.io.IOException;
+import java.text.ParseException;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.distributed.test.TestBaseImpl;
+
+/**
+ * Test {@code totalBytesToRepair}, {@code bytesAlreadyRepaired}, {@code totalKeyspaceRepairPlansToRepair},
+ * and {@code keyspaceRepairPlansAlreadyRepaired}
+ * for {@link org.apache.cassandra.repair.autorepair.AutoRepairState} scheduler with v-nodes
+ */
+public class AutoRepairSchedulerStatsVNodesTest extends TestBaseImpl
+{
+ @BeforeClass
+ public static void init() throws IOException
+ {
+ AutoRepairSchedulerStatsHelper.init(16);
+ }
+
+ @AfterClass
+ public static void tearDown()
+ {
+ AutoRepairSchedulerStatsHelper.tearDown();
+ }
+
+ @Test
+ public void testSchedulerStats() throws ParseException
+ {
+ AutoRepairSchedulerStatsHelper.testSchedulerStats();
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java
new file mode 100644
index 0000000000..8b09ad1231
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairSchedulerTest.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.repair;
+
+import java.io.IOException;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.collect.ImmutableMap;
+
+import org.apache.cassandra.Util;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.config.DurationSpec;
+import org.apache.cassandra.distributed.api.TokenSupplier;
+import org.apache.cassandra.metrics.AutoRepairMetrics;
+import org.apache.cassandra.metrics.AutoRepairMetricsManager;
+import org.apache.cassandra.schema.SystemDistributedKeyspace;
+
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.test.TestBaseImpl;
+import org.apache.cassandra.repair.autorepair.AutoRepair;
+import org.apache.cassandra.repair.autorepair.AutoRepairConfig;
+import org.apache.cassandra.service.AutoRepairService;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.schema.SchemaConstants.DISTRIBUTED_KEYSPACE_NAME;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Distributed tests for {@link org.apache.cassandra.repair.autorepair.AutoRepair} scheduler
+ */
+public class AutoRepairSchedulerTest extends TestBaseImpl
+{
+ private static Cluster cluster;
+ static SimpleDateFormat sdf;
+
+ @BeforeClass
+ public static void init() throws IOException
+ {
+ CassandraRelevantProperties.AUTOREPAIR_ENABLE.setBoolean(true);
+
+ // Define the expected date format pattern
+ String pattern = "EEE MMM dd HH:mm:ss z yyyy";
+ // Create SimpleDateFormat object with the given pattern
+ sdf = new SimpleDateFormat(pattern);
+ sdf.setLenient(false);
+ // Configure a 3-node cluster with num_tokens: 4 and auto_repair enabled
+ cluster = Cluster.build(3)
+ .withTokenCount(4)
+ .withTokenSupplier(TokenSupplier.evenlyDistributedTokens(3, 4))
+ .withConfig(config -> config
+ .set("num_tokens", 4)
+ .set("auto_repair",
+ ImmutableMap.of(
+ "repair_type_overrides",
+ ImmutableMap.of(AutoRepairConfig.RepairType.FULL.getConfigName(),
+ ImmutableMap.of(
+ "initial_scheduler_delay", "5s",
+ "enabled", "true",
+ "parallel_repair_count", "3",
+ // Allow parallel replica repair to allow replicas
+ // to execute full repair at same time.
+ "allow_parallel_replica_repair", "true",
+ "min_repair_interval", "5s"),
+ AutoRepairConfig.RepairType.INCREMENTAL.getConfigName(),
+ ImmutableMap.of(
+ "initial_scheduler_delay", "5s",
+ "enabled", "true",
+ // Set parallel repair count to 3 to provoke
+ // contention between replicas when scheduling.
+ "parallel_repair_count", "3",
+ // Disallow parallel replica repair to prevent
+ // replicas from issuing incremental repair at
+ // same time.
+ "allow_parallel_replica_repair", "false",
+ // Run more aggressively since full repair is
+ // less restrictive about when it can run repair,
+ // so need to check more frequently to allow
+ // incremental to get an attempt in.
+ "min_repair_interval", "5s"))))
+ .set("auto_repair.enabled", "true")
+ .set("auto_repair.global_settings.repair_by_keyspace", "true")
+ .set("auto_repair.global_settings.repair_retry_backoff", "5s")
+ .set("auto_repair.repair_task_min_duration", "0s")
+ .set("auto_repair.repair_check_interval", "5s"))
+ .start();
+
+ cluster.schemaChange("CREATE KEYSPACE IF NOT EXISTS " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3};");
+ cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (pk int, ck text, v1 int, v2 int, PRIMARY KEY (pk, ck)) WITH read_repair='NONE'"));
+ }
+
+ @AfterClass
+ public static void tearDown()
+ {
+ cluster.close();
+ }
+
+ @Test
+ public void testScheduler() throws ParseException
+ {
+ // ensure there was no history of previous repair runs through the scheduler
+ Object[][] rows = cluster.coordinator(1).execute(String.format("SELECT repair_type, host_id, repair_start_ts, repair_finish_ts, repair_turn FROM %s.%s", DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY), ConsistencyLevel.QUORUM);
+ assertEquals(0, rows.length);
+
+ cluster.forEach(i -> i.runOnInstance(() -> {
+ try
+ {
+ AutoRepairService.setup();
+ AutoRepair.instance.setup();
+ }
+ catch (Exception e)
+ {
+ throw new RuntimeException(e);
+ }
+ }));
+
+ // validate that the repair ran on all nodes
+ cluster.forEach(i -> i.runOnInstance(() -> {
+ String broadcastAddress = FBUtilities.getJustBroadcastAddress().toString();
+
+ // Reduce sleeping if repair finishes quickly to speed up test but make it non-zero to provoke some
+ // contention.
+ AutoRepair.SLEEP_IF_REPAIR_FINISHES_QUICKLY = new DurationSpec.IntSecondsBound("2s");
+
+ AutoRepairMetrics incrementalMetrics = AutoRepairMetricsManager.getMetrics(AutoRepairConfig.RepairType.INCREMENTAL);
+ // Since the AutoRepair sleeps up to SLEEP_IF_REPAIR_FINISHES_QUICKLY if the repair finishes quickly,
+ // so the "nodeRepairTimeInSec" metric should at least be greater than or equal to
+ // SLEEP_IF_REPAIR_FINISHES_QUICKLY
+ Util.spinAssert(String.format("%s: AutoRepair has not yet completed one INCREMENTAL repair cycle", broadcastAddress),
+ greaterThanOrEqualTo(2L),
+ () -> incrementalMetrics.nodeRepairTimeInSec.getValue().longValue(),
+ 5,
+ TimeUnit.MINUTES);
+
+ // Expect some contention on incremental repair.
+ Util.spinAssert(String.format("%s: AutoRepair has not observed any replica contention in INCREMENTAL repair", broadcastAddress),
+ greaterThan(0L),
+ incrementalMetrics.repairDelayedByReplica::getCount,
+ 5,
+ TimeUnit.MINUTES);
+ // Do not expect any contention across schedules since allow_parallel_replica_repairs across schedules
+ // was not configured.
+ assertEquals(0L, incrementalMetrics.repairDelayedBySchedule.getCount());
+
+ AutoRepairMetrics fullMetrics = AutoRepairMetricsManager.getMetrics(AutoRepairConfig.RepairType.FULL);
+ Util.spinAssert(String.format("%s: AutoRepair has not yet completed one FULL repair cycle", broadcastAddress),
+ greaterThanOrEqualTo(2L),
+ () -> fullMetrics.nodeRepairTimeInSec.getValue().longValue(),
+ 5,
+ TimeUnit.MINUTES);
+
+ // No repair contention should be observed for full repair since allow_parallel_replica_repair was true
+ assertEquals(0L, fullMetrics.repairDelayedByReplica.getCount());
+ assertEquals(0L, fullMetrics.repairDelayedBySchedule.getCount());
+ }));
+
+ validate(AutoRepairConfig.RepairType.FULL.toString());
+ validate(AutoRepairConfig.RepairType.INCREMENTAL.toString());
+ }
+
+ private void validate(String repairType) throws ParseException
+ {
+ Object[][] rows = cluster.coordinator(1).execute(String.format("SELECT repair_type, host_id, repair_start_ts, repair_finish_ts, repair_turn FROM %s.%s where repair_type='%s'", DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, repairType), ConsistencyLevel.QUORUM);
+ assertEquals(3, rows.length);
+ for (int node = 0; node < rows.length; node++)
+ {
+ Object[] row = rows[node];
+ // repair_type
+ Assert.assertEquals(repairType, row[0].toString());
+ // host_id
+ Assert.assertNotNull(UUID.fromString(row[1].toString()));
+ // ensure there is a legit repair_start_ts and repair_finish_ts
+ sdf.parse(row[2].toString());
+ sdf.parse(row[3].toString());
+ // the reason why the repair was scheduled
+ Assert.assertNotNull(row[4]);
+ Assert.assertEquals("MY_TURN", row[4].toString());
+ }
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairTablePropertyTest.java b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairTablePropertyTest.java
new file mode 100644
index 0000000000..d7f068f137
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/repair/AutoRepairTablePropertyTest.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.repair;
+
+import java.io.IOException;
+
+import com.google.common.collect.ImmutableMap;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.SimpleBuilders;
+import org.apache.cassandra.db.rows.ColumnData;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.test.TestBaseImpl;
+import org.apache.cassandra.repair.autorepair.AutoRepairConfig;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.SchemaKeyspace;
+import org.apache.cassandra.schema.SchemaKeyspaceTables;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Distributed tests that verify "auto_repair" column behavior in system_schema.tables.
+ * <p>
+ * This test verifies that:
+ * 1. When auto-repair scheduling is enabled, the auto_repair column contains data
+ * 2. When auto-repair scheduling is disabled, the auto_repair column does not contain data
+ * <p>
+ * This test uses a distributed cluster to ensure JVM properties are properly initialized
+ * before the schema is loaded.
+ */
+public class AutoRepairTablePropertyTest extends TestBaseImpl
+{
+ private static Cluster cluster;
+
+ @BeforeClass
+ public static void init() throws IOException
+ {
+ // Ensure AUTOREPAIR_ENABLE is true so the auto_repair column exists in the schema
+ CassandraRelevantProperties.AUTOREPAIR_ENABLE.setBoolean(true);
+
+ // Configure a single-node cluster with auto_repair enabled
+ cluster = Cluster.build(1)
+ .withConfig(config -> config
+ .set("auto_repair",
+ ImmutableMap.of(
+ "repair_type_overrides",
+ ImmutableMap.of(AutoRepairConfig.RepairType.FULL.getConfigName(),
+ ImmutableMap.of(
+ "enabled", "true",
+ "initial_scheduler_delay", "1h")))))
+ .start();
+ }
+
+ @AfterClass
+ public static void teardown()
+ {
+ if (cluster != null)
+ cluster.close();
+ }
+
+ @Test
+ public void testAutoRepairColumnExistsWhenEnabled()
+ {
+ // Verify the auto_repair column exists in system_schema.tables
+ cluster.get(1).runOnInstance(() -> {
+ ColumnFamilyStore tables = Keyspace.open(SchemaConstants.SCHEMA_KEYSPACE_NAME)
+ .getColumnFamilyStore(SchemaKeyspaceTables.TABLES);
+
+ ColumnMetadata autoRepairColumn = tables.metadata().getColumn(ByteBufferUtil.bytes("auto_repair"));
+
+ // When AUTOREPAIR_ENABLE is true, the column should exist
+ assertTrue("auto_repair column should exist when AUTOREPAIR_ENABLE is true",
+ autoRepairColumn != null);
+ });
+ }
+
+ @Test
+ public void testAutoRepairColumnDataWrittenWhenSchedulerEnabled()
+ {
+ // When auto-repair scheduling is enabled, verify data is written to the auto_repair column
+ cluster.get(1).runOnInstance(() -> {
+ // Enable scheduling
+ DatabaseDescriptor.getAutoRepairConfig().setAutoRepairSchedulingEnabled(true);
+
+ ColumnFamilyStore tables = Keyspace.open(SchemaConstants.SCHEMA_KEYSPACE_NAME)
+ .getColumnFamilyStore(SchemaKeyspaceTables.TABLES);
+
+ ColumnMetadata autoRepairColumn = tables.metadata().getColumn(ByteBufferUtil.bytes("auto_repair"));
+
+ SimpleBuilders.RowBuilder builder = new SimpleBuilders.RowBuilder(tables.metadata(), "table_name");
+ SchemaKeyspace.addTableParamsToRowBuilder(tables.metadata().params, builder);
+ Row row = builder.build();
+
+ ColumnData data = row.getCell(autoRepairColumn);
+
+ // When scheduling is enabled, data should be written
+ assertTrue("auto_repair data should be written when scheduling is enabled", data != null);
+ });
+ }
+
+ @Test
+ public void testAutoRepairColumnDataNotWrittenWhenSchedulerDisabled()
+ {
+ // When auto-repair scheduling is disabled, verify no data is written to the auto_repair column
+ cluster.get(1).runOnInstance(() -> {
+ // Disable scheduling
+ DatabaseDescriptor.getAutoRepairConfig().setAutoRepairSchedulingEnabled(false);
+
+ ColumnFamilyStore tables = Keyspace.open(SchemaConstants.SCHEMA_KEYSPACE_NAME)
+ .getColumnFamilyStore(SchemaKeyspaceTables.TABLES);
+
+ ColumnMetadata autoRepairColumn =
+ tables.metadata().getColumn(ByteBufferUtil.bytes("auto_repair"));
+
+ SimpleBuilders.RowBuilder builder =
+ new SimpleBuilders.RowBuilder(tables.metadata(), "table_name");
+ SchemaKeyspace.addTableParamsToRowBuilder(tables.metadata().params, builder);
+ Row row = builder.build();
+
+ ColumnData data = row.getCell(autoRepairColumn);
+
+ // When scheduling is disabled, data should NOT be written
+ assertFalse("auto_repair data should NOT be written when scheduling is disabled", data != null);
+ });
+ }
+
+ @Test
+ public void testAutoRepairColumnInViewsSchema()
+ {
+ // Verify the auto_repair column behavior in system_schema.views as well
+ cluster.get(1).runOnInstance(() -> {
+ ColumnFamilyStore views = Keyspace.open(SchemaConstants.SCHEMA_KEYSPACE_NAME)
+ .getColumnFamilyStore(SchemaKeyspaceTables.VIEWS);
+
+ ColumnMetadata autoRepairColumn = views.metadata().getColumn(ByteBufferUtil.bytes("auto_repair"));
+
+ // When AUTOREPAIR_ENABLE is true, the column should exist in views table too
+ assertTrue("auto_repair column should exist in views schema when AUTOREPAIR_ENABLE is true",
+ autoRepairColumn != null);
+ });
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/AutoRepairDisabledSchemaUpgradeTest.java b/test/distributed/org/apache/cassandra/distributed/upgrade/AutoRepairDisabledSchemaUpgradeTest.java
new file mode 100644
index 0000000000..982971f6b8
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/upgrade/AutoRepairDisabledSchemaUpgradeTest.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.upgrade;
+
+import java.util.UUID;
+
+import com.vdurmont.semver4j.Semver;
+import com.vdurmont.semver4j.Semver.SemverType;
+import org.junit.Test;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.distributed.UpgradeableCluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.api.Feature;
+import org.apache.cassandra.distributed.impl.AbstractCluster;
+import org.apache.cassandra.distributed.shared.Versions;
+
+import static org.apache.cassandra.distributed.shared.Versions.Version;
+import static org.apache.cassandra.distributed.shared.Versions.find;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+/**
+ * Tests that upgrading from 5.0.7 (where auto-repair is not included) to the current version
+ * with AUTO_REPAIR_ENABLE=false maintains schema agreement between nodes.
+ * <p>
+ * This test verifies that when the auto-repair feature is disabled via the JVM property,
+ * the schema version remains consistent between upgraded and non-upgraded nodes,
+ * ensuring no schema disagreement occurs due to conditional auto-repair schema changes.
+ */
+public class AutoRepairDisabledSchemaUpgradeTest extends UpgradeTestBase
+{
+ private static final Semver v507 = new Semver("5.0.7", SemverType.STRICT);
+
+ @Test
+ public void testSchemaAgreementWithAutoRepairDisabled() throws Throwable
+ {
+ // Disable auto-repair feature to ensure no auto-repair schema changes are made
+ CassandraRelevantProperties.AUTOREPAIR_ENABLE.setBoolean(false);
+
+ Versions versions = find();
+ Version from = versions.get(v507);
+ Version to = AbstractCluster.CURRENT_VERSION;
+
+ assertNotNull("5.0.7 version not available - ensure dtest-5.0.7.jar is built", from);
+
+ try (UpgradeableCluster cluster = init(UpgradeableCluster.create(2, from,
+ config -> config.with(Feature.GOSSIP, Feature.NETWORK))))
+ {
+ // Create a simple table to ensure schema is propagated
+ cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int PRIMARY KEY, v int)");
+ cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, v) VALUES (1, 1)", ConsistencyLevel.ALL);
+
+ // Verify initial schema agreement before upgrade
+ UUID schemaBefore1 = cluster.get(1).schemaVersion();
+ UUID schemaBefore2 = cluster.get(2).schemaVersion();
+ assertEquals("Schema versions should match before upgrade", schemaBefore1, schemaBefore2);
+
+ // Upgrade only node 1 to current version
+ cluster.get(1).shutdown().get();
+ cluster.get(1).setVersion(to);
+ cluster.get(1).startup();
+
+ // Wait for schema to settle
+ Thread.sleep(5000);
+
... 6408 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org