You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ma...@apache.org on 2026/02/13 19:54:37 UTC
(cassandra) branch cep-45-mutation-tracking updated: CEP-45: Full Repair Support for Tracked Keyspaces
This is an automated email from the ASF dual-hosted git repository.
maedhroz pushed a commit to branch cep-45-mutation-tracking
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/cep-45-mutation-tracking by this push:
new d4826900b9 CEP-45: Full Repair Support for Tracked Keyspaces
d4826900b9 is described below
commit d4826900b96c8975e69adc78f71ffbbc938e4a8e
Author: Abe Ratnofsky <ab...@aber.io>
AuthorDate: Fri Dec 12 12:32:44 2025 -0500
CEP-45: Full Repair Support for Tracked Keyspaces
Tracked keyspaces cannot accept new data without first registering it in
the log. Any unreconciled data that isn't present in the log will break
read monotonicity, since mutation tracking uses a single data read and
can only read reconcile mutation IDs that are present in the log.
Full repair sync tasks also deliver data to replicas, and require
integration with the log just like imports do. The general design of this
integration is to give repair SyncTasks the same two-phase commit as
import transfers, where we stream SSTables to a pending directory, then
once sufficient streams complete successfully, we "activate" those
streams and move them out of the pending directory and into the live set.
patch by Caleb Rackliffe; reviewed by Abe Ratnofsky for CASSANDRA-21066
Co-authored-by: Caleb Rackliffe <ca...@gmail.com>
Co-authored-by: Abe Ratnofsky <ab...@aber.io>
---
.../org/apache/cassandra/db/ColumnFamilyStore.java | 1 +
.../org/apache/cassandra/db/SSTableImporter.java | 13 +-
.../CassandraEntireSSTableStreamReader.java | 2 +-
.../db/streaming/CassandraStreamManager.java | 3 +-
.../db/streaming/CassandraStreamReader.java | 4 +-
.../db/streaming/CassandraStreamReceiver.java | 8 +-
.../cassandra/io/sstable/format/SSTableWriter.java | 16 +-
src/java/org/apache/cassandra/net/Verb.java | 13 +-
.../cassandra/repair/AsymmetricRemoteSyncTask.java | 39 +-
.../repair/CompletableRemoteSyncTask.java | 6 +-
.../org/apache/cassandra/repair/LocalSyncTask.java | 31 +-
.../org/apache/cassandra/repair/RepairJob.java | 45 +-
.../cassandra/repair/RepairMessageVerbHandler.java | 5 +-
.../org/apache/cassandra/repair/RepairSession.java | 10 +-
.../cassandra/repair/StreamingRepairTask.java | 23 +-
.../cassandra/repair/SymmetricRemoteSyncTask.java | 33 +-
src/java/org/apache/cassandra/repair/SyncStat.java | 37 +-
src/java/org/apache/cassandra/repair/SyncTask.java | 73 +-
.../org/apache/cassandra/repair/SyncTasks.java | 106 +++
.../cassandra/repair/messages/SyncRequest.java | 39 +-
.../cassandra/repair/messages/SyncResponse.java | 54 +-
.../apache/cassandra/repair/state/SyncState.java | 17 +-
.../cassandra/replication/ActivatedTransfers.java | 10 +-
.../cassandra/replication/ActivationRequest.java | 260 ++++++
.../cassandra/replication/ActivationResponse.java | 94 ++
.../cassandra/replication/ActiveLogReconciler.java | 7 +-
.../cassandra/replication/CoordinatedTransfer.java | 558 ++++--------
.../cassandra/replication/CoordinatorLog.java | 59 +-
.../ImmutableCoordinatorLogOffsets.java | 14 -
.../replication/MutationTrackingService.java | 198 ++++-
.../replication/PendingLocalTransfer.java | 26 +-
.../org/apache/cassandra/replication/Shard.java | 22 +-
.../replication/TrackedImportTransfer.java | 421 +++++++++
...dTransfers.java => TrackedImportTransfers.java} | 18 +-
.../replication/TrackedRepairTransfer.java | 127 +++
.../cassandra/replication/TransferActivation.java | 197 -----
.../cassandra/replication/TransferFailed.java | 2 +-
...Transfers.java => TransferTrackingService.java} | 190 ++++-
.../replication/UnreconciledMutations.java | 10 +-
.../apache/cassandra/streaming/SessionSummary.java | 13 +
.../cassandra/streaming/StreamOperation.java | 13 +-
.../org/apache/cassandra/streaming/StreamPlan.java | 13 +-
.../apache/cassandra/streaming/StreamSession.java | 1 +
.../cassandra/tcm/ownership/ReplicaGroups.java | 9 +-
src/java/org/apache/cassandra/utils/Pair.java | 5 +
.../serialization/5.1/service.SyncComplete.bin | Bin 258 -> 258 bytes
.../cassandra/distributed/impl/Instance.java | 6 +-
.../test/tracking/BulkTransfersTest.java | 949 ---------------------
.../test/tracking/TrackedImportFailureTest.java | 409 +++++++++
.../test/tracking/TrackedImportTransferTest.java | 203 +++++
.../TrackedNonZeroCopyRepairTransferTest.java | 14 +-
.../test/tracking/TrackedRepairFailureTest.java | 390 +++++++++
.../TrackedRepairTransferSuccessTestBase.java | 178 ++++
.../tracking/TrackedRepairTransferTestBase.java | 234 +++++
.../test/tracking/TrackedTransferBounceTest.java | 124 +++
.../test/tracking/TrackedTransferTestBase.java | 546 ++++++++++++
.../TrackedZeroCopyRepairTransferTest.java | 14 +-
.../apache/cassandra/repair/LocalSyncTaskTest.java | 4 +-
.../org/apache/cassandra/repair/RepairJobTest.java | 4 +-
.../cassandra/repair/StreamingRepairTaskTest.java | 9 +-
.../repair/SymmetricRemoteSyncTaskTest.java | 2 +-
.../messages/RepairMessageSerializationsTest.java | 5 +-
...ava => ActivationRequestSerializationTest.java} | 33 +-
.../ActivationResponseSerializationTest.java | 84 ++
.../cassandra/replication/LocalTransfersTest.java | 344 --------
.../replication/MutationTrackingServiceTest.java | 265 ++++++
.../replication/TransferTrackingServiceTest.java | 342 ++++++++
.../org/apache/cassandra/schema/MockSchema.java | 41 +-
.../cassandra/service/SerializationsTest.java | 7 +-
69 files changed, 4860 insertions(+), 2192 deletions(-)
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index f4631e689a..f289e881b3 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -1690,6 +1690,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
*
* param @ filename - filename just flushed to disk
*/
+ @VisibleForTesting
public void addSSTable(SSTableReader sstable)
{
assert sstable.getColumnFamilyName().equals(name);
diff --git a/src/java/org/apache/cassandra/db/SSTableImporter.java b/src/java/org/apache/cassandra/db/SSTableImporter.java
index 1725dcbe15..1bdb168620 100644
--- a/src/java/org/apache/cassandra/db/SSTableImporter.java
+++ b/src/java/org/apache/cassandra/db/SSTableImporter.java
@@ -235,7 +235,7 @@ public class SSTableImporter
cfs.indexManager.buildSSTableAttachedIndexesBlocking(newSSTables);
if (isTracked)
- TrackedBulkTransfer.execute(cfs.keyspace.getName(), newSSTables);
+ MutationTrackingService.instance.executeTransfers(cfs.keyspace.getName(), newSSTables, ConsistencyLevel.ALL);
else
cfs.getTracker().addSSTables(newSSTables);
@@ -253,17 +253,6 @@ public class SSTableImporter
return failedDirectories;
}
- /**
- * TODO: Support user-defined consistency level for import, for import with replicas down
- */
- private static class TrackedBulkTransfer
- {
- private static void execute(String keyspace, Set<SSTableReader> sstables)
- {
- MutationTrackingService.instance.executeTransfers(keyspace, sstables, ConsistencyLevel.ALL);
- }
- }
-
/**
* Check the state of this node and throws an {@link InterruptedException} if it is currently draining
*
diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamReader.java b/src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamReader.java
index 535c5ff7de..0defc3e822 100644
--- a/src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamReader.java
+++ b/src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamReader.java
@@ -160,7 +160,7 @@ public class CassandraEntireSSTableStreamReader implements IStreamReader
private File getDataDir(ColumnFamilyStore cfs, long totalSize) throws IOException
{
- boolean isTracked = cfs.metadata().replicationType().isTracked();
+ boolean isTracked = cfs.metadata().replicationType().isTracked() && session.streamOperation().isTrackable();
Directories.DataDirectory localDir = cfs.getDirectories().getWriteableLocation(totalSize);
if (localDir == null)
diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamManager.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamManager.java
index 2fc802b036..5aece1dc7e 100644
--- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamManager.java
+++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamManager.java
@@ -130,9 +130,10 @@ public class CassandraStreamManager implements TableStreamManager
Set<SSTableReader> sstables = Sets.newHashSet();
SSTableIntervalTree intervalTree = buildSSTableIntervalTree(ImmutableList.copyOf(view.select(SSTableSet.CANONICAL)));
Predicate<SSTableReader> predicate;
+ // reconciledKeyspaceOffsets are only included when mutation logs are streamed, since we include logs
+ // for all unreconciled mutations, and SSTables for all reconciled mutations
if (reconciledKeyspaceOffsets != null)
{
- // TODO: relax these restrictions as repair support is add
Preconditions.checkArgument(previewKind == PreviewKind.NONE);
Preconditions.checkArgument(pendingRepair == ActiveRepairService.NO_PENDING_REPAIR);
predicate = getSSTablePredicateForKeyspaceRanges(reconciledKeyspaceOffsets);
diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java
index 579fc9b29b..cbfc3bfe7c 100644
--- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java
+++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java
@@ -62,7 +62,6 @@ import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.streaming.ProgressInfo;
-import org.apache.cassandra.streaming.StreamOperation;
import org.apache.cassandra.streaming.StreamReceivedOutOfTokenRangeException;
import org.apache.cassandra.streaming.StreamReceiver;
import org.apache.cassandra.streaming.StreamSession;
@@ -191,9 +190,8 @@ public class CassandraStreamReader implements IStreamReader
ILifecycleTransaction txn = createTxn();
RangeAwareSSTableWriter writer;
- if (session.streamOperation() == StreamOperation.TRACKED_TRANSFER)
+ if (cfs.metadata().replicationType().isTracked() && session.streamOperation().isTrackable())
{
- Preconditions.checkState(cfs.metadata().replicationType().isTracked());
writer = new RangeAwarePendingSSTableWriter(cfs, estimatedKeys, repairedAt, pendingRepair, coordinatorLogOffsets, format, sstableLevel, totalSize, txn, getHeader(cfs.metadata()), session.planId());
}
else
diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java
index 89f869ee06..62f53d7363 100644
--- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java
+++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java
@@ -54,7 +54,6 @@ import org.apache.cassandra.service.accord.TimeOnlyRequestBookkeeping.LatencyReq
import org.apache.cassandra.replication.MutationTrackingService;
import org.apache.cassandra.replication.PendingLocalTransfer;
import org.apache.cassandra.streaming.IncomingStream;
-import org.apache.cassandra.streaming.StreamOperation;
import org.apache.cassandra.streaming.StreamReceiver;
import org.apache.cassandra.streaming.StreamSession;
import org.apache.cassandra.tcm.ClusterMetadata;
@@ -135,9 +134,8 @@ public class CassandraStreamReceiver implements StreamReceiver
sstables.addAll(finished);
receivedEntireSSTable = file.isEntireSSTable();
- if (session.streamOperation() == StreamOperation.TRACKED_TRANSFER)
+ if (cfs.metadata().replicationType().isTracked() && session.streamOperation().isTrackable())
{
- Preconditions.checkState(cfs.metadata().replicationType().isTracked());
PendingLocalTransfer transfer = new PendingLocalTransfer(cfs.metadata().id, session.planId(), sstables);
MutationTrackingService.instance.received(transfer);
}
@@ -266,8 +264,8 @@ public class CassandraStreamReceiver implements StreamReceiver
// add sstables (this will build non-SSTable-attached secondary indexes too, see CASSANDRA-10130)
logger.debug("[Stream #{}] Received {} sstables from {} ({})", session.planId(), readers.size(), session.peer, readers);
- // Don't mark as live until activated by the stream coordinator
- if (session.streamOperation() == StreamOperation.TRACKED_TRANSFER)
+ // SSTables involved in a coordinated transfer become live when the transfer is activated
+ if (cfs.metadata().replicationType().isTracked() && session.streamOperation().isTrackable())
return;
cfs.addSSTables(readers);
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java
index 2506d0a9a6..979cacc8af 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java
@@ -35,15 +35,12 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
-import org.apache.cassandra.replication.ImmutableCoordinatorLogOffsets;
-import org.apache.cassandra.replication.MutationTrackingService;
-import org.apache.cassandra.service.ActiveRepairService;
-import org.apache.cassandra.utils.Clock;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.lifecycle.ILifecycleTransaction;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.dht.AbstractBounds;
@@ -62,6 +59,10 @@ import org.apache.cassandra.io.sstable.metadata.MetadataComponent;
import org.apache.cassandra.io.sstable.metadata.MetadataType;
import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
import org.apache.cassandra.io.util.MmappedRegionsCache;
+import org.apache.cassandra.replication.ImmutableCoordinatorLogOffsets;
+import org.apache.cassandra.replication.MutationTrackingService;
+import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.utils.Clock;
import org.apache.cassandra.utils.Throwables;
import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.utils.concurrent.Transactional;
@@ -336,9 +337,10 @@ public abstract class SSTableWriter extends SSTable implements Transactional
protected Map<MetadataType, MetadataComponent> finalizeMetadata()
{
- // Migration from incremental repair to mutation tracking will be supported, but support for mixing
- // incremental repair and mutation tracking is not planned
- if (metadata().replicationType().isTracked() && repairedAt == ActiveRepairService.UNREPAIRED_SSTABLE)
+ // Reconciliation should not occur before activation for coordinated transfer streams for tracked keyspaces.
+ boolean reconcile = txn.opType() != OperationType.STREAM;
+
+ if (metadata().replicationType().isTracked() && repairedAt == ActiveRepairService.UNREPAIRED_SSTABLE && reconcile)
{
Preconditions.checkState(Objects.equals(pendingRepair, ActiveRepairService.NO_PENDING_REPAIR));
if (MutationTrackingService.instance.isDurablyReconciled(coordinatorLogOffsets))
diff --git a/src/java/org/apache/cassandra/net/Verb.java b/src/java/org/apache/cassandra/net/Verb.java
index 6719e71e27..a30a0361c9 100644
--- a/src/java/org/apache/cassandra/net/Verb.java
+++ b/src/java/org/apache/cassandra/net/Verb.java
@@ -74,12 +74,13 @@ import org.apache.cassandra.repair.messages.SyncRequest;
import org.apache.cassandra.repair.messages.SyncResponse;
import org.apache.cassandra.repair.messages.ValidationRequest;
import org.apache.cassandra.repair.messages.ValidationResponse;
+import org.apache.cassandra.replication.ActivationResponse;
import org.apache.cassandra.replication.BroadcastLogOffsets;
import org.apache.cassandra.replication.ForwardedWrite;
-import org.apache.cassandra.replication.LocalTransfers;
+import org.apache.cassandra.replication.TransferTrackingService;
import org.apache.cassandra.replication.PullMutationsRequest;
import org.apache.cassandra.replication.PushMutationRequest;
-import org.apache.cassandra.replication.TransferActivation;
+import org.apache.cassandra.replication.ActivationRequest;
import org.apache.cassandra.replication.TransferFailed;
import org.apache.cassandra.schema.SchemaMutationsSerializer;
import org.apache.cassandra.schema.SchemaPullVerbHandler;
@@ -339,10 +340,10 @@ public enum Verb
TRACKED_SUMMARY_RSP (910, P2, readTimeout, REQUEST_RESPONSE, () -> TrackedSummaryResponse.serializer, () -> TrackedSummaryResponse.verbHandler ),
TRACKED_SUMMARY_REQ (911, P3, readTimeout, READ, () -> TrackedRead.SummaryRequest.serializer, () -> TrackedRead.verbHandler, TRACKED_SUMMARY_RSP ),
- TRACKED_TRANSFER_ACTIVATE_RSP (912, P1, repairTimeout, REQUEST_RESPONSE, () -> NoPayload.serializer, RESPONSE_HANDLER),
- TRACKED_TRANSFER_ACTIVATE_REQ (913, P1, repairTimeout, ANTI_ENTROPY, () -> TransferActivation.serializer, () -> TransferActivation.verbHandler, TRACKED_TRANSFER_ACTIVATE_RSP),
- TRACKED_TRANSFER_FAILED_RSP (914, P1, repairTimeout, REQUEST_RESPONSE, () -> NoPayload.serializer, RESPONSE_HANDLER),
- TRACKED_TRANSFER_FAILED_REQ (915, P1, repairTimeout, ANTI_ENTROPY, () -> TransferFailed.serializer, () -> LocalTransfers.verbHandler, TRACKED_TRANSFER_FAILED_RSP),
+ TRACKED_TRANSFER_ACTIVATE_RSP (912, P1, repairTimeout, REQUEST_RESPONSE, () -> ActivationResponse.serializer, RESPONSE_HANDLER),
+ TRACKED_TRANSFER_ACTIVATE_REQ (913, P1, repairTimeout, ANTI_ENTROPY, () -> ActivationRequest.serializer, () -> ActivationRequest.verbHandler, TRACKED_TRANSFER_ACTIVATE_RSP),
+ TRACKED_TRANSFER_FAILED_RSP (914, P1, repairTimeout, REQUEST_RESPONSE, () -> NoPayload.serializer, RESPONSE_HANDLER),
+ TRACKED_TRANSFER_FAILED_REQ (915, P1, repairTimeout, ANTI_ENTROPY, () -> TransferFailed.serializer, () -> TransferTrackingService.verbHandler, TRACKED_TRANSFER_FAILED_RSP),
// accord
ACCORD_SIMPLE_RSP (119, P2, writeTimeout, IMMEDIATE, () -> accordEmbedded(EnumSerializer.simpleReply), AccordService::responseHandlerOrNoop ),
diff --git a/src/java/org/apache/cassandra/repair/AsymmetricRemoteSyncTask.java b/src/java/org/apache/cassandra/repair/AsymmetricRemoteSyncTask.java
index 475c389678..fa42242387 100644
--- a/src/java/org/apache/cassandra/repair/AsymmetricRemoteSyncTask.java
+++ b/src/java/org/apache/cassandra/repair/AsymmetricRemoteSyncTask.java
@@ -18,16 +18,21 @@
package org.apache.cassandra.repair;
+import java.util.ArrayList;
+import java.util.Collection;
import java.util.List;
+import com.google.common.base.Preconditions;
+
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.RepairException;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.repair.messages.SyncRequest;
+import org.apache.cassandra.repair.messages.SyncResponse;
+import org.apache.cassandra.replication.ShortMutationId;
import org.apache.cassandra.streaming.PreviewKind;
-import org.apache.cassandra.streaming.SessionSummary;
import org.apache.cassandra.tracing.Tracing;
/**
@@ -38,29 +43,46 @@ import org.apache.cassandra.tracing.Tracing;
*/
public class AsymmetricRemoteSyncTask extends SyncTask implements CompletableRemoteSyncTask
{
- public AsymmetricRemoteSyncTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort to, InetAddressAndPort from, List<Range<Token>> differences, PreviewKind previewKind)
+ public AsymmetricRemoteSyncTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort to, InetAddressAndPort from, List<Range<Token>> differences, PreviewKind previewKind, ShortMutationId transferId)
+ {
+ super(ctx, desc, to, from, differences, previewKind, transferId);
+ }
+
+ @Override
+ public SyncTask withRanges(Collection<Range<Token>> newRanges)
+ {
+ List<Range<Token>> rangeList = newRanges instanceof List ? (List<Range<Token>>) newRanges : new ArrayList<>(newRanges);
+ return new AsymmetricRemoteSyncTask(ctx, desc, nodePair.coordinator, nodePair.peer, rangeList, previewKind, transferId);
+ }
+
+ @Override
+ public SyncTask withTransferId(ShortMutationId transferId)
{
- super(ctx, desc, to, from, differences, previewKind);
+ Preconditions.checkState(this.transferId == null);
+ return new AsymmetricRemoteSyncTask(ctx, desc, nodePair.coordinator, nodePair.peer, rangesToSync, previewKind, transferId);
}
public void startSync()
{
InetAddressAndPort local = ctx.broadcastAddressAndPort();
- SyncRequest request = new SyncRequest(desc, local, nodePair.coordinator, nodePair.peer, rangesToSync, previewKind, true);
+ SyncRequest request = new SyncRequest(desc, local, nodePair.coordinator, nodePair.peer, rangesToSync, previewKind, true, transferId);
String message = String.format("Forwarding streaming repair of %d ranges to %s (to be streamed with %s)", request.ranges.size(), request.src, request.dst);
Tracing.traceRepair(message);
sendRequest(request, request.src);
}
- public void syncComplete(boolean success, List<SessionSummary> summaries)
+ public void syncComplete(SyncResponse response)
{
- if (success)
+ if (response.success)
{
- trySuccess(stat.withSummaries(summaries));
+ trySuccess(stat.withSummaries(response.summaries, response.planId, response.transferId));
}
else
{
- tryFailure(RepairException.warn(desc, previewKind, String.format("Sync failed between %s and %s", nodePair.coordinator, nodePair.peer)));
+ String message = transferId == null
+ ? String.format("Sync failed between %s and %s", nodePair.coordinator, nodePair.peer)
+ : String.format("Sync failed between %s and %s for transfer %s", nodePair.coordinator, nodePair.peer, transferId);
+ tryFailure(RepairException.warn(desc, previewKind, message));
}
}
@@ -70,6 +92,7 @@ public class AsymmetricRemoteSyncTask extends SyncTask implements CompletableRem
return "AsymmetricRemoteSyncTask{" +
"rangesToSync=" + rangesToSync +
", nodePair=" + nodePair +
+ ", transfer ID=" + transferId +
'}';
}
}
diff --git a/src/java/org/apache/cassandra/repair/CompletableRemoteSyncTask.java b/src/java/org/apache/cassandra/repair/CompletableRemoteSyncTask.java
index c4fe6c8b20..66791315c2 100644
--- a/src/java/org/apache/cassandra/repair/CompletableRemoteSyncTask.java
+++ b/src/java/org/apache/cassandra/repair/CompletableRemoteSyncTask.java
@@ -18,11 +18,9 @@
package org.apache.cassandra.repair;
-import java.util.List;
-
-import org.apache.cassandra.streaming.SessionSummary;
+import org.apache.cassandra.repair.messages.SyncResponse;
public interface CompletableRemoteSyncTask
{
- void syncComplete(boolean success, List<SessionSummary> summaries);
+ void syncComplete(SyncResponse response);
}
diff --git a/src/java/org/apache/cassandra/repair/LocalSyncTask.java b/src/java/org/apache/cassandra/repair/LocalSyncTask.java
index 379ba4b2a1..bd15e86baa 100644
--- a/src/java/org/apache/cassandra/repair/LocalSyncTask.java
+++ b/src/java/org/apache/cassandra/repair/LocalSyncTask.java
@@ -17,6 +17,8 @@
*/
package org.apache.cassandra.repair;
+import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -30,6 +32,7 @@ import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.RangesAtEndpoint;
+import org.apache.cassandra.replication.ShortMutationId;
import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.streaming.ProgressInfo;
import org.apache.cassandra.streaming.StreamEvent;
@@ -66,9 +69,9 @@ public class LocalSyncTask extends SyncTask implements StreamEventHandler
public LocalSyncTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort local, InetAddressAndPort remote,
List<Range<Token>> diff, TimeUUID pendingRepair,
- boolean requestRanges, boolean transferRanges, PreviewKind previewKind)
+ boolean requestRanges, boolean transferRanges, PreviewKind previewKind, ShortMutationId transferId)
{
- super(ctx, desc, local, remote, diff, previewKind);
+ super(ctx, desc, local, remote, diff, previewKind, transferId);
Preconditions.checkArgument(requestRanges || transferRanges, "Nothing to do in a sync job");
Preconditions.checkArgument(local.equals(ctx.broadcastAddressAndPort()));
@@ -77,6 +80,27 @@ public class LocalSyncTask extends SyncTask implements StreamEventHandler
this.transferRanges = transferRanges;
}
+ public LocalSyncTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort local, InetAddressAndPort remote,
+ List<Range<Token>> diff, TimeUUID pendingRepair,
+ boolean requestRanges, boolean transferRanges, PreviewKind previewKind)
+ {
+ this(ctx, desc, local, remote, diff, pendingRepair, requestRanges, transferRanges, previewKind, null);
+ }
+
+ @Override
+ public SyncTask withRanges(Collection<Range<Token>> newRanges)
+ {
+ List<Range<Token>> rangeList = newRanges instanceof List ? (List<Range<Token>>) newRanges : new ArrayList<>(newRanges);
+ return new LocalSyncTask(ctx, desc, nodePair.coordinator, nodePair.peer, rangeList, pendingRepair, requestRanges, transferRanges, previewKind, transferId);
+ }
+
+ @Override
+ public SyncTask withTransferId(ShortMutationId transferId)
+ {
+ Preconditions.checkState(this.transferId == null);
+ return new LocalSyncTask(ctx, desc, nodePair.coordinator, nodePair.peer, rangesToSync, pendingRepair, requestRanges, transferRanges, previewKind, transferId);
+ }
+
@VisibleForTesting
StreamPlan createStreamPlan()
{
@@ -167,7 +191,7 @@ public class LocalSyncTask extends SyncTask implements StreamEventHandler
status, desc.sessionId, nodePair.coordinator, nodePair.peer, desc.columnFamily);
logger.info("{} {}", previewKind.logPrefix(desc.sessionId), message);
Tracing.traceRepair(message);
- trySuccess(result.hasAbortedSession() ? stat : stat.withSummaries(result.createSummaries()));
+ trySuccess(result.hasAbortedSession() ? stat : stat.withSummaries(result.createSummaries(), result.planId, transferId));
finished();
}
}
@@ -190,6 +214,7 @@ public class LocalSyncTask extends SyncTask implements StreamEventHandler
", transferRanges=" + transferRanges +
", rangesToSync=" + rangesToSync +
", nodePair=" + nodePair +
+ ", transfer ID=" + transferId +
'}';
}
diff --git a/src/java/org/apache/cassandra/repair/RepairJob.java b/src/java/org/apache/cassandra/repair/RepairJob.java
index 28640a60a4..0a0411a982 100644
--- a/src/java/org/apache/cassandra/repair/RepairJob.java
+++ b/src/java/org/apache/cassandra/repair/RepairJob.java
@@ -50,6 +50,7 @@ import org.apache.cassandra.repair.asymmetric.HostDifferences;
import org.apache.cassandra.repair.asymmetric.PreferedNodeFilter;
import org.apache.cassandra.repair.asymmetric.ReduceHelper;
import org.apache.cassandra.repair.state.JobState;
+import org.apache.cassandra.replication.TransferTrackingService;
import org.apache.cassandra.schema.SystemDistributedKeyspace;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.accord.IAccordService;
@@ -120,7 +121,6 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
if ((!session.repairData && !session.repairPaxos) && !metadata.requiresAccordSupport())
throw new IllegalArgumentException(String.format("Cannot run accord only repair on %s.%s, which isn't configured for accord operations", cfs.keyspace.getName(), cfs.name));
-
}
public long getNowInSeconds()
@@ -136,6 +136,11 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
}
}
+ public Collection<SyncTask> getSyncTasks()
+ {
+ return syncTasks;
+ }
+
@Override
public void run()
{
@@ -254,6 +259,11 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
// that there are in memory at once. When all validations complete, submit sync tasks out of the scheduler.
syncResults = session.validationScheduler.schedule(() -> createSyncTasks(accordRepair, allSnapshotTasks, allEndpoints), taskExecutor)
.flatMap(this::executeTasks, taskExecutor);
+
+ // For tracked keyspaces, we need to ensure sync'd data is present in the log
+ boolean isTracked = cfs.metadata().replicationType().isTracked();
+ if (isTracked)
+ syncResults = TransferTrackingService.instance().onRepairSyncCompletion(this, syncResults, taskExecutor);
}
else
{
@@ -306,7 +316,7 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
}, taskExecutor);
}
- private Future<List<SyncTask>> createSyncTasks(Future<AccordRepairResult> accordRepair, Future<?> allSnapshotTasks, List<InetAddressAndPort> allEndpoints)
+ private Future<SyncTasks> createSyncTasks(Future<AccordRepairResult> accordRepair, Future<?> allSnapshotTasks, List<InetAddressAndPort> allEndpoints)
{
Future<List<TreeResponse>> treeResponses;
if (allSnapshotTasks != null)
@@ -330,9 +340,17 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
return a;
});
- return treeResponses.map(session.optimiseStreams && !session.pullRepair
- ? this::createOptimisedSyncingSyncTasks
- : this::createStandardSyncTasks, taskExecutor);
+ return treeResponses.map(trees -> {
+ List<SyncTask> syncTasks;
+ if (session.optimiseStreams && !session.pullRepair)
+ syncTasks = createOptimisedSyncingSyncTasks(trees);
+ else
+ syncTasks = createStandardSyncTasks(trees);
+
+ return ks.getMetadata().params.replicationType.isTracked()
+ ? SyncTasks.tracked(ks, syncTasks)
+ : SyncTasks.untracked(syncTasks);
+ }, taskExecutor);
}
public synchronized void abort(@Nullable Throwable reason)
@@ -413,18 +431,18 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
continue;
task = new LocalSyncTask(ctx, desc, self.endpoint, remote.endpoint, differences, isIncremental ? desc.parentSessionId : null,
- requestRanges, transferRanges, previewKind);
+ requestRanges, transferRanges, previewKind, null);
}
else if (isTransient.test(r1.endpoint) || isTransient.test(r2.endpoint))
{
// Stream only from transient replica
TreeResponse streamFrom = isTransient.test(r1.endpoint) ? r1 : r2;
TreeResponse streamTo = isTransient.test(r1.endpoint) ? r2 : r1;
- task = new AsymmetricRemoteSyncTask(ctx, desc, streamTo.endpoint, streamFrom.endpoint, differences, previewKind);
+ task = new AsymmetricRemoteSyncTask(ctx, desc, streamTo.endpoint, streamFrom.endpoint, differences, previewKind, null);
}
else
{
- task = new SymmetricRemoteSyncTask(ctx, desc, r1.endpoint, r2.endpoint, differences, previewKind);
+ task = new SymmetricRemoteSyncTask(ctx, desc, r1.endpoint, r2.endpoint, differences, previewKind, null);
}
syncTasks.add(task);
}
@@ -437,7 +455,7 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
}
@VisibleForTesting
- Future<List<SyncStat>> executeTasks(List<SyncTask> tasks)
+ Future<List<SyncStat>> executeTasks(SyncTasks tasks)
{
try
{
@@ -447,10 +465,13 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
if (!tasks.isEmpty())
state.phase.streamSubmitted();
+ if (cfs.metadata().replicationType().isTracked())
+ TransferTrackingService.instance().onRepairSyncExecution(tasks);
+
for (SyncTask task : tasks)
{
if (!task.isLocal())
- session.trackSyncCompletion(Pair.create(desc, task.nodePair()), (CompletableRemoteSyncTask) task);
+ session.trackSyncCompletion(new SyncTask.SyncTaskId(desc, task.nodePair(), task.transferId), (CompletableRemoteSyncTask) task);
taskExecutor.execute(task);
}
@@ -535,11 +556,11 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
if (address.equals(local))
{
task = new LocalSyncTask(ctx, desc, address, fetchFrom, toFetch, isIncremental ? desc.parentSessionId : null,
- true, false, previewKind);
+ true, false, previewKind, null);
}
else
{
- task = new AsymmetricRemoteSyncTask(ctx, desc, address, fetchFrom, toFetch, previewKind);
+ task = new AsymmetricRemoteSyncTask(ctx, desc, address, fetchFrom, toFetch, previewKind, null);
}
syncTasks.add(task);
diff --git a/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java b/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java
index 200c246fb3..a0ed59d979 100644
--- a/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java
+++ b/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java
@@ -311,7 +311,7 @@ public class RepairMessageVerbHandler implements IVerbHandler<RepairMessage>
logErrorAndSendFailureResponse("Unknown repair " + desc.parentSessionId, message);
return;
}
- SyncState state = new SyncState(ctx.clock(), desc, request.initiator, request.src, request.dst);
+ SyncState state = new SyncState(ctx.clock(), desc, request.initiator, request.src, request.dst, request.transferId);
if (!register(message, participate, state,
participate::register,
participate::sync))
@@ -324,7 +324,8 @@ public class RepairMessageVerbHandler implements IVerbHandler<RepairMessage>
request.ranges,
isIncremental(desc.parentSessionId) ? desc.parentSessionId : null,
request.previewKind,
- request.asymmetric);
+ request.asymmetric,
+ request.transferId);
task.run();
sendAck(message);
}
diff --git a/src/java/org/apache/cassandra/repair/RepairSession.java b/src/java/org/apache/cassandra/repair/RepairSession.java
index 106950bb4a..8e52fbf6f3 100644
--- a/src/java/org/apache/cassandra/repair/RepairSession.java
+++ b/src/java/org/apache/cassandra/repair/RepairSession.java
@@ -134,7 +134,7 @@ public class RepairSession extends AsyncFuture<RepairSessionResult> implements I
// Each validation task waits response from replica in validating ConcurrentMap (keyed by CF name and endpoint address)
private final ConcurrentMap<Pair<RepairJobDesc, InetAddressAndPort>, ValidationTask> validating = new ConcurrentHashMap<>();
// Remote syncing jobs wait response in syncingTasks map
- private final ConcurrentMap<Pair<RepairJobDesc, SyncNodePair>, CompletableRemoteSyncTask> syncingTasks = new ConcurrentHashMap<>();
+ private final ConcurrentMap<SyncTask.SyncTaskId, CompletableRemoteSyncTask> syncingTasks = new ConcurrentHashMap<>();
// Tasks(snapshot, validate request, differencing, ...) are run on taskExecutor
public final SafeExecutor taskExecutor;
@@ -224,7 +224,7 @@ public class RepairSession extends AsyncFuture<RepairSessionResult> implements I
validating.put(key, task);
}
- public synchronized void trackSyncCompletion(Pair<RepairJobDesc, SyncNodePair> key, CompletableRemoteSyncTask task)
+ public synchronized void trackSyncCompletion(SyncTask.SyncTaskId key, CompletableRemoteSyncTask task)
{
if (terminated)
return;
@@ -269,7 +269,7 @@ public class RepairSession extends AsyncFuture<RepairSessionResult> implements I
public void syncComplete(RepairJobDesc desc, Message<SyncResponse> message)
{
SyncNodePair nodes = message.payload.nodes;
- CompletableRemoteSyncTask task = syncingTasks.remove(Pair.create(desc, nodes));
+ CompletableRemoteSyncTask task = syncingTasks.remove(new SyncTask.SyncTaskId(desc, nodes, message.payload.transferId));
// replies without a callback get dropped, so if in mixed mode this should be ignored
ctx.messaging().send(message.emptyResponse(), message.from());
if (task == null)
@@ -277,11 +277,11 @@ public class RepairSession extends AsyncFuture<RepairSessionResult> implements I
if (logger.isDebugEnabled())
logger.debug("{} Repair completed between {} and {} on {}", previewKind.logPrefix(getId()), nodes.coordinator, nodes.peer, desc.columnFamily);
- task.syncComplete(message.payload.success, message.payload.summaries);
+ task.syncComplete(message.payload);
}
@VisibleForTesting
- Map<Pair<RepairJobDesc, SyncNodePair>, CompletableRemoteSyncTask> getSyncingTasks()
+ Map<SyncTask.SyncTaskId, CompletableRemoteSyncTask> getSyncingTasks()
{
return Collections.unmodifiableMap(syncingTasks);
}
diff --git a/src/java/org/apache/cassandra/repair/StreamingRepairTask.java b/src/java/org/apache/cassandra/repair/StreamingRepairTask.java
index 0f84d66893..cb67c2ebfc 100644
--- a/src/java/org/apache/cassandra/repair/StreamingRepairTask.java
+++ b/src/java/org/apache/cassandra/repair/StreamingRepairTask.java
@@ -20,6 +20,8 @@ package org.apache.cassandra.repair;
import java.util.Collections;
import java.util.Collection;
+import javax.annotation.Nullable;
+
import com.google.common.annotations.VisibleForTesting;
import org.apache.cassandra.locator.RangesAtEndpoint;
@@ -32,6 +34,7 @@ import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.repair.messages.RepairMessage;
import org.apache.cassandra.repair.messages.SyncResponse;
import org.apache.cassandra.repair.state.SyncState;
+import org.apache.cassandra.replication.ShortMutationId;
import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.streaming.StreamEvent;
import org.apache.cassandra.streaming.StreamEventHandler;
@@ -65,7 +68,20 @@ public class StreamingRepairTask implements Runnable, StreamEventHandler
private final TimeUUID pendingRepair;
private final PreviewKind previewKind;
- public StreamingRepairTask(SharedContext ctx, SyncState state, RepairJobDesc desc, InetAddressAndPort initiator, InetAddressAndPort src, InetAddressAndPort dst, Collection<Range<Token>> ranges, TimeUUID pendingRepair, PreviewKind previewKind, boolean asymmetric)
+ @Nullable
+ private final ShortMutationId transferId;
+
+ public StreamingRepairTask(SharedContext ctx,
+ SyncState state,
+ RepairJobDesc desc,
+ InetAddressAndPort initiator,
+ InetAddressAndPort src,
+ InetAddressAndPort dst,
+ Collection<Range<Token>> ranges,
+ TimeUUID pendingRepair,
+ PreviewKind previewKind,
+ boolean asymmetric,
+ ShortMutationId transferId)
{
this.ctx = ctx;
this.state = state;
@@ -77,6 +93,7 @@ public class StreamingRepairTask implements Runnable, StreamEventHandler
this.asymmetric = asymmetric;
this.pendingRepair = pendingRepair;
this.previewKind = previewKind;
+ this.transferId = transferId;
}
public void run()
@@ -120,7 +137,7 @@ public class StreamingRepairTask implements Runnable, StreamEventHandler
{
logger.info("[repair #{}] streaming task succeed, returning response to {}", desc.sessionId, initiator);
this.state.phase.success();
- RepairMessage.sendMessageWithRetries(ctx, new SyncResponse(desc, src, dst, true, state.createSummaries()), SYNC_RSP, initiator);
+ RepairMessage.sendMessageWithRetries(ctx, new SyncResponse(desc, src, dst, true, state.createSummaries(), state.planId, transferId), SYNC_RSP, initiator);
}
/**
@@ -130,6 +147,6 @@ public class StreamingRepairTask implements Runnable, StreamEventHandler
public void onFailure(Throwable t)
{
this.state.phase.fail(t);
- RepairMessage.sendMessageWithRetries(ctx, new SyncResponse(desc, src, dst, false, Collections.emptyList()), SYNC_RSP, initiator);
+ RepairMessage.sendMessageWithRetries(ctx, new SyncResponse(desc, src, dst, false, Collections.emptyList(), null, transferId), SYNC_RSP, initiator);
}
}
diff --git a/src/java/org/apache/cassandra/repair/SymmetricRemoteSyncTask.java b/src/java/org/apache/cassandra/repair/SymmetricRemoteSyncTask.java
index 95b8959407..ed954bd627 100644
--- a/src/java/org/apache/cassandra/repair/SymmetricRemoteSyncTask.java
+++ b/src/java/org/apache/cassandra/repair/SymmetricRemoteSyncTask.java
@@ -17,6 +17,8 @@
*/
package org.apache.cassandra.repair;
+import java.util.ArrayList;
+import java.util.Collection;
import java.util.List;
import com.google.common.base.Preconditions;
@@ -28,8 +30,9 @@ import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.RepairException;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.repair.messages.SyncRequest;
+import org.apache.cassandra.repair.messages.SyncResponse;
+import org.apache.cassandra.replication.ShortMutationId;
import org.apache.cassandra.streaming.PreviewKind;
-import org.apache.cassandra.streaming.SessionSummary;
import org.apache.cassandra.tracing.Tracing;
/**
@@ -42,16 +45,30 @@ public class SymmetricRemoteSyncTask extends SyncTask implements CompletableRemo
{
private static final Logger logger = LoggerFactory.getLogger(SymmetricRemoteSyncTask.class);
- public SymmetricRemoteSyncTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort r1, InetAddressAndPort r2, List<Range<Token>> differences, PreviewKind previewKind)
+ public SymmetricRemoteSyncTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort r1, InetAddressAndPort r2, List<Range<Token>> differences, PreviewKind previewKind, ShortMutationId transferId)
{
- super(ctx, desc, r1, r2, differences, previewKind);
+ super(ctx, desc, r1, r2, differences, previewKind, transferId);
+ }
+
+ @Override
+ public SyncTask withRanges(Collection<Range<Token>> newRanges)
+ {
+ List<Range<Token>> rangeList = newRanges instanceof List ? (List<Range<Token>>) newRanges : new ArrayList<>(newRanges);
+ return new SymmetricRemoteSyncTask(ctx, desc, nodePair.coordinator, nodePair.peer, rangeList, previewKind, transferId);
+ }
+
+ @Override
+ public SyncTask withTransferId(ShortMutationId transferId)
+ {
+ Preconditions.checkState(this.transferId == null);
+ return new SymmetricRemoteSyncTask(ctx, desc, nodePair.coordinator, nodePair.peer, rangesToSync, previewKind, transferId);
}
@Override
protected void startSync()
{
InetAddressAndPort local = ctx.broadcastAddressAndPort();
- SyncRequest request = new SyncRequest(desc, local, nodePair.coordinator, nodePair.peer, rangesToSync, previewKind, false);
+ SyncRequest request = new SyncRequest(desc, local, nodePair.coordinator, nodePair.peer, rangesToSync, previewKind, false, transferId);
Preconditions.checkArgument(nodePair.coordinator.equals(request.src));
String message = String.format("Forwarding streaming repair of %d ranges to %s (to be streamed with %s)", request.ranges.size(), request.src, request.dst);
logger.info("{} {}", previewKind.logPrefix(desc.sessionId), message);
@@ -59,11 +76,12 @@ public class SymmetricRemoteSyncTask extends SyncTask implements CompletableRemo
sendRequest(request, request.src);
}
- public void syncComplete(boolean success, List<SessionSummary> summaries)
+ @Override
+ public void syncComplete(SyncResponse response)
{
- if (success)
+ if (response.success)
{
- trySuccess(stat.withSummaries(summaries));
+ trySuccess(stat.withSummaries(response.summaries, response.planId, response.transferId));
}
else
{
@@ -78,6 +96,7 @@ public class SymmetricRemoteSyncTask extends SyncTask implements CompletableRemo
return "SymmetricRemoteSyncTask{" +
"rangesToSync=" + rangesToSync +
", nodePair=" + nodePair +
+ ", transfer ID=" + transferId +
'}';
}
}
diff --git a/src/java/org/apache/cassandra/repair/SyncStat.java b/src/java/org/apache/cassandra/repair/SyncStat.java
index 2241915ab4..b1231a8569 100644
--- a/src/java/org/apache/cassandra/repair/SyncStat.java
+++ b/src/java/org/apache/cassandra/repair/SyncStat.java
@@ -20,9 +20,13 @@ package org.apache.cassandra.repair;
import java.util.Collection;
import java.util.List;
+import javax.annotation.Nullable;
+
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.replication.ShortMutationId;
import org.apache.cassandra.streaming.SessionSummary;
+import org.apache.cassandra.utils.TimeUUID;
/**
* Statistics about synchronizing two replica
@@ -32,21 +36,48 @@ public class SyncStat
public final SyncNodePair nodes;
public final Collection<Range<Token>> differences;
public final List<SessionSummary> summaries;
+ public final TimeUUID planId;
+
+ @Nullable
+ public final ShortMutationId transferId;
public SyncStat(SyncNodePair nodes, Collection<Range<Token>> differences)
{
this(nodes, differences, null);
}
- public SyncStat(SyncNodePair nodes, Collection<Range<Token>> differences, List<SessionSummary> summaries)
+ private SyncStat(SyncNodePair nodes, Collection<Range<Token>> differences, List<SessionSummary> summaries)
{
this.nodes = nodes;
this.summaries = summaries;
this.differences = differences;
+ this.planId = null;
+ this.transferId = null;
+ }
+
+ private SyncStat(SyncNodePair nodes, Collection<Range<Token>> differences, List<SessionSummary> summaries, TimeUUID planId, ShortMutationId transferId)
+ {
+ this.nodes = nodes;
+ this.summaries = summaries;
+ this.differences = differences;
+ this.planId = planId;
+ this.transferId = transferId;
+ }
+
+ public SyncStat withSummaries(List<SessionSummary> summaries, TimeUUID planId, ShortMutationId transferId)
+ {
+ return new SyncStat(nodes, differences, summaries, planId, transferId);
}
- public SyncStat withSummaries(List<SessionSummary> summaries)
+ @Override
+ public String toString()
{
- return new SyncStat(nodes, differences, summaries);
+ return "SyncStat{" +
+ "nodes=" + nodes +
+ ", differences=" + differences +
+ ", summaries=" + summaries +
+ ", planId=" + planId +
+ ", transfer ID=" + transferId +
+ '}';
}
}
diff --git a/src/java/org/apache/cassandra/repair/SyncTask.java b/src/java/org/apache/cassandra/repair/SyncTask.java
index a3b1a57493..9f986bc26b 100644
--- a/src/java/org/apache/cassandra/repair/SyncTask.java
+++ b/src/java/org/apache/cassandra/repair/SyncTask.java
@@ -18,13 +18,16 @@
package org.apache.cassandra.repair;
+import java.util.Collection;
import java.util.List;
+import java.util.Objects;
import java.util.concurrent.TimeUnit;
+import javax.annotation.Nullable;
+
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
-import org.apache.cassandra.utils.concurrent.AsyncFuture;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -34,8 +37,11 @@ import org.apache.cassandra.dht.Token;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.repair.messages.RepairMessage;
import org.apache.cassandra.repair.messages.SyncRequest;
+import org.apache.cassandra.replication.ShortMutationId;
import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.utils.concurrent.AsyncFuture;
import static org.apache.cassandra.net.Verb.SYNC_REQ;
import static org.apache.cassandra.repair.messages.RepairMessage.notDone;
@@ -50,11 +56,13 @@ public abstract class SyncTask extends AsyncFuture<SyncStat> implements Runnable
public final List<Range<Token>> rangesToSync;
protected final PreviewKind previewKind;
protected final SyncNodePair nodePair;
+ protected final ShortMutationId transferId;
+ protected volatile TimeUUID planId;
protected volatile long startTime = Long.MIN_VALUE;
protected final SyncStat stat;
- protected SyncTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort primaryEndpoint, InetAddressAndPort peer, List<Range<Token>> rangesToSync, PreviewKind previewKind)
+ protected SyncTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort primaryEndpoint, InetAddressAndPort peer, List<Range<Token>> rangesToSync, PreviewKind previewKind, ShortMutationId transferId)
{
Preconditions.checkArgument(!peer.equals(primaryEndpoint), "Sending and receiving node are the same: %s", peer);
this.ctx = ctx;
@@ -62,16 +70,47 @@ public abstract class SyncTask extends AsyncFuture<SyncStat> implements Runnable
this.rangesToSync = rangesToSync;
this.nodePair = new SyncNodePair(primaryEndpoint, peer);
this.previewKind = previewKind;
+ this.transferId = transferId;
this.stat = new SyncStat(nodePair, rangesToSync);
+
+ addCallback((syncStat, failure) -> {
+ if (syncStat != null && syncStat.planId != null)
+ this.planId = syncStat.planId;
+ else if (failure instanceof org.apache.cassandra.streaming.StreamException)
+ this.planId = ((org.apache.cassandra.streaming.StreamException) failure).finalState.planId;
+ });
}
protected abstract void startSync();
+ /**
+ * Creates a new SyncTask with the same parameters but different ranges.
+ * Used for splitting sync tasks on shard boundaries.
+ */
+ public abstract SyncTask withRanges(Collection<Range<Token>> newRanges);
+
+ public abstract SyncTask withTransferId(ShortMutationId transferId);
+
public SyncNodePair nodePair()
{
return nodePair;
}
+ public ShortMutationId getTransferId()
+ {
+ return transferId;
+ }
+
+ /**
+ * Returns the planId associated with this sync task's streaming operation.
+ * The planId is captured when the task completes (successfully or with a StreamException).
+ * @return the planId if streaming has completed, null otherwise
+ */
+ public TimeUUID getPlanId()
+ {
+ return planId;
+ }
+
/**
* Compares trees, and triggers repairs for any ranges that mismatch.
*/
@@ -118,4 +157,34 @@ public abstract class SyncTask extends AsyncFuture<SyncStat> implements Runnable
to,
this::tryFailure);
}
+
+ public static class SyncTaskId
+ {
+ RepairJobDesc desc;
+ SyncNodePair pair;
+
+ @Nullable
+ ShortMutationId transferId;
+
+ public SyncTaskId(RepairJobDesc desc, SyncNodePair pair, ShortMutationId transferId)
+ {
+ this.desc = desc;
+ this.pair = pair;
+ this.transferId = transferId;
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (o == null || getClass() != o.getClass()) return false;
+ SyncTaskId that = (SyncTaskId) o;
+ return Objects.equals(desc, that.desc) && Objects.equals(pair, that.pair) && Objects.equals(transferId, that.transferId);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(desc, pair, transferId);
+ }
+ }
}
diff --git a/src/java/org/apache/cassandra/repair/SyncTasks.java b/src/java/org/apache/cassandra/repair/SyncTasks.java
new file mode 100644
index 0000000000..d5d60a81d1
--- /dev/null
+++ b/src/java/org/apache/cassandra/repair/SyncTasks.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.repair;
+
+import java.util.AbstractCollection;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.function.Consumer;
+
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.replication.MutationTrackingService;
+import org.apache.cassandra.replication.Participants;
+import org.apache.cassandra.replication.Shard;
+
+public class SyncTasks extends AbstractCollection<SyncTask>
+{
+ private final List<ShardedSyncTask> shardedTasks = new ArrayList<>();
+
+ public static class ShardedSyncTask
+ {
+ public final String keyspace;
+ public final Participants participants;
+ public final SyncTask task;
+ public final Range<Token> range;
+
+ private ShardedSyncTask(String keyspace, Participants participants, SyncTask task, Range<Token> range)
+ {
+ this.keyspace = keyspace;
+ this.participants = participants;
+ this.task = task;
+ this.range = range;
+ }
+ }
+
+ static SyncTasks untracked(Collection<SyncTask> tasks)
+ {
+ SyncTasks syncTasks = new SyncTasks();
+ tasks.forEach(t -> syncTasks.shardedTasks.add(new ShardedSyncTask(null, null, t, null)));
+ return syncTasks;
+ }
+
+ /**
+ * Mutation Tracking manages tracking metadata within shards that are each responsible for a piece of the owned
+ * token space. Executing a full repair across an entire node's ownership will span multiple shards, so repair sync
+ * tasks need to be split to each align within a single tracking shard.
+ */
+ static SyncTasks tracked(Keyspace keyspace, List<SyncTask> tasks)
+ {
+ return MutationTrackingService.instance.alignToShardBoundaries(keyspace, tasks);
+ }
+
+ public void add(Shard shard, SyncTask task)
+ {
+ // Narrow the ultimate scope of activation to the ranges in the sync tasks rather than the entire shard.
+ Set<Range<Token>> ranges = new HashSet<>(task.rangesToSync);
+ Range<Token> span = span(ranges);
+ shardedTasks.add(new ShardedSyncTask(shard.keyspace, shard.participants, task, span));
+ }
+
+ public static Range<Token> span(Set<Range<Token>> ranges)
+ {
+ List<Range<Token>> normalized = Range.normalize(ranges);
+ Token min = normalized.get(0).left;
+ Token max = normalized.get(normalized.size() - 1).right;
+ return new Range<>(min, max);
+ }
+
+ public void apply(Consumer<ShardedSyncTask> consumer)
+ {
+ shardedTasks.forEach(consumer);
+ }
+
+ @Override
+ public Iterator<SyncTask> iterator()
+ {
+ return shardedTasks.stream().map(shardedSyncTask -> shardedSyncTask.task).iterator();
+ }
+
+ @Override
+ public int size()
+ {
+ return shardedTasks.size();
+ }
+}
diff --git a/src/java/org/apache/cassandra/repair/messages/SyncRequest.java b/src/java/org/apache/cassandra/repair/messages/SyncRequest.java
index 137147d898..b9b09ee48b 100644
--- a/src/java/org/apache/cassandra/repair/messages/SyncRequest.java
+++ b/src/java/org/apache/cassandra/repair/messages/SyncRequest.java
@@ -23,6 +23,8 @@ import java.util.Collection;
import java.util.List;
import java.util.Objects;
+import javax.annotation.Nullable;
+
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.dht.AbstractBounds;
import org.apache.cassandra.dht.IPartitioner;
@@ -32,7 +34,9 @@ import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.repair.RepairJobDesc;
+import org.apache.cassandra.replication.ShortMutationId;
import org.apache.cassandra.streaming.PreviewKind;
import static org.apache.cassandra.locator.InetAddressAndPort.Serializer.inetAddressAndPortSerializer;
@@ -51,6 +55,9 @@ public class SyncRequest extends RepairMessage
public final Collection<Range<Token>> ranges;
public final PreviewKind previewKind;
public final boolean asymmetric;
+
+ @Nullable
+ public final ShortMutationId transferId;
public SyncRequest(RepairJobDesc desc,
InetAddressAndPort initiator,
@@ -58,7 +65,8 @@ public class SyncRequest extends RepairMessage
InetAddressAndPort dst,
Collection<Range<Token>> ranges,
PreviewKind previewKind,
- boolean asymmetric)
+ boolean asymmetric,
+ ShortMutationId transferId)
{
super(desc);
this.initiator = initiator;
@@ -67,6 +75,7 @@ public class SyncRequest extends RepairMessage
this.ranges = ranges;
this.previewKind = previewKind;
this.asymmetric = asymmetric;
+ this.transferId = transferId;
}
@Override
@@ -81,13 +90,14 @@ public class SyncRequest extends RepairMessage
dst.equals(req.dst) &&
ranges.equals(req.ranges) &&
previewKind == req.previewKind &&
- asymmetric == req.asymmetric;
+ asymmetric == req.asymmetric &&
+ transferId == req.transferId;
}
@Override
public int hashCode()
{
- return Objects.hash(desc, initiator, src, dst, ranges, previewKind);
+ return Objects.hash(desc, initiator, src, dst, ranges, previewKind, transferId);
}
public static final IVersionedSerializer<SyncRequest> serializer = new IVersionedSerializer<SyncRequest>()
@@ -103,6 +113,13 @@ public class SyncRequest extends RepairMessage
AbstractBounds.tokenSerializer.serialize(range, out, version);
out.writeInt(message.previewKind.getSerializationVal());
out.writeBoolean(message.asymmetric);
+
+ if (version >= MessagingService.Version.VERSION_52.value)
+ {
+ out.writeBoolean(message.transferId != null);
+ if (message.transferId != null)
+ ShortMutationId.serializer.serialize(message.transferId, out, version);
+ }
}
public SyncRequest deserialize(DataInputPlus in, int version) throws IOException
@@ -118,7 +135,12 @@ public class SyncRequest extends RepairMessage
ranges.add((Range<Token>) AbstractBounds.tokenSerializer.deserialize(in, partitioner, version));
PreviewKind previewKind = PreviewKind.deserialize(in.readInt());
boolean asymmetric = in.readBoolean();
- return new SyncRequest(desc, initiator, src, dst, ranges, previewKind, asymmetric);
+
+ ShortMutationId transferId = version >= MessagingService.Version.VERSION_52.value && in.readBoolean()
+ ? ShortMutationId.serializer.deserialize(in, version)
+ : null;
+
+ return new SyncRequest(desc, initiator, src, dst, ranges, previewKind, asymmetric, transferId);
}
public long serializedSize(SyncRequest message, int version)
@@ -132,6 +154,14 @@ public class SyncRequest extends RepairMessage
size += AbstractBounds.tokenSerializer.serializedSize(range, version);
size += TypeSizes.sizeof(message.previewKind.getSerializationVal());
size += TypeSizes.sizeof(message.asymmetric);
+
+ if (version >= MessagingService.Version.VERSION_52.value)
+ {
+ size += TypeSizes.sizeof(false);
+ if (message.transferId != null)
+ size += ShortMutationId.serializer.serializedSize(message.transferId, version);
+ }
+
return size;
}
};
@@ -146,6 +176,7 @@ public class SyncRequest extends RepairMessage
", ranges=" + ranges +
", previewKind=" + previewKind +
", asymmetric=" + asymmetric +
+ ", transfer ID=" + transferId +
"} " + super.toString();
}
}
diff --git a/src/java/org/apache/cassandra/repair/messages/SyncResponse.java b/src/java/org/apache/cassandra/repair/messages/SyncResponse.java
index e7b5446bad..338a6e02e6 100644
--- a/src/java/org/apache/cassandra/repair/messages/SyncResponse.java
+++ b/src/java/org/apache/cassandra/repair/messages/SyncResponse.java
@@ -22,15 +22,20 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
+import javax.annotation.Nullable;
+
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.IPartitionerDependentSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.repair.RepairJobDesc;
import org.apache.cassandra.repair.SyncNodePair;
+import org.apache.cassandra.replication.ShortMutationId;
import org.apache.cassandra.streaming.SessionSummary;
+import org.apache.cassandra.utils.TimeUUID;
/**
*
@@ -45,20 +50,29 @@ public class SyncResponse extends RepairMessage
public final List<SessionSummary> summaries;
- public SyncResponse(RepairJobDesc desc, SyncNodePair nodes, boolean success, List<SessionSummary> summaries)
+ @Nullable
+ public final TimeUUID planId;
+ @Nullable
+ public final ShortMutationId transferId;
+
+ public SyncResponse(RepairJobDesc desc, SyncNodePair nodes, boolean success, List<SessionSummary> summaries, TimeUUID planId, ShortMutationId transferId)
{
super(desc);
this.nodes = nodes;
this.success = success;
this.summaries = summaries;
+ this.planId = planId;
+ this.transferId = transferId;
}
- public SyncResponse(RepairJobDesc desc, InetAddressAndPort endpoint1, InetAddressAndPort endpoint2, boolean success, List<SessionSummary> summaries)
+ public SyncResponse(RepairJobDesc desc, InetAddressAndPort endpoint1, InetAddressAndPort endpoint2, boolean success, List<SessionSummary> summaries, TimeUUID planId, ShortMutationId transferId)
{
super(desc);
this.summaries = summaries;
this.nodes = new SyncNodePair(endpoint1, endpoint2);
this.success = success;
+ this.planId = planId;
+ this.transferId = transferId;
}
@Override
@@ -70,13 +84,15 @@ public class SyncResponse extends RepairMessage
return desc.equals(other.desc) &&
success == other.success &&
nodes.equals(other.nodes) &&
- summaries.equals(other.summaries);
+ summaries.equals(other.summaries) &&
+ Objects.equals(planId, other.planId) &&
+ Objects.equals(transferId, other.transferId);
}
@Override
public int hashCode()
{
- return Objects.hash(desc, success, nodes, summaries);
+ return Objects.hash(desc, success, nodes, summaries, planId, transferId);
}
public static final IPartitionerDependentSerializer<SyncResponse> serializer = new IPartitionerDependentSerializer<SyncResponse>()
@@ -92,6 +108,17 @@ public class SyncResponse extends RepairMessage
{
SessionSummary.serializer.serialize(summary, out, version);
}
+
+ if (version >= MessagingService.Version.VERSION_52.value)
+ {
+ out.writeBoolean(message.planId != null);
+ if (message.planId != null)
+ TimeUUID.Serializer.instance.serialize(message.planId, out);
+
+ out.writeBoolean(message.transferId != null);
+ if (message.transferId != null)
+ ShortMutationId.serializer.serialize(message.transferId, out, version);
+ }
}
@Override
@@ -108,7 +135,13 @@ public class SyncResponse extends RepairMessage
summaries.add(SessionSummary.serializer.deserialize(in, partitioner, version));
}
- return new SyncResponse(desc, nodes, success, summaries);
+ TimeUUID planId = version >= MessagingService.Version.VERSION_52.value && in.readBoolean()
+ ? TimeUUID.Serializer.instance.deserialize(in) : null;
+
+ ShortMutationId transferId = version >= MessagingService.Version.VERSION_52.value && in.readBoolean()
+ ? ShortMutationId.serializer.deserialize(in, version) : null;
+
+ return new SyncResponse(desc, nodes, success, summaries, planId, transferId);
}
public long serializedSize(SyncResponse message, int version)
@@ -123,6 +156,17 @@ public class SyncResponse extends RepairMessage
size += SessionSummary.serializer.serializedSize(summary, version);
}
+ if (version >= MessagingService.Version.VERSION_52.value)
+ {
+ size += TypeSizes.sizeof(false);
+ if (message.planId != null)
+ size += TimeUUID.Serializer.instance.serializedSize(message.planId);
+
+ size += TypeSizes.sizeof(false);
+ if (message.transferId != null)
+ size += ShortMutationId.serializer.serializedSize(message.transferId, version);
+ }
+
return size;
}
};
diff --git a/src/java/org/apache/cassandra/repair/state/SyncState.java b/src/java/org/apache/cassandra/repair/state/SyncState.java
index 9c88aa1877..d2d49d73bb 100644
--- a/src/java/org/apache/cassandra/repair/state/SyncState.java
+++ b/src/java/org/apache/cassandra/repair/state/SyncState.java
@@ -20,8 +20,11 @@ package org.apache.cassandra.repair.state;
import java.util.Objects;
+import javax.annotation.Nullable;
+
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.repair.RepairJobDesc;
+import org.apache.cassandra.replication.ShortMutationId;
import org.apache.cassandra.utils.Clock;
public class SyncState extends AbstractState<SyncState.State, SyncState.Id>
@@ -31,9 +34,9 @@ public class SyncState extends AbstractState<SyncState.State, SyncState.Id>
public final Phase phase = new Phase();
- public SyncState(Clock clock, RepairJobDesc desc, InetAddressAndPort initiator, InetAddressAndPort src, InetAddressAndPort dst)
+ public SyncState(Clock clock, RepairJobDesc desc, InetAddressAndPort initiator, InetAddressAndPort src, InetAddressAndPort dst, ShortMutationId transferId)
{
- super(clock, new Id(desc, initiator, src, dst), State.class);
+ super(clock, new Id(desc, initiator, src, dst, transferId), State.class);
}
public final class Phase extends BaseSkipPhase
@@ -58,13 +61,17 @@ public class SyncState extends AbstractState<SyncState.State, SyncState.Id>
{
public final RepairJobDesc desc;
public final InetAddressAndPort initiator, src, dst;
+
+ @Nullable
+ public final ShortMutationId transferId;
- public Id(RepairJobDesc desc, InetAddressAndPort initiator, InetAddressAndPort src, InetAddressAndPort dst)
+ public Id(RepairJobDesc desc, InetAddressAndPort initiator, InetAddressAndPort src, InetAddressAndPort dst, ShortMutationId transferId)
{
this.desc = desc;
this.initiator = initiator;
this.src = src;
this.dst = dst;
+ this.transferId = transferId;
}
@Override
@@ -73,13 +80,13 @@ public class SyncState extends AbstractState<SyncState.State, SyncState.Id>
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Id id = (Id) o;
- return desc.equals(id.desc) && initiator.equals(id.initiator) && src.equals(id.src) && dst.equals(id.dst);
+ return desc.equals(id.desc) && initiator.equals(id.initiator) && src.equals(id.src) && dst.equals(id.dst) && Objects.equals(transferId, id.transferId);
}
@Override
public int hashCode()
{
- return Objects.hash(desc, initiator, src, dst);
+ return Objects.hash(desc, initiator, src, dst, transferId);
}
}
}
diff --git a/src/java/org/apache/cassandra/replication/ActivatedTransfers.java b/src/java/org/apache/cassandra/replication/ActivatedTransfers.java
index bcee80ecba..76e7a40303 100644
--- a/src/java/org/apache/cassandra/replication/ActivatedTransfers.java
+++ b/src/java/org/apache/cassandra/replication/ActivatedTransfers.java
@@ -87,11 +87,6 @@ public class ActivatedTransfers implements Iterable<ShortMutationId>
this.bounds = bounds;
}
- private ActivatedTransfer(ShortMutationId id, Collection<SSTableReader> sstables)
- {
- this(id, covering(sstables));
- }
-
public static final IVersionedSerializer<ActivatedTransfer> serializer = new IVersionedSerializer<>()
{
@Override
@@ -151,7 +146,6 @@ public class ActivatedTransfers implements Iterable<ShortMutationId>
transfers.removeIf(transfer -> transfer.id.offset() == offset);
}
- @VisibleForTesting
public void add(ShortMutationId transferId, Bounds<Token> bounds)
{
transfers.add(new ActivatedTransfer(transferId, bounds));
@@ -159,7 +153,7 @@ public class ActivatedTransfers implements Iterable<ShortMutationId>
public void add(ShortMutationId transferId, Collection<SSTableReader> sstables)
{
- transfers.add(new ActivatedTransfer(transferId, sstables));
+ transfers.add(new ActivatedTransfer(transferId, covering(sstables)));
}
public void addAll(ActivatedTransfers other)
@@ -192,7 +186,7 @@ public class ActivatedTransfers implements Iterable<ShortMutationId>
return transfers.isEmpty();
}
- private static Bounds<Token> covering(Collection<SSTableReader> sstables)
+ public static Bounds<Token> covering(Collection<SSTableReader> sstables)
{
Preconditions.checkArgument(!sstables.isEmpty());
Iterator<SSTableReader> iter = sstables.iterator();
diff --git a/src/java/org/apache/cassandra/replication/ActivationRequest.java b/src/java/org/apache/cassandra/replication/ActivationRequest.java
new file mode 100644
index 0000000000..1b9f260739
--- /dev/null
+++ b/src/java/org/apache/cassandra/replication/ActivationRequest.java
@@ -0,0 +1,260 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.replication;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import javax.annotation.Nullable;
+
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.exceptions.RequestFailureReason;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.IVerbHandler;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.streaming.StreamOperation;
+import org.apache.cassandra.tcm.membership.NodeId;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.TimeUUID;
+
+/**
+ * Activation transitions a {@link PendingLocalTransfer} from being pending (durable on disk but not yet visible to
+ * reads) to live (visible to reads and compactions), by associating the streaming plan ID with a mutation ID, referred
+ * to as the transfer ID.
+ * <p>
+ * For tracked repairs, activation may occur without a plan ID and therefore without a {@link PendingLocalTransfer}. In
+ * this case, activation skips this step (as the data is already present) and simply updates log offsets.
+ * <p>
+ * See {@link CoordinatedTransfer} for the lifecycle of a transfer and when a {@link ActivationRequest} is sent.
+ */
+public class ActivationRequest
+{
+ public final StreamOperation operation;
+ public final Pair<InetAddressAndPort, InetAddressAndPort> pair;
+ public final Phase phase;
+ public final ShortMutationId transferId;
+ public final NodeId coordinatorId;
+ public final String keyspace;
+ public final Range<Token> range;
+
+ @Nullable
+ public final TimeUUID planId;
+
+ public enum Phase
+ {
+ PREPARE(0),
+ COMMIT(1);
+
+ private final int id;
+ private static final Phase[] ids;
+ static
+ {
+ ids = new Phase[values().length];
+ for (Phase phase : values())
+ ids[phase.id] = phase;
+ }
+
+ Phase(int id)
+ {
+ this.id = id;
+ }
+
+ static Phase from(int id)
+ {
+ Phase phase = ids[id];
+ Preconditions.checkState(phase.id == id);
+ return phase;
+ }
+ }
+
+ public ActivationRequest(StreamOperation operation,
+ Pair<InetAddressAndPort, InetAddressAndPort> pair,
+ Phase phase,
+ ShortMutationId transferId,
+ NodeId coordinatorId,
+ Range<Token> range,
+ String keyspace,
+ TimeUUID planId)
+ {
+ Preconditions.checkArgument(operation.isTrackable(), "Operation " + operation.getDescription() + " is not trackable");
+ Preconditions.checkNotNull(pair, "Activations require an address pair");
+ Preconditions.checkArgument(!transferId.isNone(), "Activations require a transfer ID");
+ Preconditions.checkNotNull(coordinatorId, "Activations require a coordinator node ID");
+
+ this.operation = operation;
+ this.pair = pair;
+ this.transferId = transferId;
+ this.coordinatorId = coordinatorId;
+ this.phase = phase;
+ this.keyspace = keyspace;
+ this.range = range;
+ this.planId = planId;
+ }
+
+ ShortMutationId id()
+ {
+ return transferId;
+ }
+
+ public boolean isPrepare()
+ {
+ return phase == Phase.PREPARE;
+ }
+
+ public boolean isCommit()
+ {
+ return phase == Phase.COMMIT;
+ }
+
+ public void apply()
+ {
+ MutationTrackingService.instance.activateLocal(this);
+ }
+
+ public static final Serializer serializer = new Serializer();
+
+ public static class Serializer implements IVersionedSerializer<ActivationRequest>
+ {
+ @Override
+ public void serialize(ActivationRequest request, DataOutputPlus out, int version) throws IOException
+ {
+ out.writeUTF(request.operation.getDescription());
+
+ InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serialize(request.pair.left, out, version);
+ InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serialize(request.pair.right, out, version);
+
+ ShortMutationId.serializer.serialize(request.id(), out, version);
+ NodeId.messagingSerializer.serialize(request.coordinatorId, out, version);
+ out.writeByte(request.phase.id);
+ out.writeUTF(request.keyspace);
+ Range.serializer.serialize(request.range, out, null);
+
+ out.writeBoolean(request.planId != null);
+ if (request.planId != null)
+ TimeUUID.Serializer.instance.serialize(request.planId, out, version);
+ }
+
+ @Override
+ public ActivationRequest deserialize(DataInputPlus in, int version) throws IOException
+ {
+ StreamOperation operation = StreamOperation.fromString(in.readUTF());
+
+ InetAddressAndPort sender = InetAddressAndPort.Serializer.inetAddressAndPortSerializer.deserialize(in, version);
+ InetAddressAndPort receiver = InetAddressAndPort.Serializer.inetAddressAndPortSerializer.deserialize(in, version);
+
+ ShortMutationId id = ShortMutationId.serializer.deserialize(in, version);
+ NodeId coordinatorId = NodeId.messagingSerializer.deserialize(in, version);
+ Phase phase = Phase.from(in.readByte());
+ String keyspace = in.readUTF();
+ Range<Token> range = Range.serializer.deserialize(in, null);
+
+ TimeUUID planId = in.readBoolean() ? TimeUUID.Serializer.instance.deserialize(in, version) : null;
+
+ return new ActivationRequest(operation, Pair.create(sender, receiver), phase, id, coordinatorId, range, keyspace, planId);
+ }
+
+ @Override
+ public long serializedSize(ActivationRequest request, int version)
+ {
+ long size = 0;
+
+ size += TypeSizes.sizeof(request.operation.getDescription());
+
+ size += InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serializedSize(request.pair.left, version);
+ size += InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serializedSize(request.pair.right, version);
+
+ size += ShortMutationId.serializer.serializedSize(request.id(), version);
+ size += NodeId.messagingSerializer.serializedSize(request.coordinatorId, version);
+ size += TypeSizes.BYTE_SIZE; // Enum ordinal
+
+ size += TypeSizes.sizeof(request.keyspace);
+ size += Range.serializer.serializedSize(request.range, null);
+
+ size += TypeSizes.BOOL_SIZE;
+ if (request.planId != null)
+ size += TimeUUID.Serializer.instance.serializedSize(request.planId, version);
+
+ return size;
+ }
+ }
+
+ public static class VerbHandler implements IVerbHandler<ActivationRequest>
+ {
+ private static final Logger logger = LoggerFactory.getLogger(VerbHandler.class);
+
+ @Override
+ public void doVerb(Message<ActivationRequest> msg) throws IOException
+ {
+ TransferTrackingService.instance().executor.submit(() -> {
+ try
+ {
+ msg.payload.apply();
+ MessagingService.instance().respond(new ActivationResponse(msg.payload.pair), msg);
+ }
+ catch (Throwable t)
+ {
+ logger.error("Local activation of {} failed due to error", msg.payload, t);
+ MessagingService.instance().respondWithFailure(RequestFailureReason.forException(t), msg);
+ }
+ });
+ }
+ }
+
+ public static final VerbHandler verbHandler = new VerbHandler();
+
+ @Override
+ public String toString()
+ {
+ return "ActivationRequest{" +
+ "operation=" + operation +
+ ", pair=" + pair +
+ ", phase=" + phase +
+ ", transferId=" + transferId +
+ ", coordinatorId=" + coordinatorId +
+ ", keyspace='" + keyspace + '\'' +
+ ", range=" + range +
+ ", planId=" + planId +
+ '}';
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (o == null || getClass() != o.getClass()) return false;
+ ActivationRequest that = (ActivationRequest) o;
+ return operation == that.operation && Objects.equals(pair, that.pair) && phase == that.phase && Objects.equals(transferId, that.transferId) && Objects.equals(coordinatorId, that.coordinatorId) && Objects.equals(keyspace, that.keyspace) && Objects.equals(range, that.range) && Objects.equals(planId, that.planId);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(operation, pair, phase, transferId, coordinatorId, keyspace, range, planId);
+ }
+}
diff --git a/src/java/org/apache/cassandra/replication/ActivationResponse.java b/src/java/org/apache/cassandra/replication/ActivationResponse.java
new file mode 100644
index 0000000000..b7c6584f28
--- /dev/null
+++ b/src/java/org/apache/cassandra/replication/ActivationResponse.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.replication;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import com.google.common.base.Preconditions;
+
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.utils.Pair;
+
+public class ActivationResponse
+{
+ public final Pair<InetAddressAndPort, InetAddressAndPort> syncPair;
+
+ public ActivationResponse(Pair<InetAddressAndPort, InetAddressAndPort> syncPair)
+ {
+ Preconditions.checkNotNull(syncPair, "Activations require a sync node address pair");
+
+ this.syncPair = syncPair;
+ }
+
+ public static final Serializer serializer = new Serializer();
+
+ @Override
+ public String toString()
+ {
+ return "ActivationResponse{" +
+ "syncPair=" + syncPair +
+ '}';
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (o == null || getClass() != o.getClass()) return false;
+ ActivationResponse that = (ActivationResponse) o;
+ return Objects.equals(syncPair, that.syncPair);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(syncPair);
+ }
+
+ public static class Serializer implements IVersionedSerializer<ActivationResponse>
+ {
+ @Override
+ public void serialize(ActivationResponse activate, DataOutputPlus out, int version) throws IOException
+ {
+ InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serialize(activate.syncPair.left, out, version);
+ InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serialize(activate.syncPair.right, out, version);
+ }
+
+ @SuppressWarnings("SuspiciousNameCombination")
+ @Override
+ public ActivationResponse deserialize(DataInputPlus in, int version) throws IOException
+ {
+ InetAddressAndPort left = InetAddressAndPort.Serializer.inetAddressAndPortSerializer.deserialize(in, version);
+ InetAddressAndPort right = InetAddressAndPort.Serializer.inetAddressAndPortSerializer.deserialize(in, version);
+ return new ActivationResponse(Pair.create(left, right));
+ }
+
+ @Override
+ public long serializedSize(ActivationResponse activate, int version)
+ {
+ long size = 0;
+ size += InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serializedSize(activate.syncPair.left, version);
+ size += InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serializedSize(activate.syncPair.right, version);
+ return size;
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/replication/ActiveLogReconciler.java b/src/java/org/apache/cassandra/replication/ActiveLogReconciler.java
index e7d5504e90..ccc597d546 100644
--- a/src/java/org/apache/cassandra/replication/ActiveLogReconciler.java
+++ b/src/java/org/apache/cassandra/replication/ActiveLogReconciler.java
@@ -17,7 +17,6 @@
*/
package org.apache.cassandra.replication;
-import java.util.Collections;
import java.io.IOException;
import java.util.concurrent.TimeUnit;
@@ -132,7 +131,7 @@ public final class ActiveLogReconciler implements Shutdownable
{
private static Task from(ShortMutationId id, InetAddressAndPort toHost)
{
- CoordinatedTransfer transfer = LocalTransfers.instance().getActivatedTransfer(id);
+ CoordinatedTransfer transfer = TransferTrackingService.instance().getActivatedTransfer(id);
if (transfer != null)
return new TransferTask(transfer, toHost);
else
@@ -246,10 +245,10 @@ public final class ActiveLogReconciler implements Shutdownable
void send()
{
logger.debug("Sending activation to {}", toHost);
- LocalTransfers.instance().executor.submit(() -> {
+ TransferTrackingService.instance().executor.submit(() -> {
try
{
- transfer.activateOn(Collections.singleton(toHost));
+ transfer.activate(toHost);
onResponse(null);
}
catch (Throwable t)
diff --git a/src/java/org/apache/cassandra/replication/CoordinatedTransfer.java b/src/java/org/apache/cassandra/replication/CoordinatedTransfer.java
index d82b27c9b2..65456705dd 100644
--- a/src/java/org/apache/cassandra/replication/CoordinatedTransfer.java
+++ b/src/java/org/apache/cassandra/replication/CoordinatedTransfer.java
@@ -20,66 +20,40 @@ package org.apache.cassandra.replication;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Collections;
import java.util.EnumSet;
-import java.util.HashSet;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.function.Supplier;
-
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
import javax.annotation.CheckReturnValue;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ConsistencyLevel;
-import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.streaming.CassandraOutgoingFile;
+import org.apache.cassandra.dht.Bounds;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.RequestFailure;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.locator.AbstractReplicationStrategy;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.net.NoPayload;
import org.apache.cassandra.net.RequestCallbackWithFailure;
import org.apache.cassandra.net.Verb;
-import org.apache.cassandra.streaming.OutgoingStream;
-import org.apache.cassandra.streaming.StreamException;
-import org.apache.cassandra.streaming.StreamOperation;
-import org.apache.cassandra.streaming.StreamPlan;
-import org.apache.cassandra.streaming.StreamResultFuture;
-import org.apache.cassandra.streaming.StreamState;
-import org.apache.cassandra.tcm.ClusterMetadata;
-import org.apache.cassandra.tcm.membership.NodeId;
import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.Throwables;
import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.utils.concurrent.AsyncFuture;
-import org.apache.cassandra.utils.concurrent.Future;
import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.COMMITTED;
-import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.COMMITTING;
-import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.PREPARE_FAILED;
-import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.PREPARING;
-import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.INIT;
import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.STREAM_COMPLETE;
-import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.STREAM_FAILED;
-import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.STREAM_NOOP;
-import static org.apache.cassandra.replication.TransferActivation.Phase;
+import static org.apache.cassandra.replication.ActivationRequest.Phase;
/**
* Orchestrates the lifecycle of a tracked bulk data transfer for a single replica set, where the current instance is
@@ -96,8 +70,8 @@ import static org.apache.cassandra.replication.TransferActivation.Phase;
* </li>
* <li>
* <b>Activation {@link Phase#PREPARE}</b>
- * The coordinator sends PREPARE messages to verify replicas have the data persisted on disk and are ready for
- * activation.
+ * The coordinator optinally sends PREPARE messages to verify replicas have the data persisted on disk and are
+ * ready for activation. Implementations may customize this via {@link #prepare}
* </li>
* <li>
* <b>Activation {@link Phase#COMMIT}</b>
@@ -107,57 +81,41 @@ import static org.apache.cassandra.replication.TransferActivation.Phase;
* reconciliation processes (read reconciliation and background reconciliation).
* </li>
* </ol>
- *
- * For simplicity, the coordinator streams to itself rather than using direct file copy. This ensures we can use the
- * same lifecycle management for crash-safety and atomic add.
- * <p>
+ * <p>
* If a tracked data read is executed on a replica that's missing an activation, the read reconciliation process will
* apply the missing activation during reconciliation and a subsequent read will succeed. To minimize the gap between
* activations across replicas, avoid expensive operations like file copies or index builds during
- * {@link TransferActivation#apply()}.
+ * {@link ActivationRequest#apply()}.
*/
-public class CoordinatedTransfer
+public abstract class CoordinatedTransfer
{
private static final Logger logger = LoggerFactory.getLogger(CoordinatedTransfer.class);
+ protected final String keyspace;
+ protected final Range<Token> range;
+
String logPrefix()
{
- return String.format("[CoordinatedTransfer #%s]", id);
+ return String.format("[%s #%s]", getClass().getSimpleName(), id);
}
private final ShortMutationId id;
- private final String keyspace;
- private final Range<Token> range;
- private final ConsistencyLevel cl;
- final Collection<SSTableReader> sstables;
- final ConcurrentMap<InetAddressAndPort, SingleTransferResult> streamResults;
-
- @VisibleForTesting
- CoordinatedTransfer(Range<Token> range, MutationId id)
+ final ConcurrentMap<Pair<InetAddressAndPort, InetAddressAndPort>, SingleTransferResult> streamResults;
+
+ public CoordinatedTransfer(ShortMutationId id, String keyspace, Range<Token> range)
{
- this.keyspace = null;
- this.range = range;
- this.sstables = Collections.emptyList();
- this.cl = null;
this.id = id;
this.streamResults = new ConcurrentHashMap<>();
+ this.keyspace = keyspace;
+ this.range = range;
}
- CoordinatedTransfer(String keyspace, Range<Token> range, Participants participants, Collection<SSTableReader> sstables, ConsistencyLevel cl, Supplier<MutationId> nextId)
+ public CoordinatedTransfer(ShortMutationId id, Participants participants, String keyspace, Range<Token> range)
{
+ this.id = id;
+ this.streamResults = new ConcurrentHashMap<>(participants.size());
this.keyspace = keyspace;
this.range = range;
- this.sstables = sstables;
- this.cl = cl;
- this.id = nextId.get();
-
- ClusterMetadata cm = ClusterMetadata.current();
- this.streamResults = new ConcurrentHashMap<>(participants.size());
- for (int i = 0; i < participants.size(); i++)
- {
- InetAddressAndPort addr = cm.directory.getNodeAddresses(new NodeId(participants.get(i))).broadcastAddress;
- this.streamResults.put(addr, SingleTransferResult.Init());
- }
}
ShortMutationId id()
@@ -165,343 +123,173 @@ public class CoordinatedTransfer
return id;
}
- void execute()
- {
- logger.debug("{} Executing tracked bulk transfer {}", logPrefix(), this);
- LocalTransfers.instance().save(this);
- stream();
- }
-
- private void stream()
- {
- // TODO: Don't stream multiple copies over the WAN, send one copy and indicate forwarding
- List<Future<Void>> streaming = new ArrayList<>(streamResults.size());
- for (InetAddressAndPort to : streamResults.keySet())
- {
- Future<Void> stream = LocalTransfers.instance().executor.submit(() -> {
- stream(to);
- return null;
- });
- streaming.add(stream);
- }
-
- // Wait for all streams to complete, so we can clean up after failures. If we exit at the first failure, a
- // future stream can complete.
- LinkedList<Throwable> failures = null;
- for (Future<Void> stream : streaming)
- {
- try
- {
- stream.get();
- }
- catch (InterruptedException | ExecutionException e)
- {
- if (failures == null)
- failures = new LinkedList<>();
- failures.add(e);
- logger.error("{} Failed transfer due to", logPrefix(), e);
- }
- }
-
- if (failures != null && !failures.isEmpty())
- {
- Throwable failure = failures.element();
- Throwable cause = failure instanceof ExecutionException ? failure.getCause() : failure;
- maybeCleanupFailedStreams(cause);
-
- String msg = String.format("Failed streaming on %s instance(s): %s", failures.size(), failures);
- throw new RuntimeException(msg, Throwables.unchecked(cause));
- }
-
- logger.info("{} All streaming completed successfully", logPrefix());
- }
-
- private boolean sufficient()
+ Bounds<Token> bounds()
{
- AbstractReplicationStrategy ars = Keyspace.open(keyspace).getReplicationStrategy();
- int blockFor = cl.blockFor(ars);
- int responses = 0;
- for (Map.Entry<InetAddressAndPort, SingleTransferResult> entry : streamResults.entrySet())
- {
- if (entry.getValue().state == STREAM_COMPLETE)
- responses++;
- }
- return responses >= blockFor;
+ return new Bounds<>(range.left.nextValidToken(), range.right);
}
- void stream(InetAddressAndPort to)
+ public boolean isCommitted()
{
- SingleTransferResult result;
- try
- {
- result = streamTask(to);
- }
- catch (StreamException | ExecutionException | InterruptedException | TimeoutException e)
- {
- Throwable cause = e instanceof ExecutionException ? e.getCause() : e;
- markStreamFailure(to, cause);
- throw Throwables.unchecked(cause);
- }
-
- try
- {
- streamComplete(to, result);
- }
- catch (ExecutionException | InterruptedException | TimeoutException e)
+ for (SingleTransferResult result : streamResults.values())
{
- Throwable cause = e instanceof ExecutionException ? e.getCause() : e;
- throw Throwables.unchecked(cause);
+ if (result.state != SingleTransferResult.State.COMMITTED)
+ return false;
}
+ return true;
}
- private void notifyFailure() throws ExecutionException, InterruptedException
- {
- class NotifyFailure extends AsyncFuture<Void> implements RequestCallbackWithFailure<NoPayload>
- {
- final Set<InetAddressAndPort> responses = ConcurrentHashMap.newKeySet(streamResults.size());
-
- @Override
- public void onResponse(Message<NoPayload> msg)
- {
- responses.remove(msg.from());
- if (responses.isEmpty())
- trySuccess(null);
- }
-
- @Override
- public void onFailure(InetAddressAndPort from, RequestFailure failure)
- {
- tryFailure(failure.failure);
- }
- }
-
- NotifyFailure notifyFailure = new NotifyFailure();
- for (Map.Entry<InetAddressAndPort, SingleTransferResult> entry : streamResults.entrySet())
- {
- InetAddressAndPort to = entry.getKey();
- // Coordinator cleans up CoordinatedTransfer and PendingLocalTransfer separately, does not need to notify
- if (FBUtilities.getBroadcastAddressAndPort().equals(to))
- continue;
-
- SingleTransferResult result = entry.getValue();
- if (result.planId == null)
- {
- logger.warn("{} Skipping notification of transfer failure to {} due to unknown planId", logPrefix(), to);
- continue;
- }
+ protected abstract ActivationRequest createActivation(Pair<InetAddressAndPort, InetAddressAndPort> pair, ActivationRequest.Phase phase);
- logger.debug("{}, Notifying {} of transfer failure for plan {}", logPrefix(), to, result.planId);
- notifyFailure.responses.add(to);
- Message<TransferFailed> msg = Message.out(Verb.TRACKED_TRANSFER_FAILED_REQ, new TransferFailed(result.planId));
- MessagingService.instance().sendWithCallback(msg, to, notifyFailure);
- }
- notifyFailure.get();
- }
-
- private void markStreamFailure(InetAddressAndPort to, Throwable cause)
+ final void activate(InetAddressAndPort peer)
{
- TimeUUID planId;
- if (cause instanceof StreamException)
- planId = ((StreamException) cause).finalState.planId;
- else
- planId = null;
- streamResults.computeIfPresent(to, (peer, result) -> result.streamFailed(planId));
+ activate(streamResults.keySet().stream().filter(pair -> pair.right.equals(peer)).collect(Collectors.toList()));
}
- /**
- * This shouldn't throw an exception, even if we fail to notify peers of the streaming failure.
- */
- private void maybeCleanupFailedStreams(Throwable cause)
+ final void activate(Collection<Pair<InetAddressAndPort, InetAddressAndPort>> pairs)
{
- try
- {
- boolean purgeable = LocalTransfers.instance().purger.test(this);
- if (!purgeable)
- return;
+ // There's no reason to try to re-activate already COMMITTED peers...
+ List<Pair<InetAddressAndPort, InetAddressAndPort>> uncommittedPairs = new ArrayList<>(pairs.size());
- notifyFailure();
- LocalTransfers.instance().scheduleCleanup();
- }
- catch (Throwable t)
- {
- if (cause != null)
- t.addSuppressed(cause);
- logger.error("{} Failed to notify peers of stream failure", logPrefix(), t);
- }
- }
-
- private void streamComplete(InetAddressAndPort to, SingleTransferResult result) throws ExecutionException, InterruptedException, TimeoutException
- {
- streamResults.put(to, result);
- logger.info("{} Completed streaming to {}, {}", logPrefix(), to, this);
- maybeActivate();
- }
+ for (Pair<InetAddressAndPort, InetAddressAndPort> pair : pairs)
+ if (streamResults.get(pair).state != COMMITTED)
+ uncommittedPairs.add(pair);
- synchronized void maybeActivate()
- {
- // If any activations have already been sent out, send new activations to any received plans that have not yet
- // been activated
- boolean anyActivated = false;
- Set<InetAddressAndPort> awaitingActivation = new HashSet<>();
- for (Map.Entry<InetAddressAndPort, SingleTransferResult> entry : streamResults.entrySet())
- {
- InetAddressAndPort peer = entry.getKey();
- SingleTransferResult result = entry.getValue();
- if (result.state == COMMITTING || result.state == COMMITTED)
- {
- anyActivated = true;
- }
- else if (result.state == STREAM_COMPLETE)
- awaitingActivation.add(peer);
- }
- if (anyActivated && !awaitingActivation.isEmpty())
+ if (uncommittedPairs.isEmpty())
{
- logger.debug("{} Transfer already activated on some peers, sending activations to remaining: {}", logPrefix(), awaitingActivation);
- activateOn(awaitingActivation);
- return;
- }
- // If no activations have been sent out, check whether we have enough planIds back to meet the required CL
- else if (sufficient())
- {
- Set<InetAddressAndPort> peers = new HashSet<>();
- for (Map.Entry<InetAddressAndPort, SingleTransferResult> entry : streamResults.entrySet())
- {
- InetAddressAndPort peer = entry.getKey();
- SingleTransferResult result = entry.getValue();
- if (result.state == STREAM_COMPLETE)
- peers.add(peer);
- }
- logger.debug("{} Transfer meets consistency level {}, sending activations to {}", logPrefix(), cl, peers);
- activateOn(peers);
+ logAlreadyCommitted(pairs);
return;
}
- logger.debug("{} Nothing to activate", logPrefix());
+ activateInternal(uncommittedPairs);
}
- void activateOn(Collection<InetAddressAndPort> peers)
+ private synchronized void activateInternal(Collection<Pair<InetAddressAndPort, InetAddressAndPort>> targets)
{
- Preconditions.checkState(!peers.isEmpty());
- logger.debug("{} Activating {} on {}", logPrefix(), this, peers);
- LocalTransfers.instance().activating(this);
+ logger.debug("{} Activating {} for {}", logPrefix(), this, targets);
+
+ prepare(targets);
+ logger.debug("{} Activation prepare complete for {}", logPrefix(), targets);
- // First phase ensures data is present on disk, then second phase does the actual import. This ensures that if
- // something goes wrong (like a topology change during import), we don't have divergence.
- class Prepare extends AsyncFuture<Void> implements RequestCallbackWithFailure<NoPayload>
+ // Acknowledgement of activation is equivalent to a remote write acknowledgement. The imported SSTables
+ // are now part of the live set, visible to reads.
+ class Commit extends AsyncFuture<Void> implements RequestCallbackWithFailure<ActivationResponse>
{
- final Set<InetAddressAndPort> responses = ConcurrentHashMap.newKeySet();
+ final AtomicInteger responses = new AtomicInteger(0);
- public Prepare()
+ private Commit(Collection<Pair<InetAddressAndPort, InetAddressAndPort>> pairs)
{
- responses.addAll(peers);
+ responses.addAndGet(pairs.size());
}
@Override
- public void onResponse(Message<NoPayload> msg)
+ public void onResponse(Message<ActivationResponse> msg)
{
- logger.debug("{} Got response from: {}", logPrefix(), msg.from());
- responses.remove(msg.from());
- if (responses.isEmpty())
+ logger.debug("{} Activation successfully applied on {}", logPrefix(), msg.from());
+ streamResults.computeIfPresent(msg.payload.syncPair, (peer, result) -> result.committed());
+
+ MutationTrackingService.instance.receivedActivationResponse(CoordinatedTransfer.this, msg.from());
+
+ if (responses.decrementAndGet() == 0)
+ {
+ // All activations complete, schedule cleanup to purge pending SSTables
+ TransferTrackingService.instance().scheduleCleanup();
trySuccess(null);
+ }
}
@Override
public void onFailure(InetAddressAndPort from, RequestFailure failure)
{
- logger.debug("{} Got failure {} from {}", logPrefix(), failure, from);
- CoordinatedTransfer.this.streamResults.computeIfPresent(from, (peer, result) -> result.prepareFailed());
- tryFailure(new RuntimeException("Tracked import failed during PREPARE on " + from + " due to " + failure.reason));
+ logger.error("{} Failed activation commit on {} due to {}", logPrefix(), from, failure);
+ // TODO(expected): should only fail if we don't meet requested CL
+ tryFailure(new RuntimeException("Tracked transfer failed during COMMIT on " + from + " due to " + failure.reason));
}
}
- Prepare prepare = new Prepare();
- for (InetAddressAndPort peer : peers)
+ Commit commit = new Commit(targets);
+ for (Pair<InetAddressAndPort, InetAddressAndPort> target : targets)
{
- TransferActivation activation = new TransferActivation(this, peer, Phase.PREPARE);
- Message<TransferActivation> msg = Message.out(Verb.TRACKED_TRANSFER_ACTIVATE_REQ, activation);
- logger.debug("{} Sending {} to peer {}", logPrefix(), activation, peer);
- MessagingService.instance().sendWithCallback(msg, peer, prepare);
- CoordinatedTransfer.this.streamResults.computeIfPresent(peer, (peer0, result) -> result.preparing());
+ ActivationRequest activation = createActivation(target, Phase.COMMIT);
+ Message<ActivationRequest> msg = Message.out(Verb.TRACKED_TRANSFER_ACTIVATE_REQ, activation);
+
+ logger.debug("{} Sending commit {} to peer {}", logPrefix(), activation, target.right);
+ MessagingService.instance().sendWithCallback(msg, target.right, commit);
+ streamResults.computeIfPresent(target, (peer0, result) -> result.committing());
}
+
try
{
- prepare.get();
+ commit.get();
}
catch (InterruptedException | ExecutionException e)
{
Throwable cause = e instanceof ExecutionException ? e.getCause() : e;
throw Throwables.unchecked(cause);
}
- logger.debug("{} Activation prepare complete for {}", logPrefix(), peers);
+ logger.debug("{} Activation commit complete for {}", logPrefix(), targets);
+ }
- // Acknowledgement of activation is equivalent to a remote write acknowledgement. The imported SSTables
- // are now part of the live set, visible to reads.
- class Commit extends AsyncFuture<Void> implements RequestCallbackWithFailure<Void>
- {
- final Set<InetAddressAndPort> responses = ConcurrentHashMap.newKeySet();
+ protected abstract void prepare(Collection<Pair<InetAddressAndPort, InetAddressAndPort>> targets);
- private Commit(Collection<InetAddressAndPort> peers)
- {
- responses.addAll(peers);
- }
+ private void logAlreadyCommitted(Collection<Pair<InetAddressAndPort, InetAddressAndPort>> pairs)
+ {
+ logger.debug("Transfer {} for {} is already committed. Skipping activation...", this, pairs);
+ }
+
+ /**
+ * Notify all replicas that this transfer failed, triggering cleanup of pending SSTables.
+ * This is used by both {@link TrackedRepairTransfer} and {@link TrackedImportTransfer}.
+ */
+ protected void notifyFailure() throws ExecutionException, InterruptedException
+ {
+ class NotifyFailure extends AsyncFuture<Void> implements RequestCallbackWithFailure<NoPayload>
+ {
+ // TODO: Does this actually work? What if there is a race between notification and callbacks where we decrement to zero before submitting a second request?
+ // It seems like we should add the pair up-front that actally have a plan ID...?
+ final AtomicInteger responses = new AtomicInteger(0);
@Override
- public void onResponse(Message<Void> msg)
+ public void onResponse(Message<NoPayload> msg)
{
- logger.debug("{} Activation successfully applied on {}", logPrefix(), msg.from());
- CoordinatedTransfer.this.streamResults.computeIfPresent(msg.from(), (peer, result) -> result.committed());
-
- MutationTrackingService.instance.receivedActivationResponse(CoordinatedTransfer.this, msg.from());
- responses.remove(msg.from());
- if (responses.isEmpty())
- {
- // All activations complete, schedule cleanup to purge pending SSTables
- LocalTransfers.instance().scheduleCleanup();
+ if (responses.decrementAndGet() == 0)
trySuccess(null);
- }
}
@Override
public void onFailure(InetAddressAndPort from, RequestFailure failure)
{
- logger.error("{} Failed activation on {} due to {}", logPrefix(), from, failure);
- MutationTrackingService.instance.retryFailedTransfer(CoordinatedTransfer.this, from, failure.failure);
- // TODO(expected): should only fail if we don't meet requested CL
- tryFailure(new RuntimeException("Tracked import failed during COMMIT on " + from + " due to " + failure.reason));
+ // Log but don't fail - best effort cleanup
+ logger.warn("{} Failed to notify {} of transfer failure: {}", logPrefix(), from, failure);
+ if (responses.decrementAndGet() == 0)
+ trySuccess(null);
}
}
- Commit commit = new Commit(peers);
- for (InetAddressAndPort peer : peers)
+ NotifyFailure notifyFailure = new NotifyFailure();
+ for (Map.Entry<Pair<InetAddressAndPort, InetAddressAndPort>, SingleTransferResult> entry : streamResults.entrySet())
{
- TransferActivation activation = new TransferActivation(this, peer, Phase.COMMIT);
- Message<TransferActivation> msg = Message.out(Verb.TRACKED_TRANSFER_ACTIVATE_REQ, activation);
+ InetAddressAndPort to = entry.getKey().right;
+ // Coordinator cleans up CoordinatedTransfer and PendingLocalTransfer separately, does not need to notify
+ if (FBUtilities.getBroadcastAddressAndPort().equals(to))
+ continue;
- logger.debug("{} Sending {} to peer {}", logPrefix(), activation, peer);
- MessagingService.instance().sendWithCallback(msg, peer, commit);
- CoordinatedTransfer.this.streamResults.computeIfPresent(peer, (peer0, result) -> result.committing());
- }
+ SingleTransferResult result = entry.getValue();
+ if (result.planId() == null)
+ {
+ // No planId means streaming never completed, so there's nothing to clean up on the replica
+ logger.debug("{} Skipping notification of transfer failure to {} - no planId", logPrefix(), to);
+ continue;
+ }
- try
- {
- commit.get();
- }
- catch (InterruptedException | ExecutionException e)
- {
- Throwable cause = e instanceof ExecutionException ? e.getCause() : e;
- throw Throwables.unchecked(cause);
+ logger.debug("{} Notifying {} of transfer failure for plan {}", logPrefix(), to, result.planId());
+ notifyFailure.responses.incrementAndGet();
+ Message<TransferFailed> msg = Message.out(Verb.TRACKED_TRANSFER_FAILED_REQ, new TransferFailed(result.planId()));
+ MessagingService.instance().sendWithCallback(msg, to, notifyFailure);
}
- logger.debug("{} Activation commit complete for {}", logPrefix(), peers);
- }
- public boolean isCommitted()
- {
- for (SingleTransferResult result : streamResults.values())
- {
- if (result.state != COMMITTED)
- return false;
- }
- return true;
+ // Only wait if we actually sent notifications
+ if (notifyFailure.responses.get() > 0)
+ notifyFailure.get();
}
/**
@@ -511,6 +299,7 @@ public class CoordinatedTransfer
* <li>{@link State#INIT}: Transfer created, not yet streaming.</li>
* <li>{@link State#STREAM_COMPLETE}: Streaming successful, SSTables received on replica in pending directory.</li>
* <li>{@link State#STREAM_NOOP}: No data streamed (e.g., SSTable contains no rows in target range).</li>
+ * <li>{@link State#EMPTY_SYNC}: No repair sync (e.g., No Merkle tree disagreement).</li>
* <li>{@link State#STREAM_FAILED}: Streaming failed, may not have a streaming plan ID yet.</li>
* <li>{@link State#PREPARING}: Preparing for activation (first phase).</li>
* <li>{@link State#PREPARE_FAILED}: Prepare failed, aborting transfer.</li>
@@ -523,8 +312,10 @@ public class CoordinatedTransfer
* ┌────────────────┐
* ↓ │
* INIT ──┬──→ STREAM_COMPLETE ──→ PREPARING ──┬──→ COMMITTING ──→ COMMITTED
- * │ │
- * ├──→ STREAM_NOOP └──→ PREPARE_FAILED
+ * │ ↑ │
+ * ├──→ EMPTY_SYNC ─────────────┘ │
+ * │ └──→ PREPARE_FAILED
+ * ├──→ STREAM_NOOP
* │
* └──→ STREAM_FAILED
* </pre>
@@ -543,17 +334,19 @@ public class CoordinatedTransfer
PREPARING,
PREPARE_FAILED,
COMMITTING,
- COMMITTED;
+ COMMITTED,
+ EMPTY_SYNC;
EnumSet<State> transitionFrom;
static
{
INIT.transitionFrom = EnumSet.noneOf(State.class);
+ EMPTY_SYNC.transitionFrom = EnumSet.noneOf(State.class);
STREAM_NOOP.transitionFrom = EnumSet.of(INIT);
STREAM_FAILED.transitionFrom = EnumSet.of(INIT);
STREAM_COMPLETE.transitionFrom = EnumSet.of(INIT);
- PREPARING.transitionFrom = EnumSet.of(STREAM_COMPLETE, COMMITTING);
+ PREPARING.transitionFrom = EnumSet.of(STREAM_COMPLETE, COMMITTING, EMPTY_SYNC);
PREPARE_FAILED.transitionFrom = EnumSet.of(PREPARING);
COMMITTING.transitionFrom = EnumSet.of(PREPARING);
COMMITTED.transitionFrom = EnumSet.of(COMMITTING);
@@ -570,14 +363,14 @@ public class CoordinatedTransfer
this.planId = planId;
}
- private boolean canTransition(SingleTransferResult.State to)
+ boolean canTransition(State to)
{
return to.transitionFrom.contains(state);
}
public static SingleTransferResult Init()
{
- return new SingleTransferResult(INIT, null);
+ return new SingleTransferResult(State.INIT, null);
}
@VisibleForTesting
@@ -589,7 +382,13 @@ public class CoordinatedTransfer
@VisibleForTesting
static SingleTransferResult Noop()
{
- return new SingleTransferResult(STREAM_NOOP, null);
+ return new SingleTransferResult(State.STREAM_NOOP, null);
+ }
+
+ @VisibleForTesting
+ static SingleTransferResult EmptySync()
+ {
+ return new SingleTransferResult(State.EMPTY_SYNC, null);
}
@CheckReturnValue
@@ -607,31 +406,31 @@ public class CoordinatedTransfer
@CheckReturnValue
public SingleTransferResult streamFailed(TimeUUID planId)
{
- return transition(STREAM_FAILED, planId);
+ return transition(State.STREAM_FAILED, planId);
}
@CheckReturnValue
public SingleTransferResult preparing()
{
- return transition(PREPARING, this.planId);
+ return transition(State.PREPARING, this.planId);
}
@CheckReturnValue
public SingleTransferResult prepareFailed()
{
- return transition(PREPARE_FAILED, this.planId);
+ return transition(State.PREPARE_FAILED, this.planId);
}
@CheckReturnValue
public SingleTransferResult committing()
{
- return transition(COMMITTING, this.planId);
+ return transition(State.COMMITTING, this.planId);
}
@CheckReturnValue
public SingleTransferResult committed()
{
- return transition(COMMITTED, this.planId);
+ return transition(State.COMMITTED, this.planId);
}
public TimeUUID planId()
@@ -648,73 +447,4 @@ public class CoordinatedTransfer
'}';
}
}
-
- private SingleTransferResult streamTask(InetAddressAndPort to) throws StreamException, ExecutionException, InterruptedException, TimeoutException
- {
- StreamPlan plan = new StreamPlan(StreamOperation.TRACKED_TRANSFER);
-
- // No need to flush, only using non-live SSTables already on disk
- plan.flushBeforeTransfer(false);
-
- for (SSTableReader sstable : sstables)
- {
- List<Range<Token>> ranges = Collections.singletonList(range);
- List<SSTableReader.PartitionPositionBounds> positions = sstable.getPositionsForRanges(ranges);
- long estimatedKeys = sstable.estimatedKeysForRanges(ranges);
- OutgoingStream stream = new CassandraOutgoingFile(StreamOperation.TRACKED_TRANSFER, sstable.ref(), positions, ranges, estimatedKeys);
- plan.transferStreams(to, Collections.singleton(stream));
- }
-
- long timeout = DatabaseDescriptor.getStreamTransferTaskTimeout().toMilliseconds();
-
- logger.info("{} Starting streaming transfer {} to peer {}", logPrefix(), this, to);
- StreamResultFuture execute = plan.execute();
- StreamState state;
- try
- {
- state = execute.get(timeout, TimeUnit.MILLISECONDS);
- logger.debug("{} Completed streaming transfer {} to peer {}", logPrefix(), this, to);
- }
- catch (InterruptedException | ExecutionException | TimeoutException e)
- {
- logger.error("Stream session failed with error", e);
- throw e;
- }
-
- if (state.hasFailedSession() || state.hasAbortedSession())
- throw new StreamException(state, "Stream failed due to failed or aborted sessions");
-
- // If the SSTable doesn't contain any rows in the provided range, no streams delivered, nothing to activate
- if (state.sessions().isEmpty())
- return SingleTransferResult.Noop();
-
- return SingleTransferResult.StreamComplete(plan.planId());
- }
-
- @Override
- public boolean equals(Object o)
- {
- if (o == null || getClass() != o.getClass()) return false;
- CoordinatedTransfer transfer = (CoordinatedTransfer) o;
- return Objects.equals(keyspace, transfer.keyspace) && Objects.equals(range, transfer.range) && Objects.equals(streamResults, transfer.streamResults) && Objects.equals(sstables, transfer.sstables) && cl == transfer.cl && Objects.equals(id, transfer.id);
- }
-
- @Override
- public int hashCode()
- {
- return Objects.hash(keyspace, range, streamResults, sstables, cl, id);
- }
-
- @Override
- public String toString()
- {
- return "CoordinatedTransfer{" +
- "id=" + id +
- ", keyspace='" + keyspace + '\'' +
- ", range=" + range +
- ", cl=" + cl +
- ", sstables=" + sstables +
- ", streamResults=" + streamResults +
- '}';
- }
}
diff --git a/src/java/org/apache/cassandra/replication/CoordinatorLog.java b/src/java/org/apache/cassandra/replication/CoordinatorLog.java
index 56180c2594..70479c12be 100644
--- a/src/java/org/apache/cassandra/replication/CoordinatorLog.java
+++ b/src/java/org/apache/cassandra/replication/CoordinatorLog.java
@@ -44,6 +44,7 @@ import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.db.marshal.ListType;
import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Bounds;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.metrics.MutationTrackingMetrics;
@@ -329,12 +330,31 @@ public abstract class CoordinatorLog
MutationTrackingMetrics.instance.writeTimeOffsetsDiscovered.inc();
unreconciledMutations.finishWriting(mutation);
+ maybeMoveOffset(offset);
+ }
+ finally
+ {
+ lock.writeLock().unlock();
+ }
+ }
- if (remoteReplicasWitnessed(offset))
- {
- reconciledOffsets.add(offset);
- unreconciledMutations.remove(offset);
- }
+ /*
+ - On local replicas after they've completed activation (onHostId == me)
+ */
+ void finishActivation(Bounds<Token> bounds, ActivationRequest activation)
+ {
+ logger.trace("witnessed local transfer {}", activation.id());
+
+ lock.writeLock().lock();
+ try
+ {
+ int offset = activation.id().offset();
+ // we've raced with another write, no need to do anything else
+ if (!witnessedOffsets.get(localNodeId).add(offset))
+ return;
+
+ unreconciledMutations.activatedTransfer(activation.id(), bounds);
+ maybeMoveOffset(offset);
}
finally
{
@@ -358,33 +378,12 @@ public abstract class CoordinatorLog
return othersWitnessed(offset, localNodeId);
}
- /*
- - On local replicas after they've completed activation (onHostId == me)
- */
- void finishActivation(PendingLocalTransfer transfer, TransferActivation activation)
+ private void maybeMoveOffset(int offset)
{
- logger.trace("witnessed local transfer {}", activation.id());
-
- lock.writeLock().lock();
- try
- {
- int offset = activation.id().offset();
- // we've raced with another write, no need to do anything else
- if (!witnessedOffsets.get(localNodeId).add(offset))
- return;
-
- // This is the only difference with finishWriting - can we consolidate these methods?
- unreconciledMutations.activatedTransfer(activation.id(), transfer.sstables);
-
- if (remoteReplicasWitnessed(offset))
- {
- reconciledOffsets.add(offset);
- unreconciledMutations.remove(offset);
- }
- }
- finally
+ if (remoteReplicasWitnessed(offset))
{
- lock.writeLock().unlock();
+ reconciledOffsets.add(offset);
+ unreconciledMutations.remove(offset);
}
}
diff --git a/src/java/org/apache/cassandra/replication/ImmutableCoordinatorLogOffsets.java b/src/java/org/apache/cassandra/replication/ImmutableCoordinatorLogOffsets.java
index 61bc98dcb0..f20c4be45d 100644
--- a/src/java/org/apache/cassandra/replication/ImmutableCoordinatorLogOffsets.java
+++ b/src/java/org/apache/cassandra/replication/ImmutableCoordinatorLogOffsets.java
@@ -19,14 +19,12 @@
package org.apache.cassandra.replication;
import java.io.IOException;
-import java.util.Collection;
import java.util.Iterator;
import java.util.Map;
import java.util.Objects;
import java.util.function.Predicate;
import javax.annotation.concurrent.NotThreadSafe;
-import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Iterators;
import org.slf4j.Logger;
@@ -36,7 +34,6 @@ import org.agrona.collections.Long2ObjectHashMap;
import org.apache.cassandra.dht.Bounds;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.IVersionedSerializer;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.net.MessagingService;
@@ -161,7 +158,6 @@ public class ImmutableCoordinatorLogOffsets implements CoordinatorLogOffsets<Off
return this;
}
- @VisibleForTesting
public Builder addTransfer(ShortMutationId transferId, Bounds<Token> bounds)
{
if (transferId.isNone())
@@ -172,16 +168,6 @@ public class ImmutableCoordinatorLogOffsets implements CoordinatorLogOffsets<Off
return this;
}
- public Builder addTransfer(ShortMutationId transferId, Collection<SSTableReader> sstables)
- {
- if (transferId.isNone())
- return this;
- if (transfers == null)
- transfers = new ActivatedTransfers();
- transfers.add(transferId, sstables);
- return this;
- }
-
public Builder addTransfers(ActivatedTransfers other)
{
if (other.isEmpty())
diff --git a/src/java/org/apache/cassandra/replication/MutationTrackingService.java b/src/java/org/apache/cassandra/replication/MutationTrackingService.java
index 18394bdf53..7dc59c7988 100644
--- a/src/java/org/apache/cassandra/replication/MutationTrackingService.java
+++ b/src/java/org/apache/cassandra/replication/MutationTrackingService.java
@@ -18,6 +18,7 @@
package org.apache.cassandra.replication;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -28,6 +29,7 @@ import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
@@ -46,28 +48,35 @@ import org.apache.cassandra.concurrent.ScheduledExecutorPlus;
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.PartitionPosition;
import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Bounds;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Splitter;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.RequestFailure;
import org.apache.cassandra.gms.FailureDetector;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.locator.EndpointsForRange;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.metrics.MutationTrackingMetrics;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.repair.SyncTask;
+import org.apache.cassandra.repair.SyncTasks;
import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.schema.ReplicationParams;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.reads.tracked.TrackedLocalReads;
+import org.apache.cassandra.streaming.StreamOperation;
import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.Epoch;
import org.apache.cassandra.tcm.membership.NodeId;
import org.apache.cassandra.tcm.ClusterMetadataService;
import org.apache.cassandra.tcm.listeners.ChangeListener;
@@ -234,6 +243,35 @@ public class MutationTrackingService
}
}
+ // Requires that ranges is aligned to a single shard
+ public MutationId nextMutationId(String keyspace, Collection<Range<Token>> ranges)
+ {
+ shardLock.readLock().lock();
+ try
+ {
+ KeyspaceShards shards = getOrCreateShards(keyspace);
+ Shard shard = null;
+ for (Range<Token> range : ranges)
+ {
+ Shard curShard = shards.lookUp(range);
+ if (curShard == null)
+ throw new UnknownShardException(range, shards.groups);
+ if (shard == null)
+ shard = curShard;
+ else if (shard != curShard)
+ throw new IllegalStateException(String.format("Cannot generate a mutation ID for ranges (%s) that span across more than one shard (%s, %s)", ranges, shard, curShard));
+ }
+ Preconditions.checkNotNull(shard);
+ MutationId id = shard.nextId();
+ logger.trace("Created new mutation id {}", id);
+ return id;
+ }
+ finally
+ {
+ shardLock.readLock().unlock();
+ }
+ }
+
public void sentWriteRequest(Mutation mutation, IntHashSet toHostIds)
{
Preconditions.checkArgument(!mutation.id().isNone());
@@ -375,10 +413,10 @@ public class MutationTrackingService
logger.info("Creating tracked bulk transfers for keyspace '{}' SSTables {}...", keyspace, sstables);
KeyspaceShards shards = checkNotNull(keyspaceShards.get(keyspace));
- CoordinatedTransfers transfers = CoordinatedTransfers.create(keyspace, shards, sstables, cl);
+ TrackedImportTransfers transfers = TrackedImportTransfers.create(keyspace, shards, sstables, cl);
logger.info("Split input SSTables into transfers {}", transfers);
- for (CoordinatedTransfer transfer : transfers)
+ for (TrackedImportTransfer transfer : transfers)
transfer.execute();
}
finally
@@ -390,23 +428,52 @@ public class MutationTrackingService
public void received(PendingLocalTransfer transfer)
{
logger.debug("Received pending transfer for tracked table {}", transfer);
- LocalTransfers.instance().received(transfer);
+ TransferTrackingService.instance().received(transfer);
}
- void activateLocal(TransferActivation activation)
+ void activateLocal(ActivationRequest request)
{
- PendingLocalTransfer pending = LocalTransfers.instance().getPendingTransfer(activation.planId);
- if (pending == null)
- throw new IllegalStateException(String.format("Cannot activate unknown local pending transfer %s", activation));
- pending.activate(activation);
+ boolean committed = false;
+ String keyspace = request.keyspace;
+ Bounds<Token> bounds;
+ PendingLocalTransfer pending = null;
+
+ if (request.operation == StreamOperation.REPAIR)
+ {
+ bounds = new Bounds<>(request.range.left.nextValidToken(), request.range.right);
+
+ // A sync task does not necessarily stream to both replicas, which means there may be a plan ID without a
+ // pending local transfer. In this case, we simply treat this as an already committed transfer and update
+ // the required offsets in the log (unless we're just preparing).
+ committed = request.isCommit();
+
+ // If we have no plan ID, it means this replica did not participate in a sync.
+ if (request.planId != null)
+ pending = TransferTrackingService.instance().getPendingTransfer(request.planId);
+ }
+ else if (request.operation == StreamOperation.IMPORT)
+ {
+ pending = TransferTrackingService.instance().getPendingTransfer(request.planId);
+ if (pending == null)
+ throw new IllegalStateException(String.format("Cannot activate unknown local pending transfer %s", request));
+
+ bounds = ActivatedTransfers.covering(pending.sstables);
+ }
+ else
+ {
+ throw new IllegalArgumentException("Cannot activate transfer for stream operation " + request.operation);
+ }
+
+ if (pending != null)
+ committed = pending.activate(request, bounds);
shardLock.readLock().lock();
try
{
- if (activation.isCommit())
+ if (committed)
{
- keyspaceShards.get(pending.keyspace).lookUp(pending.range).finishActivation(pending, activation);
- incomingMutations.invokeListeners(activation.transferId);
+ keyspaceShards.get(keyspace).lookUp(request.range).finishActivation(bounds, request);
+ incomingMutations.invokeListeners(request.transferId);
}
}
finally
@@ -780,6 +847,66 @@ public class MutationTrackingService
forEachKeyspace(keyspace -> keyspace.collectDurablyReconciledOffsets(into));
}
+ public SyncTasks alignToShardBoundaries(Keyspace keyspace, List<SyncTask> tasks)
+ {
+ Preconditions.checkArgument(keyspace.getMetadata().replicationStrategy.replicationType.isTracked(), "Keyspace " + keyspace.getName() + " is not tracked");
+
+ KeyspaceShards shards = keyspaceShards.get(keyspace.getName());
+ Map<Shard, List<SyncTask>> tasksByShard = new HashMap<>();
+
+ // Shard ranges do not wrap, so unwrap the task ranges before we start comparing them.
+ for (SyncTask task : unwrapped(tasks))
+ {
+ Set<Shard> intersectingShards = new HashSet<>();
+ shards.forEachIntersectingShard(task.rangesToSync, intersectingShards::add);
+ for (Shard shard : intersectingShards)
+ {
+ // Ensure that we don't expand outside the ranges of the original sync tasks.
+ Set<Range<Token>> intersectingSyncRanges = new HashSet<>();
+ for (Range<Token> syncRange : task.rangesToSync)
+ intersectingSyncRanges.addAll(syncRange.intersectionWith(shard.range));
+
+ if (!intersectingSyncRanges.isEmpty())
+ tasksByShard.computeIfAbsent(shard, key -> new ArrayList<>()).add(task.withRanges(intersectingSyncRanges));
+ }
+ }
+
+ SyncTasks into = new SyncTasks();
+
+ for (Map.Entry<Shard, List<SyncTask>> entry : tasksByShard.entrySet())
+ {
+ Shard shard = entry.getKey();
+ Collection<SyncTask> syncTasks = entry.getValue();
+
+ // Assign a new transfer ID to each sync task and add to the tasks container
+ for (SyncTask task : syncTasks)
+ into.add(shard, task.withTransferId(shard.nextId()));
+ }
+
+ return into;
+ }
+
+ private static List<SyncTask> unwrapped(Collection<SyncTask> tasks)
+ {
+ List<SyncTask> unwrapped = new ArrayList<>();
+
+ for (SyncTask task : tasks)
+ {
+ List<Range<Token>> unwrappedRanges = new ArrayList<>();
+ for (Range<Token> range : task.rangesToSync)
+ {
+ if (range.isTrulyWrapAround())
+ unwrappedRanges.addAll(range.unwrap());
+ else
+ unwrappedRanges.add(range);
+ }
+
+ unwrapped.add(task.withRanges(unwrappedRanges));
+ }
+
+ return unwrapped;
+ }
+
public static class KeyspaceShards
{
private enum UpdateDecision
@@ -993,6 +1120,14 @@ public class MutationTrackingService
});
}
+ private void forEachIntersectingShard(Collection<Range<Token>> ranges, Consumer<Shard> consumer)
+ {
+ shards.forEach((range0, shard) -> {
+ if (shard.range.intersects(ranges))
+ consumer.accept(shard);
+ });
+ }
+
void collectShardReconciledOffsetsToBuilder(ReconciledLogSnapshot.Builder builder)
{
ReconciledKeyspaceOffsets.Builder keyspaceBuilder = builder.getKeyspaceBuilder(keyspace);
@@ -1098,8 +1233,9 @@ public class MutationTrackingService
private static class ReplicatedOffsetsBroadcaster
{
// TODO (later): a more intelligent heuristic for scheduling broadcasts
- private static final long TRANSIENT_BROADCAST_INTERVAL_MILLIS = 200;
- private static final long DURABLE_BROADCAST_INTERVAL_MILLIS = 60_000;
+ // TODO: Revert before merge, just increased frequency for test
+ private static final long TRANSIENT_BROADCAST_INTERVAL_MILLIS = 1_000;
+ private static final long DURABLE_BROADCAST_INTERVAL_MILLIS = 1_000;
private volatile boolean isPaused = false;
@@ -1148,11 +1284,13 @@ public class MutationTrackingService
private static class LogStatePersister implements Runnable
{
// TODO (expected): consider a different interval
- private static final long PERSIST_INTERVAL_MINUTES = 1;
+ // TODO: Revert before merge, just increased frequency for test
+ // private static final long PERSIST_INTERVAL_MILLIS = 60_000;
+ private static final long PERSIST_INTERVAL_MILLIS = 1_000;
void start()
{
- executor.scheduleWithFixedDelay(this, PERSIST_INTERVAL_MINUTES, PERSIST_INTERVAL_MINUTES, TimeUnit.MINUTES);
+ executor.scheduleWithFixedDelay(this, PERSIST_INTERVAL_MILLIS, PERSIST_INTERVAL_MILLIS, TimeUnit.MILLISECONDS);
}
@Override
@@ -1222,5 +1360,35 @@ public class MutationTrackingService
{
return service.keyspaceShards.get(keyspace);
}
+
+ /**
+ * Creates a test KeyspaceShards with the given shard ranges.
+ * The shards are created with minimal configuration suitable for testing.
+ */
+ public static KeyspaceShards createTestKeyspaceShards(String keyspace, Set<Range<Token>> shardRanges)
+ {
+ Map<Range<Token>, Shard> shards = new HashMap<>();
+ Map<Range<Token>, VersionedEndpoints.ForRange> groups = new HashMap<>();
+
+ int localNodeId = 1;
+ AtomicInteger hostLogId = new AtomicInteger(0);
+ LongSupplier logId = () -> CoordinatorLogId.asLong(localNodeId, hostLogId.getAndIncrement());
+ Participants participants = new Participants(List.of(localNodeId));
+ for (Range<Token> range : shardRanges)
+ {
+ shards.put(range, new Shard(localNodeId, keyspace, range, participants, logId, (s, l) -> {}));
+ groups.put(range, VersionedEndpoints.forRange(Epoch.EMPTY, EndpointsForRange.empty(range)));
+ }
+
+ return new KeyspaceShards(keyspace, shards, new ReplicaGroups(groups));
+ }
+
+ /**
+ * Sets the keyspace shards for testing purposes.
+ */
+ public static void setKeyspaceShards(MutationTrackingService service, String keyspace, KeyspaceShards shards)
+ {
+ service.keyspaceShards.put(keyspace, shards);
+ }
}
}
diff --git a/src/java/org/apache/cassandra/replication/PendingLocalTransfer.java b/src/java/org/apache/cassandra/replication/PendingLocalTransfer.java
index a2c0d833a7..05ad017311 100644
--- a/src/java/org/apache/cassandra/replication/PendingLocalTransfer.java
+++ b/src/java/org/apache/cassandra/replication/PendingLocalTransfer.java
@@ -33,6 +33,7 @@ import org.slf4j.LoggerFactory;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Bounds;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.sstable.format.SSTableReader;
@@ -48,7 +49,7 @@ import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
/**
* Represents a bulk data transfer received on a replica, from completion of streaming into the pending location,
* through activation when it's made visible to reads. Pending transfers are identified by their streaming plan ID,
- * and made live by {@link TransferActivation} which associates the streaming plan with a transfer ID that can be
+ * and made live by {@link ActivationRequest} which associates the streaming plan with a transfer ID that can be
* represented in mutation summaries.
*/
public class PendingLocalTransfer
@@ -133,16 +134,16 @@ public class PendingLocalTransfer
* preserve is a transfer is only added to the live set iff the transfer ID is present in its mutation summaries.
* <p>
* We don't validate checksums here, mostly because a transfer can be activated during a read, if one replica
- * missed the TransferActivation. Transfers should not be pending for very long, and should be protected by
+ * missed the {@link ActivationRequest}. Transfers should not be pending for very long, and should be protected by
* internode integrity checks provided by TLS.
* <p>
* Synchronized to prevent a single activation from running multiple times if requested during read reconciliation
* and in the background via {@link ActiveLogReconciler}.
*/
- public synchronized void activate(TransferActivation activation)
+ public synchronized boolean activate(ActivationRequest request, Bounds<Token> bounds)
{
if (activated)
- return;
+ return false;
Preconditions.checkState(isFullReplica());
@@ -152,21 +153,19 @@ public class PendingLocalTransfer
Preconditions.checkNotNull(cfs);
Preconditions.checkState(!sstables.isEmpty());
- if (activation.isPrepare())
+ if (request.isPrepare())
{
- logger.info("{} Not adding SSTables to live set for dryRun {}", logPrefix(), activation);
- return;
+ logger.info("{} Not adding SSTables to live set for dryRun {}", logPrefix(), request);
+ return false;
}
// Modify SSTables metadata to durably set transfer ID before importing
- ImmutableCoordinatorLogOffsets logOffsets = new ImmutableCoordinatorLogOffsets.Builder()
- .addTransfer(activation.transferId, sstables)
- .build();
+ ImmutableCoordinatorLogOffsets logOffsets =
+ new ImmutableCoordinatorLogOffsets.Builder().addTransfer(request.transferId, bounds).build();
// Ensure no lingering mutation IDs, only activation IDs
for (SSTableReader sstable : sstables)
{
- Preconditions.checkState(sstable.getCoordinatorLogOffsets().mutations().isEmpty());
try
{
sstable.mutateCoordinatorLogOffsetsAndReload(logOffsets);
@@ -185,7 +184,7 @@ public class PendingLocalTransfer
// Retain the original SSTables in pending/ dir on the coordinator, so future streams can get the originals, and
// we don't need to isolate activated SSTables during compaction
- boolean isCoordinator = activation.transferId.hostId == ClusterMetadata.current().myNodeId().id();
+ boolean isCoordinator = request.transferId.hostId == ClusterMetadata.current().myNodeId().id();
logger.debug("{} {} pending SSTables for activation to {}", isCoordinator ? "Copying" : "Moving", logPrefix(), dst);
dst.createFileIfNotExists();
@@ -210,7 +209,8 @@ public class PendingLocalTransfer
long finishedActivation = currentTimeMillis();
logger.info("{} Finished activating transfer {} in {} ms", logPrefix(), this, finishedActivation - startedActivation);
- LocalTransfers.instance().scheduleCleanup();
+ TransferTrackingService.instance().scheduleCleanup();
+ return true;
}
@Override
diff --git a/src/java/org/apache/cassandra/replication/Shard.java b/src/java/org/apache/cassandra/replication/Shard.java
index 5f7f1e7ee6..f282edf426 100644
--- a/src/java/org/apache/cassandra/replication/Shard.java
+++ b/src/java/org/apache/cassandra/replication/Shard.java
@@ -41,6 +41,7 @@ import org.apache.cassandra.db.PartitionPosition;
import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Bounds;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.locator.InetAddressAndPort;
@@ -58,9 +59,9 @@ public class Shard
private static final Logger logger = LoggerFactory.getLogger(Shard.class);
final int localNodeId;
- final String keyspace;
- final Range<Token> range;
- final Participants participants;
+ public final String keyspace;
+ public final Range<Token> range;
+ public final Participants participants;
private final LongSupplier logIdProvider;
private final BiConsumer<Shard, CoordinatorLog> onNewLog;
private final NonBlockingHashMapLong<CoordinatorLog> logs;
@@ -180,9 +181,9 @@ public class Shard
getOrCreate(mutationId).receivedWriteResponse(mutationId, fromHostId);
}
- void finishActivation(PendingLocalTransfer transfer, TransferActivation activation)
+ void finishActivation(Bounds<Token> bounds, ActivationRequest activation)
{
- getOrCreate(activation.transferId).finishActivation(transfer, activation);
+ getOrCreate(activation.transferId).finishActivation(bounds, activation);
}
void receivedActivationResponse(CoordinatedTransfer transfer, InetAddressAndPort onHost)
@@ -406,6 +407,17 @@ public class Shard
logs.values().forEach(log -> keyspaceBuilder.put(log.logId, log.collectReconciledOffsets(), range));
}
+ @Override
+ public String toString()
+ {
+ return "Shard{" +
+ "participants=" + participants +
+ ", range=" + range +
+ ", keyspace='" + keyspace + '\'' +
+ ", localNodeId=" + localNodeId +
+ '}';
+ }
+
public DebugInfo getDebugInfo()
{
SortedMap<CoordinatorLogId, CoordinatorLog.DebugInfo> logDebugState = new TreeMap<>(Comparator.comparing(CoordinatorLogId::asLong));
diff --git a/src/java/org/apache/cassandra/replication/TrackedImportTransfer.java b/src/java/org/apache/cassandra/replication/TrackedImportTransfer.java
new file mode 100644
index 0000000000..b0fd787adb
--- /dev/null
+++ b/src/java/org/apache/cassandra/replication/TrackedImportTransfer.java
@@ -0,0 +1,421 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.replication;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Supplier;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.streaming.CassandraOutgoingFile;
+import org.apache.cassandra.dht.Bounds;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.exceptions.RequestFailure;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.locator.AbstractReplicationStrategy;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.RequestCallbackWithFailure;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.streaming.OutgoingStream;
+import org.apache.cassandra.streaming.StreamException;
+import org.apache.cassandra.streaming.StreamOperation;
+import org.apache.cassandra.streaming.StreamPlan;
+import org.apache.cassandra.streaming.StreamResultFuture;
+import org.apache.cassandra.streaming.StreamState;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.membership.NodeId;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.Throwables;
+import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.utils.concurrent.AsyncFuture;
+import org.apache.cassandra.utils.concurrent.Future;
+
+import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.COMMITTED;
+import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.COMMITTING;
+import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.PREPARE_FAILED;
+import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.STREAM_COMPLETE;
+import static org.apache.cassandra.replication.ActivationRequest.Phase;
+
+/**
+ * Tracked imports mostly follow the default coordination process. They are notable in two ways. The first is that
+ * {@link #prepare} strictly verifies that streaming has completed on replicas. The second is that activation is based
+ * on the {@link #bounds} generated from {@link #sstables} rather than a sync range (as is the case with repair).
+ * <p>
+ * For simplicity, the coordinator streams to itself rather than using direct file copy. This ensures we can use the
+ * same lifecycle management for crash-safety and atomic add.
+ */
+public class TrackedImportTransfer extends CoordinatedTransfer
+{
+ private static final Logger logger = LoggerFactory.getLogger(TrackedImportTransfer.class);
+
+ final Collection<SSTableReader> sstables;
+ private final ConsistencyLevel cl;
+
+ @VisibleForTesting
+ TrackedImportTransfer(Range<Token> range, MutationId id)
+ {
+ super(id, null, range);
+ this.sstables = Collections.emptyList();
+ this.cl = null;
+ }
+
+ TrackedImportTransfer(String keyspace, Range<Token> range, Participants participants, Collection<SSTableReader> sstables, ConsistencyLevel cl, Supplier<MutationId> nextId)
+ {
+ super(nextId.get(), participants, keyspace, range);
+ this.sstables = sstables;
+ this.cl = cl;
+
+ ClusterMetadata cm = ClusterMetadata.current();
+
+ for (int i = 0; i < participants.size(); i++)
+ {
+ InetAddressAndPort addr = cm.directory.getNodeAddresses(new NodeId(participants.get(i))).broadcastAddress;
+ this.streamResults.put(Pair.create(FBUtilities.getBroadcastAddressAndPort(), addr), SingleTransferResult.Init());
+ }
+ }
+
+ @Override
+ Bounds<Token> bounds()
+ {
+ return ActivatedTransfers.covering(sstables);
+ }
+
+ void execute()
+ {
+ logger.debug("{} Executing tracked import transfer {}", logPrefix(), this);
+ TransferTrackingService.instance().save(this);
+ stream();
+ }
+
+ private void stream()
+ {
+ // TODO: Don't stream multiple copies over the WAN, send one copy and indicate forwarding
+ List<Future<Void>> streaming = new ArrayList<>(streamResults.size());
+ for (Pair<InetAddressAndPort, InetAddressAndPort> pair : streamResults.keySet())
+ {
+ Future<Void> stream = TransferTrackingService.instance().executor.submit(() -> {
+ stream(pair);
+ return null;
+ });
+ streaming.add(stream);
+ }
+
+ // Wait for all streams to complete, so we can clean up after failures. If we exit at the first failure, a
+ // future stream can complete.
+ LinkedList<Throwable> failures = null;
+ for (Future<Void> stream : streaming)
+ {
+ try
+ {
+ stream.get();
+ }
+ catch (InterruptedException | ExecutionException e)
+ {
+ if (failures == null)
+ failures = new LinkedList<>();
+ failures.add(e);
+ logger.error("{} Failed transfer due to", logPrefix(), e);
+ }
+ }
+
+ if (failures != null && !failures.isEmpty())
+ {
+ Throwable failure = failures.element();
+ Throwable cause = failure instanceof ExecutionException ? failure.getCause() : failure;
+ maybeCleanupFailedStreams(cause);
+
+ String msg = String.format("Failed streaming on %s instance(s): %s", failures.size(), failures);
+ throw new RuntimeException(msg, Throwables.unchecked(cause));
+ }
+
+ logger.info("{} All streaming completed successfully", logPrefix());
+ }
+
+ private boolean sufficient()
+ {
+ AbstractReplicationStrategy ars = Keyspace.open(keyspace).getReplicationStrategy();
+ int blockFor = cl.blockFor(ars);
+ int responses = 0;
+ for (SingleTransferResult result : streamResults.values())
+ {
+ if (result.state == STREAM_COMPLETE)
+ responses++;
+ }
+ return responses >= blockFor;
+ }
+
+ void stream(Pair<InetAddressAndPort, InetAddressAndPort> pair)
+ {
+ SingleTransferResult result;
+ try
+ {
+ result = streamTask(pair.right);
+ }
+ catch (StreamException | ExecutionException | InterruptedException | TimeoutException e)
+ {
+ Throwable cause = e instanceof ExecutionException ? e.getCause() : e;
+ markStreamFailure(pair, cause);
+ throw Throwables.unchecked(cause);
+ }
+
+ try
+ {
+ streamComplete(pair, result);
+ }
+ catch (ExecutionException | InterruptedException | TimeoutException e)
+ {
+ Throwable cause = e instanceof ExecutionException ? e.getCause() : e;
+ throw Throwables.unchecked(cause);
+ }
+ }
+
+ private void markStreamFailure(Pair<InetAddressAndPort, InetAddressAndPort> pair, Throwable cause)
+ {
+ TimeUUID planId;
+ if (cause instanceof StreamException)
+ planId = ((StreamException) cause).finalState.planId;
+ else
+ planId = null;
+ streamResults.computeIfPresent(pair, (p, result) -> result.streamFailed(planId));
+ }
+
+ /**
+ * This shouldn't throw an exception, even if we fail to notify peers of the streaming failure.
+ */
+ private void maybeCleanupFailedStreams(Throwable cause)
+ {
+ try
+ {
+ boolean purgeable = TransferTrackingService.instance().purger.test(this);
+ if (!purgeable)
+ return;
+
+ notifyFailure();
+ TransferTrackingService.instance().scheduleCleanup();
+ }
+ catch (Throwable t)
+ {
+ if (cause != null)
+ t.addSuppressed(cause);
+ logger.error("{} Failed to notify peers of stream failure", logPrefix(), t);
+ }
+ }
+
+ private void streamComplete(Pair<InetAddressAndPort, InetAddressAndPort> pair, SingleTransferResult result) throws ExecutionException, InterruptedException, TimeoutException
+ {
+ streamResults.put(pair, result);
+ logger.info("{} Completed streaming for pair {}, {}", logPrefix(), pair, this);
+ maybeActivate();
+ }
+
+ synchronized void maybeActivate()
+ {
+ // If any activations have already been sent out, send new activations to any received plans that have not yet
+ // been activated
+ boolean anyActivated = false;
+ Set<Pair<InetAddressAndPort, InetAddressAndPort>> awaitingActivation = new HashSet<>();
+ for (Map.Entry<Pair<InetAddressAndPort, InetAddressAndPort>, SingleTransferResult> entry : streamResults.entrySet())
+ {
+ Pair<InetAddressAndPort, InetAddressAndPort> peer = entry.getKey();
+ SingleTransferResult result = entry.getValue();
+ if (result.state == COMMITTING || result.state == COMMITTED)
+ {
+ anyActivated = true;
+ }
+ else if (result.state == STREAM_COMPLETE)
+ awaitingActivation.add(peer);
+ }
+ if (anyActivated && !awaitingActivation.isEmpty())
+ {
+ logger.debug("{} Transfer already activated on some peers, sending activations to remaining: {}", logPrefix(), awaitingActivation);
+ activate(awaitingActivation);
+ return;
+ }
+ // If no activations have been sent out, check whether we have enough planIds back to meet the required CL
+ else if (sufficient())
+ {
+ Set<Pair<InetAddressAndPort, InetAddressAndPort>> pairs = new HashSet<>();
+ for (Map.Entry<Pair<InetAddressAndPort, InetAddressAndPort>, SingleTransferResult> entry : streamResults.entrySet())
+ {
+ Pair<InetAddressAndPort, InetAddressAndPort> pair = entry.getKey();
+ SingleTransferResult result = entry.getValue();
+ if (result.state == STREAM_COMPLETE)
+ pairs.add(pair);
+ }
+ logger.debug("{} Transfer meets consistency level {}, sending activations to {}", logPrefix(), cl, pairs);
+ activate(pairs);
+ return;
+ }
+
+ logger.debug("{} Nothing to activate", logPrefix());
+ }
+
+ protected void prepare(Collection<Pair<InetAddressAndPort, InetAddressAndPort>> targets)
+ {
+ // First phase ensures data is present on disk, then second phase does the actual import. This ensures that if
+ // something goes wrong (like a topology change during import), we don't have divergence.
+ class Prepare extends AsyncFuture<Void> implements RequestCallbackWithFailure<ActivationResponse>
+ {
+ final AtomicInteger responses = new AtomicInteger(0);
+
+ public Prepare()
+ {
+ responses.addAndGet(targets.size());
+ }
+
+ @Override
+ public void onResponse(Message<ActivationResponse> msg)
+ {
+ logger.debug("{} Got prepare response from: {}", logPrefix(), msg.from());
+ if (responses.decrementAndGet() == 0)
+ trySuccess(null);
+ }
+
+ @Override
+ public void onFailure(InetAddressAndPort from, RequestFailure failure)
+ {
+ logger.debug("{} Got prepare failure {} from {}", logPrefix(), failure, from);
+
+ // Any failure fails the whole transfer here, so marking all results for streams to this remote as
+ // failed is harmless and allows the purging logic to clean up doomed transfer artifacts.
+ streamResults.forEach((pair, result) -> {
+ if (pair.right.equals(from) && result.canTransition(PREPARE_FAILED))
+ streamResults.put(pair, result.prepareFailed());
+ });
+
+ tryFailure(new RuntimeException("Tracked transfer failed during PREPARE on " + from + " due to " + failure.reason));
+ }
+ }
+
+ Prepare prepare = new Prepare();
+ for (Pair<InetAddressAndPort, InetAddressAndPort> target : targets)
+ {
+ ActivationRequest activation = createActivation(target, Phase.PREPARE);
+ Message<ActivationRequest> msg = Message.out(Verb.TRACKED_TRANSFER_ACTIVATE_REQ, activation);
+ logger.debug("{} Sending prepare {} to peer {}", logPrefix(), activation, target.right);
+ MessagingService.instance().sendWithCallback(msg, target.right, prepare);
+ streamResults.computeIfPresent(target, (peer0, result) -> result.preparing());
+ }
+ try
+ {
+ prepare.get();
+ }
+ catch (InterruptedException | ExecutionException e)
+ {
+ Throwable cause = e instanceof ExecutionException ? e.getCause() : e;
+ throw Throwables.unchecked(cause);
+ }
+ }
+
+ @Override
+ protected ActivationRequest createActivation(Pair<InetAddressAndPort, InetAddressAndPort> pair, Phase phase)
+ {
+ return new ActivationRequest(StreamOperation.IMPORT, pair, phase, id(), ClusterMetadata.current().myNodeId(), range, keyspace, streamResults.get(pair).planId());
+ }
+
+ private SingleTransferResult streamTask(InetAddressAndPort to) throws StreamException, ExecutionException, InterruptedException, TimeoutException
+ {
+ StreamPlan plan = new StreamPlan(StreamOperation.IMPORT);
+
+ // No need to flush, only using non-live SSTables already on disk
+ plan.flushBeforeTransfer(false);
+
+ for (SSTableReader sstable : sstables)
+ {
+ List<Range<Token>> ranges = Collections.singletonList(range);
+ List<SSTableReader.PartitionPositionBounds> positions = sstable.getPositionsForRanges(ranges);
+ long estimatedKeys = sstable.estimatedKeysForRanges(ranges);
+ OutgoingStream stream = new CassandraOutgoingFile(StreamOperation.IMPORT, sstable.ref(), positions, ranges, estimatedKeys);
+ plan.transferStreams(to, Collections.singleton(stream));
+ }
+
+ long timeout = DatabaseDescriptor.getStreamTransferTaskTimeout().toMilliseconds();
+
+ logger.info("{} Starting streaming transfer {} to peer {}", logPrefix(), this, to);
+ StreamResultFuture execute = plan.execute();
+ StreamState state;
+ try
+ {
+ state = execute.get(timeout, TimeUnit.MILLISECONDS);
+ logger.debug("{} Completed streaming transfer {} to peer {}", logPrefix(), this, to);
+ }
+ catch (InterruptedException | ExecutionException | TimeoutException e)
+ {
+ logger.error("Stream session failed with error", e);
+ throw e;
+ }
+
+ if (state.hasFailedSession() || state.hasAbortedSession())
+ throw new StreamException(state, "Stream failed due to failed or aborted sessions");
+
+ // If the SSTable doesn't contain any rows in the provided range, no streams delivered, nothing to activate
+ if (state.sessions().isEmpty())
+ return SingleTransferResult.Noop();
+
+ return SingleTransferResult.StreamComplete(plan.planId());
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (o == null || getClass() != o.getClass()) return false;
+ TrackedImportTransfer that = (TrackedImportTransfer) o;
+ return Objects.equals(keyspace, that.keyspace) && Objects.equals(range, that.range) && cl == that.cl && Objects.equals(streamResults, that.streamResults);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(keyspace, range, cl, streamResults);
+ }
+
+ @Override
+ public String toString()
+ {
+ return "TrackedImportTransfer{" +
+ "keyspace='" + keyspace + '\'' +
+ ", range=" + range +
+ ", cl=" + cl +
+ ", streamResults=" + streamResults +
+ ", sstables=" + sstables +
+ ", streamResults=" + streamResults +
+ '}';
+ }
+}
diff --git a/src/java/org/apache/cassandra/replication/CoordinatedTransfers.java b/src/java/org/apache/cassandra/replication/TrackedImportTransfers.java
similarity index 76%
rename from src/java/org/apache/cassandra/replication/CoordinatedTransfers.java
rename to src/java/org/apache/cassandra/replication/TrackedImportTransfers.java
index 2adb546fc5..250c240e0a 100644
--- a/src/java/org/apache/cassandra/replication/CoordinatedTransfers.java
+++ b/src/java/org/apache/cassandra/replication/TrackedImportTransfers.java
@@ -33,20 +33,20 @@ import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.utils.Interval;
/**
- * Factory and container for creating multiple {@link CoordinatedTransfer} instances from a collection
+ * Factory and container for creating multiple {@link TrackedImportTransfer} instances from a collection
* of SSTables, partitioned by {@link MutationTrackingService.KeyspaceShards}, which are aligned to replica ownership
* ranges. Each shard receives its own CoordinatedTransfer instance, which can be executed independently.
*/
-class CoordinatedTransfers implements Iterable<CoordinatedTransfer>
+public class TrackedImportTransfers implements Iterable<TrackedImportTransfer>
{
- private final Collection<CoordinatedTransfer> transfers;
+ private final Collection<TrackedImportTransfer> transfers;
- private CoordinatedTransfers(Collection<CoordinatedTransfer> transfers)
+ private TrackedImportTransfers(Collection<TrackedImportTransfer> transfers)
{
this.transfers = transfers;
}
- static CoordinatedTransfers create(String keyspace, MutationTrackingService.KeyspaceShards shards, Collection<SSTableReader> sstables, ConsistencyLevel cl)
+ static TrackedImportTransfers create(String keyspace, MutationTrackingService.KeyspaceShards shards, Collection<SSTableReader> sstables, ConsistencyLevel cl)
{
// Clean up incoming SSTables to remove any existing untrusted CoordinatorLogOffsets
for (SSTableReader sstable : sstables)
@@ -62,7 +62,7 @@ class CoordinatedTransfers implements Iterable<CoordinatedTransfer>
}
SSTableIntervalTree intervals = SSTableIntervalTree.buildSSTableIntervalTree(sstables);
- List<CoordinatedTransfer> transfers = new ArrayList<>();
+ List<TrackedImportTransfer> transfers = new ArrayList<>();
shards.forEachShard(shard -> {
Range<Token> range = shard.tokenRange();
@@ -70,14 +70,14 @@ class CoordinatedTransfers implements Iterable<CoordinatedTransfer>
if (sstablesForRange.isEmpty())
return;
- CoordinatedTransfer transfer = new CoordinatedTransfer(keyspace, range, shard.participants, sstablesForRange, cl, shard::nextId);
+ TrackedImportTransfer transfer = new TrackedImportTransfer(keyspace, range, shard.participants, sstablesForRange, cl, shard::nextId);
transfers.add(transfer);
});
- return new CoordinatedTransfers(transfers);
+ return new TrackedImportTransfers(transfers);
}
@Override
- public Iterator<CoordinatedTransfer> iterator()
+ public Iterator<TrackedImportTransfer> iterator()
{
return transfers.iterator();
}
diff --git a/src/java/org/apache/cassandra/replication/TrackedRepairTransfer.java b/src/java/org/apache/cassandra/replication/TrackedRepairTransfer.java
new file mode 100644
index 0000000000..c1e7479cfa
--- /dev/null
+++ b/src/java/org/apache/cassandra/replication/TrackedRepairTransfer.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.replication;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+
+import com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.repair.SyncStat;
+import org.apache.cassandra.repair.SyncTask;
+import org.apache.cassandra.repair.SyncTasks;
+import org.apache.cassandra.streaming.StreamOperation;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.membership.NodeId;
+import org.apache.cassandra.utils.Pair;
+
+/**
+ * Repair sync tasks (that stream SSTable contents) must be integrated with Mutation Tracking's bulk transfer handling
+ * because any data that is not present on all instances must be expressed as unreconciled in the log, since read
+ * reconciliations depend on the log state to guarantee monotonicity of subsequent reads. Streaming sessions for full
+ * repair can complete on some instances before others, so we need to represent those completed sessions as unreconciled
+ * in the log.
+ * <p>
+ * Tracked repairs mostly follow the default coordination process. They are notable in two ways. The first is that
+ * {@link #prepare} is essentially a no-op. Sync completion assumes that streamed SSTables have arrived at replicas.
+ * The second is that activation is based on the {@link #bounds} implied by the {@link SyncTask} owned by the transfer.
+ */
+public class TrackedRepairTransfer extends CoordinatedTransfer
+{
+ private static final Logger logger = LoggerFactory.getLogger(TrackedRepairTransfer.class);
+
+ public TrackedRepairTransfer(SyncTasks.ShardedSyncTask shardedTask)
+ {
+ this(shardedTask.task.getTransferId(), shardedTask.participants, shardedTask.task, shardedTask.keyspace, shardedTask.range);
+ }
+
+ private TrackedRepairTransfer(ShortMutationId id, Participants participants, SyncTask task, String keyspace, Range<Token> range)
+ {
+ super(id, keyspace, range);
+
+ Set<Integer> replicaNodeIds = participants.asSet();
+ Set<InetAddressAndPort> participating = new HashSet<>();
+
+ participating.add(task.nodePair().coordinator);
+ participating.add(task.nodePair().peer);
+ streamResults.put(Pair.create(task.nodePair().coordinator, task.nodePair().peer), SingleTransferResult.Init());
+
+ ClusterMetadata cm = ClusterMetadata.current();
+ for (Integer replicaNodeId : replicaNodeIds)
+ {
+ InetAddressAndPort addr = cm.directory.endpoint(new NodeId(replicaNodeId));
+
+ // Need to activate on all replicas, not just ones with SyncTasks. For replicas that don't receive any data
+ // as part of a repair, they still need to activate the transfer ID as a no-op, to allow read
+ // reconciliations to complete.
+ if (!participating.contains(addr))
+ streamResults.put(Pair.create(addr, addr), SingleTransferResult.EmptySync());
+ }
+ }
+
+ /**
+ * When every {@link SyncTask} for a repair has completed, follow the bulk transfer activation path to safely make
+ * the new data live and tracked in the log. This needs to include all replicas of {@link #id()}, even those that
+ * did not receive anything as part of the repair. Otherwise, any read reconciliation will fail to complete.
+ */
+ public void activate(SyncStat stat)
+ {
+ Preconditions.checkNotNull(stat.planId, "A complete sync should have a plan ID");
+
+ logger.debug("{} Activating {}", logPrefix(), this);
+
+ Pair<InetAddressAndPort, InetAddressAndPort> pair = Pair.create(stat.nodes.coordinator, stat.nodes.peer);
+ // Account for syncs where data streams both ways. (i.e. We need to activate both receivers.)
+ streamResults.put(pair, SingleTransferResult.StreamComplete(stat.planId));
+ streamResults.put(pair.reverse(), SingleTransferResult.StreamComplete(stat.planId));
+
+ activate(streamResults.keySet());
+ }
+
+ @Override
+ protected void prepare(Collection<Pair<InetAddressAndPort, InetAddressAndPort>> targets)
+ {
+ // There should be no need to prepare on stream completion for repair, as not all activations will involve a
+ // local pending transfer to begin with.
+ for (Pair<InetAddressAndPort, InetAddressAndPort> target : targets)
+ streamResults.computeIfPresent(target, (pair, result) -> result.preparing());
+ }
+
+ @Override
+ protected ActivationRequest createActivation(Pair<InetAddressAndPort, InetAddressAndPort> pair, ActivationRequest.Phase phase)
+ {
+ return new ActivationRequest(StreamOperation.REPAIR, pair, phase, id(), ClusterMetadata.current().myNodeId(), range, keyspace, streamResults.get(pair).planId());
+ }
+
+ @Override
+ public String toString()
+ {
+ return "TrackedRepairTransfer{" +
+ "keyspace='" + keyspace + '\'' +
+ ", range=" + range +
+ ", streamResults=" + streamResults +
+ '}';
+ }
+}
diff --git a/src/java/org/apache/cassandra/replication/TransferActivation.java b/src/java/org/apache/cassandra/replication/TransferActivation.java
deleted file mode 100644
index ba0c918286..0000000000
--- a/src/java/org/apache/cassandra/replication/TransferActivation.java
+++ /dev/null
@@ -1,197 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.replication;
-
-import java.io.IOException;
-import java.util.Objects;
-
-import com.google.common.base.Preconditions;
-
-import org.apache.cassandra.db.TypeSizes;
-import org.apache.cassandra.exceptions.RequestFailureReason;
-import org.apache.cassandra.io.IVersionedSerializer;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.net.IVerbHandler;
-import org.apache.cassandra.net.Message;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.net.NoPayload;
-import org.apache.cassandra.tcm.ClusterMetadata;
-import org.apache.cassandra.tcm.membership.NodeId;
-import org.apache.cassandra.utils.TimeUUID;
-
-/**
- * Activation transitions a {@link PendingLocalTransfer} from being pending (durable on disk but not yet visible to
- * reads) to live (visible to reads and compactions), by associating the streaming plan ID with a mutation ID, referred
- * to as the transfer ID.
- * <p>
- * See {@link CoordinatedTransfer} for the lifecycle of a transfer and when a {@link TransferActivation} is sent.
- */
-public class TransferActivation
-{
- public final TimeUUID planId;
- public final ShortMutationId transferId;
- public final NodeId coordinatorId;
- public final Phase phase;
-
- public enum Phase
- {
- PREPARE(0),
- COMMIT(1);
-
- private final int id;
- private static final Phase[] ids;
- static
- {
- ids = new Phase[values().length];
- for (Phase phase : values())
- ids[phase.id] = phase;
- }
-
- Phase(int id)
- {
- this.id = id;
- }
-
- static Phase from(int id)
- {
- Phase phase = ids[id];
- Preconditions.checkState(phase.id == id);
- return phase;
- }
- }
-
- public TransferActivation(CoordinatedTransfer transfer, InetAddressAndPort peer, Phase phase)
- {
- this(transfer.streamResults.get(peer).planId(), transfer.id(), ClusterMetadata.current().myNodeId(), phase);
- }
-
- TransferActivation(TimeUUID planId, ShortMutationId transferId, NodeId coordinatorId, Phase phase)
- {
- Preconditions.checkArgument(!transferId.isNone());
- Preconditions.checkNotNull(planId);
- Preconditions.checkNotNull(coordinatorId);
- this.planId = planId;
- this.transferId = transferId;
- this.coordinatorId = coordinatorId;
- this.phase = phase;
- }
-
- ShortMutationId id()
- {
- return transferId;
- }
-
- public boolean isPrepare()
- {
- return phase == Phase.PREPARE;
- }
-
- public boolean isCommit()
- {
- return phase == Phase.COMMIT;
- }
-
- public void apply()
- {
- MutationTrackingService.instance.activateLocal(this);
- }
-
- public static final Serializer serializer = new Serializer();
-
- public static class Serializer implements IVersionedSerializer<TransferActivation>
- {
- @Override
- public void serialize(TransferActivation activate, DataOutputPlus out, int version) throws IOException
- {
- TimeUUID.Serializer.instance.serialize(activate.planId, out, version);
- ShortMutationId.serializer.serialize(activate.id(), out, version);
- NodeId.messagingSerializer.serialize(activate.coordinatorId, out, version);
- out.writeByte(activate.phase.id);
- }
-
- @Override
- public TransferActivation deserialize(DataInputPlus in, int version) throws IOException
- {
- TimeUUID planId = TimeUUID.Serializer.instance.deserialize(in, version);
- ShortMutationId id = ShortMutationId.serializer.deserialize(in, version);
- NodeId coordinatorId = NodeId.messagingSerializer.deserialize(in, version);
- Phase phase = Phase.from(in.readByte());
- return new TransferActivation(planId, id, coordinatorId, phase);
- }
-
- @Override
- public long serializedSize(TransferActivation activate, int version)
- {
- long size = 0;
- size += TimeUUID.Serializer.instance.serializedSize(activate.planId, version);
- size += ShortMutationId.serializer.serializedSize(activate.id(), version);
- size += NodeId.messagingSerializer.serializedSize(activate.coordinatorId, version);
- size += TypeSizes.BYTE_SIZE; // Enum ordinal
- return size;
- }
- }
-
- public static class VerbHandler implements IVerbHandler<TransferActivation>
- {
- @Override
- public void doVerb(Message<TransferActivation> msg) throws IOException
- {
- LocalTransfers.instance().executor.submit(() -> {
- try
- {
- msg.payload.apply();
- MessagingService.instance().respond(NoPayload.noPayload, msg);
- }
- catch (Throwable t)
- {
- MessagingService.instance().respondWithFailure(RequestFailureReason.forException(t), msg);
- }
- });
- }
- }
-
- public static final VerbHandler verbHandler = new VerbHandler();
-
- @Override
- public String toString()
- {
- return "TransferActivation{" +
- ", planId=" + planId +
- ", transferId=" + transferId +
- ", coordinatorId=" + coordinatorId +
- ", phase=" + phase +
- '}';
- }
-
- @Override
- public boolean equals(Object o)
- {
- if (o == null || getClass() != o.getClass()) return false;
- TransferActivation that = (TransferActivation) o;
- return Objects.equals(planId, that.planId) && Objects.equals(transferId, that.transferId) && Objects.equals(coordinatorId, that.coordinatorId) && phase == that.phase;
- }
-
- @Override
- public int hashCode()
- {
- return Objects.hash(planId, transferId, coordinatorId, phase);
- }
-}
diff --git a/src/java/org/apache/cassandra/replication/TransferFailed.java b/src/java/org/apache/cassandra/replication/TransferFailed.java
index 6a956df870..ef86ba8a77 100644
--- a/src/java/org/apache/cassandra/replication/TransferFailed.java
+++ b/src/java/org/apache/cassandra/replication/TransferFailed.java
@@ -28,7 +28,7 @@ import org.apache.cassandra.utils.TimeUUID;
/**
* Notification from coordinator to replicas when a bulk data transfer fails, triggering cleanup of the pending
* transfer state.
- * @see CoordinatedTransfer
+ * @see TrackedImportTransfer
* @see PendingLocalTransfer
*/
public class TransferFailed
diff --git a/src/java/org/apache/cassandra/replication/LocalTransfers.java b/src/java/org/apache/cassandra/replication/TransferTrackingService.java
similarity index 52%
rename from src/java/org/apache/cassandra/replication/LocalTransfers.java
rename to src/java/org/apache/cassandra/replication/TransferTrackingService.java
index 876bbeb47e..8c85cbbb3e 100644
--- a/src/java/org/apache/cassandra/replication/LocalTransfers.java
+++ b/src/java/org/apache/cassandra/replication/TransferTrackingService.java
@@ -18,25 +18,39 @@
package org.apache.cassandra.replication;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executor;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import javax.annotation.Nullable;
import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.FutureCallback;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.concurrent.ExecutorPlus;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.IVerbHandler;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.net.NoPayload;
+import org.apache.cassandra.repair.RepairJob;
+import org.apache.cassandra.repair.SyncStat;
+import org.apache.cassandra.repair.SyncTask;
+import org.apache.cassandra.repair.SyncTasks;
import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.utils.concurrent.AsyncPromise;
+import org.apache.cassandra.utils.concurrent.Future;
import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
@@ -50,9 +64,9 @@ import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFac
* TODO: Make changes to pending set durable with SystemKeyspace.savePendingLocalTransfer(transfer)?
* TODO: Add vtable for visibility into local and coordinated transfers
*/
-public class LocalTransfers
+public class TransferTrackingService
{
- private static final Logger logger = LoggerFactory.getLogger(LocalTransfers.class);
+ private static final Logger logger = LoggerFactory.getLogger(TransferTrackingService.class);
private final ReadWriteLock lock = new ReentrantReadWriteLock();
private final Map<ShortMutationId, CoordinatedTransfer> coordinating = new ConcurrentHashMap<>();
@@ -60,19 +74,18 @@ public class LocalTransfers
final ExecutorPlus executor = executorFactory().pooled("LocalTrackedTransfers", Integer.MAX_VALUE);
- private static final LocalTransfers instance = new LocalTransfers();
- static LocalTransfers instance()
+ private static final TransferTrackingService instance = new TransferTrackingService();
+ public static TransferTrackingService instance()
{
return instance;
}
- void save(CoordinatedTransfer transfer)
+ void save(TrackedImportTransfer transfer)
{
lock.writeLock().lock();
try
{
- CoordinatedTransfer existing = coordinating.put(transfer.id(), transfer);
- Preconditions.checkState(existing == null);
+ saveInternal(transfer);
}
finally
{
@@ -80,13 +93,24 @@ public class LocalTransfers
}
}
- void activating(CoordinatedTransfer transfer)
+ private void saveInternal(CoordinatedTransfer transfer)
+ {
+ Preconditions.checkNotNull(transfer.id(), "Cannot coordinate a transfar with no ID");
+ logger.debug("{} Saving {}", transfer.logPrefix(), transfer);
+ CoordinatedTransfer existing = coordinating.put(transfer.id(), transfer);
+ Preconditions.checkState(existing == null, "Attempted to save transfer multiple times");
+ }
+
+ void received(PendingLocalTransfer transfer)
{
- Preconditions.checkNotNull(transfer.id());
lock.writeLock().lock();
try
{
- coordinating.put(transfer.id(), transfer);
+ logger.debug("received: {}", transfer);
+ Preconditions.checkState(!transfer.sstables.isEmpty());
+
+ PendingLocalTransfer existing = local.put(transfer.planId, transfer);
+ Preconditions.checkState(existing == null);
}
finally
{
@@ -94,16 +118,21 @@ public class LocalTransfers
}
}
- void received(PendingLocalTransfer transfer)
+ /**
+ * Track a repair as a set of {@link TrackedRepairTransfer} instances corresponding to sync tasks prior to task
+ * execution so when the syncs are done, we can activate them via {@link ActivationRequest} or fail by
+ * sending {@link TransferFailed} to all replicas. In other words, one {@link RepairJob} will have as many
+ * transfers as sync tasks.
+ */
+ public void onRepairSyncExecution(SyncTasks tasks)
{
lock.writeLock().lock();
try
{
- logger.debug("received: {}", transfer);
- Preconditions.checkState(!transfer.sstables.isEmpty());
-
- PendingLocalTransfer existing = local.put(transfer.planId, transfer);
- Preconditions.checkState(existing == null);
+ tasks.apply((SyncTasks.ShardedSyncTask shardedTask) -> {
+ TrackedRepairTransfer transfer = new TrackedRepairTransfer(shardedTask);
+ saveInternal(transfer);
+ });
}
finally
{
@@ -111,6 +140,110 @@ public class LocalTransfers
}
}
+ /**
+ * Begin activation for transfers whose repair sync tasks have completed.
+ */
+ public Future<List<SyncStat>> onRepairSyncCompletion(RepairJob job, Future<List<SyncStat>> syncCompletion, Executor executor)
+ {
+ AsyncPromise<List<SyncStat>> activationFuture = new AsyncPromise<>();
+
+ syncCompletion.addCallback(new FutureCallback<>()
+ {
+ @Override
+ public void onSuccess(List<SyncStat> syncs)
+ {
+ logger.info("Completed syncs {}. Coordinating: {} Local: {}", syncs, coordinating, local);
+
+ List<Pair<SyncStat, TrackedRepairTransfer>> transfersToActivate = new ArrayList<>();
+
+ lock.writeLock().lock();
+ try
+ {
+ // Look up transfers while holding the lock
+ for (SyncStat sync : syncs)
+ {
+ TrackedRepairTransfer transfer = (TrackedRepairTransfer) coordinating.get(sync.transferId);
+ transfersToActivate.add(Pair.create(sync, transfer));
+ }
+ }
+ finally
+ {
+ lock.writeLock().unlock();
+ }
+
+ // Activate transfers WITHOUT holding the lock (activate() acquires its own locks and can block)
+ for (Pair<SyncStat, TrackedRepairTransfer> pair : transfersToActivate)
+ {
+ TrackedRepairTransfer transfer = pair.right;
+
+ try
+ {
+ logger.info("{} Activating transfer...", transfer.logPrefix());
+ transfer.activate(pair.left);
+ }
+ catch (Throwable t)
+ {
+ // Note: cleanup will be triggered automatically when the async COMMIT responses complete
+ logger.error("{} Failed to activate transfer", transfer.logPrefix(), t);
+ activationFuture.tryFailure(t);
+ }
+ }
+
+ // Activation succeeded, complete the future with the sync stats
+ activationFuture.trySuccess(syncs);
+ }
+
+ @Override
+ public void onFailure(Throwable t)
+ {
+ logger.error("Failed to complete sync tasks. Cleaning up pending transfers... ", t);
+
+ lock.writeLock().lock();
+ try
+ {
+ Set<ShortMutationId> transferIds = new HashSet<>();
+ for (SyncTask task : job.getSyncTasks())
+ {
+ ShortMutationId transferId = task.getTransferId();
+ Preconditions.checkNotNull(transferId);
+ transferIds.add(transferId);
+
+ TimeUUID planId = task.getPlanId();
+ if (planId == null)
+ continue;
+
+ CoordinatedTransfer transfer = coordinating.get(transferId);
+ Pair<InetAddressAndPort, InetAddressAndPort> pair = Pair.create(task.nodePair().coordinator, task.nodePair().peer);
+ transfer.streamResults.put(pair, CoordinatedTransfer.SingleTransferResult.Init().streamFailed(planId));
+ transfer.streamResults.put(pair.reverse(), CoordinatedTransfer.SingleTransferResult.Init().streamFailed(planId));
+ }
+
+ for (ShortMutationId transferId : transferIds)
+ {
+ CoordinatedTransfer transfer = coordinating.get(transferId);
+ try
+ {
+ transfer.notifyFailure();
+ }
+ catch (Throwable t0)
+ {
+ logger.error("{} Failed to notify peers of repair failure", transfer.logPrefix(), t0);
+ }
+ }
+
+ scheduleCleanup();
+ activationFuture.tryFailure(t);
+ }
+ finally
+ {
+ lock.writeLock().unlock();
+ }
+ }
+ }, executor);
+
+ return activationFuture;
+ }
+
Purger purger = new Purger();
static class Purger
@@ -126,7 +259,7 @@ public class LocalTransfers
boolean failedBeforeActivation = false;
boolean noneActivated = true;
boolean allComplete = true;
- for (CoordinatedTransfer.SingleTransferResult result : transfer.streamResults.values())
+ for (TrackedImportTransfer.SingleTransferResult result : transfer.streamResults.values())
{
switch (result.state)
{
@@ -230,12 +363,18 @@ public class LocalTransfers
if (transfer.id() != null)
coordinating.remove(transfer.id());
-
- CoordinatedTransfer.SingleTransferResult localPending = transfer.streamResults.get(FBUtilities.getBroadcastAddressAndPort());
- PendingLocalTransfer localTransfer;
- TimeUUID planId;
- if (localPending != null && (planId = localPending.planId()) != null && (localTransfer = local.get(planId)) != null)
- purge(localTransfer);
+
+ for (Map.Entry<Pair<InetAddressAndPort, InetAddressAndPort>, CoordinatedTransfer.SingleTransferResult> result : transfer.streamResults.entrySet())
+ {
+ if (result.getKey().right.equals(FBUtilities.getBroadcastAddressAndPort()))
+ {
+ CoordinatedTransfer.SingleTransferResult localPending = transfer.streamResults.get(result.getKey());
+ PendingLocalTransfer localTransfer;
+ TimeUUID planId;
+ if (localPending != null && (planId = localPending.planId()) != null && (localTransfer = local.get(planId)) != null)
+ purge(localTransfer);
+ }
+ }
}
finally
{
@@ -270,7 +409,8 @@ public class LocalTransfers
}
}
- @Nullable CoordinatedTransfer getActivatedTransfer(ShortMutationId transferId)
+ @Nullable
+ CoordinatedTransfer getActivatedTransfer(ShortMutationId transferId)
{
lock.readLock().lock();
try
@@ -284,7 +424,7 @@ public class LocalTransfers
}
public static IVerbHandler<TransferFailed> verbHandler = message -> {
- LocalTransfers.instance().purge(message.payload);
+ TransferTrackingService.instance().purge(message.payload);
MessagingService.instance().respond(NoPayload.noPayload, message);
};
}
diff --git a/src/java/org/apache/cassandra/replication/UnreconciledMutations.java b/src/java/org/apache/cassandra/replication/UnreconciledMutations.java
index 22c1550230..61b5e1746a 100644
--- a/src/java/org/apache/cassandra/replication/UnreconciledMutations.java
+++ b/src/java/org/apache/cassandra/replication/UnreconciledMutations.java
@@ -34,8 +34,8 @@ import org.agrona.collections.Int2ObjectHashMap;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.PartitionPosition;
import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Bounds;
import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.schema.TableId;
/**
@@ -151,9 +151,9 @@ public class UnreconciledMutations
statesSet.remove(state);
}
- public void activatedTransfer(ShortMutationId id, Collection<SSTableReader> sstables)
+ public void activatedTransfer(ShortMutationId id, Bounds<Token> bounds)
{
- transfers.add(id, sstables);
+ transfers.add(id, bounds);
}
public UnreconciledMutations copy()
@@ -274,10 +274,10 @@ public class UnreconciledMutations
result.addDirectly(mutation);
continue;
}
- CoordinatedTransfer transfer = LocalTransfers.instance().getActivatedTransfer(id);
+ CoordinatedTransfer transfer = TransferTrackingService.instance().getActivatedTransfer(id);
if (transfer != null)
{
- result.transfers.add(transfer.id(), transfer.sstables);
+ result.transfers.add(transfer.id(), transfer.bounds());
continue;
}
diff --git a/src/java/org/apache/cassandra/streaming/SessionSummary.java b/src/java/org/apache/cassandra/streaming/SessionSummary.java
index 8bb1a1eb81..6887947c3c 100644
--- a/src/java/org/apache/cassandra/streaming/SessionSummary.java
+++ b/src/java/org/apache/cassandra/streaming/SessionSummary.java
@@ -57,6 +57,7 @@ public class SessionSummary
this.sendingSummaries = sendingSummaries;
}
+ @Override
public boolean equals(Object o)
{
if (this == o) return true;
@@ -70,6 +71,7 @@ public class SessionSummary
return sendingSummaries.equals(summary.sendingSummaries);
}
+ @Override
public int hashCode()
{
int result = coordinator.hashCode();
@@ -79,6 +81,17 @@ public class SessionSummary
return result;
}
+ @Override
+ public String toString()
+ {
+ return "SessionSummary{" +
+ "coordinator=" + coordinator +
+ ", peer=" + peer +
+ ", receivingSummaries=" + receivingSummaries +
+ ", sendingSummaries=" + sendingSummaries +
+ '}';
+ }
+
public static IPartitionerDependentSerializer<SessionSummary> serializer = new IPartitionerDependentSerializer<SessionSummary>()
{
public void serialize(SessionSummary summary, DataOutputPlus out, int version) throws IOException
diff --git a/src/java/org/apache/cassandra/streaming/StreamOperation.java b/src/java/org/apache/cassandra/streaming/StreamOperation.java
index 45dd7e9a2c..5e9f8316aa 100644
--- a/src/java/org/apache/cassandra/streaming/StreamOperation.java
+++ b/src/java/org/apache/cassandra/streaming/StreamOperation.java
@@ -27,7 +27,7 @@ public enum StreamOperation
REBUILD("Rebuild", false, true, false),
BULK_LOAD("Bulk Load", true, false, false),
REPAIR("Repair", true, false, true),
- TRACKED_TRANSFER("Tracked Transfer", false, false, false);
+ IMPORT("Import", false, false, false);
private final String description;
private final boolean requiresViewBuild;
@@ -47,6 +47,17 @@ public enum StreamOperation
this.requiresBarrierTransaction = requiresBarrierTransaction;
}
+ /**
+ * For tracked keyspaces, streams that are received while serving reads must be tracked in the log, to ensure that
+ * read reconciliation can represent the new data.
+ *
+ * @see org.apache.cassandra.replication.CoordinatedTransfer
+ */
+ public boolean isTrackable()
+ {
+ return this == REPAIR || this == IMPORT;
+ }
+
public static StreamOperation fromString(String text)
{
for (StreamOperation b : StreamOperation.values())
diff --git a/src/java/org/apache/cassandra/streaming/StreamPlan.java b/src/java/org/apache/cassandra/streaming/StreamPlan.java
index 4d24992626..8fa5a9a231 100644
--- a/src/java/org/apache/cassandra/streaming/StreamPlan.java
+++ b/src/java/org/apache/cassandra/streaming/StreamPlan.java
@@ -116,10 +116,8 @@ public class StreamPlan
// TODO: add flag for fully reconciled data only if this is for a tracked keyspace
session.addStreamRequest(keyspace, fullRanges, transientRanges, Arrays.asList(columnFamilies));
- // Automatically include mutation logs for tracked keyspaces
- if (isTrackedReplicationEnabled(keyspace)) {
+ if (includeMutationLogs(keyspace, session))
session.addMutationLogRequest(keyspace, fullRanges, transientRanges);
- }
return this;
}
@@ -136,9 +134,7 @@ public class StreamPlan
public StreamPlan transferRanges(InetAddressAndPort to, String keyspace, RangesAtEndpoint replicas, String... columnFamilies)
{
StreamSession session = coordinator.getOrCreateOutboundSession(to);
-
- // Automatically include mutation logs for tracked keyspaces
- ReconciledKeyspaceOffsets reconciledKeyspaceOffsets = isTrackedReplicationEnabled(keyspace)
+ ReconciledKeyspaceOffsets reconciledKeyspaceOffsets = includeMutationLogs(keyspace, session)
? session.addMutationLogTransfer(keyspace, replicas)
: null;
@@ -147,6 +143,11 @@ public class StreamPlan
return this;
}
+ private boolean includeMutationLogs(String keyspace, StreamSession session)
+ {
+ return isTrackedReplicationEnabled(keyspace) && session.getStreamOperation() != StreamOperation.REPAIR;
+ }
+
/**
* Add transfer task to send given streams
*
diff --git a/src/java/org/apache/cassandra/streaming/StreamSession.java b/src/java/org/apache/cassandra/streaming/StreamSession.java
index e93dec699f..4489f27bea 100644
--- a/src/java/org/apache/cassandra/streaming/StreamSession.java
+++ b/src/java/org/apache/cassandra/streaming/StreamSession.java
@@ -1020,6 +1020,7 @@ public class StreamSession
manifest.keyspaceRanges.forEach((keyspace, ranges) ->
logTransfer.addKeyspaceRanges(keyspace, ranges));
}
+
/**
* In the case where we have an error checking disk space we allow the Operation to continue.
* In the case where we do _not_ have available space, this method raises a RTE.
diff --git a/src/java/org/apache/cassandra/tcm/ownership/ReplicaGroups.java b/src/java/org/apache/cassandra/tcm/ownership/ReplicaGroups.java
index dc43ae2ee1..fa7f4a9bd8 100644
--- a/src/java/org/apache/cassandra/tcm/ownership/ReplicaGroups.java
+++ b/src/java/org/apache/cassandra/tcm/ownership/ReplicaGroups.java
@@ -130,11 +130,11 @@ public class ReplicaGroups
}
/**
- * This method is intended to be used on read/write path, not forRange.
+ * This method is intended to be used on read/write path, not forRange. Returns a {@link VersionedEndpoints.ForRange}
+ * with the full ownership range, which necessarily contains the input range but may be wider.
*/
public VersionedEndpoints.ForRange matchRange(Range<Token> range)
{
- EndpointsForRange.Builder builder = new EndpointsForRange.Builder(range);
Epoch lastModified = Epoch.EMPTY;
// find a range containing the *right* token for the given range - Range is start exclusive so if we looked for the
// left one we could get the wrong range
@@ -142,10 +142,9 @@ public class ReplicaGroups
if (pos >= 0 && pos < ranges.size() && ranges.get(pos).contains(range))
{
VersionedEndpoints.ForRange eps = endpoints.get(pos);
- lastModified = eps.lastModified();
- builder.addAll(eps.get(), ReplicaCollection.Builder.Conflict.ALL);
+ return VersionedEndpoints.forRange(eps.lastModified(), eps.get());
}
- return VersionedEndpoints.forRange(lastModified, builder.build());
+ return VersionedEndpoints.forRange(lastModified, EndpointsForRange.empty(range));
}
public VersionedEndpoints.ForRange forRange(Token token)
diff --git a/src/java/org/apache/cassandra/utils/Pair.java b/src/java/org/apache/cassandra/utils/Pair.java
index cb0952947f..c08b1a8b3d 100644
--- a/src/java/org/apache/cassandra/utils/Pair.java
+++ b/src/java/org/apache/cassandra/utils/Pair.java
@@ -69,4 +69,9 @@ public class Pair<T1, T2>
{
return new Pair<X, Y>(x, y);
}
+
+ public Pair<T2, T1> reverse()
+ {
+ return new Pair<>(right, left);
+ }
}
diff --git a/test/data/serialization/5.1/service.SyncComplete.bin b/test/data/serialization/5.1/service.SyncComplete.bin
index 5e27e345fd..006f0a1cc7 100644
Binary files a/test/data/serialization/5.1/service.SyncComplete.bin and b/test/data/serialization/5.1/service.SyncComplete.bin differ
diff --git a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java
index dfa676b074..6f831c51e6 100644
--- a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java
+++ b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java
@@ -1036,11 +1036,13 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance
AccordService.instance().shutdownAndWait(1l, MINUTES);
});
+ // The periodic tasks in MTS might hit the CommitLog, so make sure those shut down first...
+ error = parallelRun(error, executor, MutationTrackingService.instance::shutdownBlocking);
+ error = parallelRun(error, executor, MutationJournal.instance::shutdownBlocking);
+
// CommitLog must shut down after Stage, or threads from the latter may attempt to use the former.
// (ex. A Mutation stage thread may attempt to add a mutation to the CommitLog.)
error = parallelRun(error, executor, CommitLog.instance::shutdownBlocking);
- error = parallelRun(error, executor, MutationJournal.instance::shutdownBlocking);
- error = parallelRun(error, executor, MutationTrackingService.instance::shutdownBlocking);
error = parallelRun(error, executor,
() -> shutdownAndWait(Collections.singletonList(JMXBroadcastExecutor.executor))
);
diff --git a/test/distributed/org/apache/cassandra/distributed/test/tracking/BulkTransfersTest.java b/test/distributed/org/apache/cassandra/distributed/test/tracking/BulkTransfersTest.java
deleted file mode 100644
index 2c85dd81b9..0000000000
--- a/test/distributed/org/apache/cassandra/distributed/test/tracking/BulkTransfersTest.java
+++ /dev/null
@@ -1,949 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.distributed.test.tracking;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.file.Files;
-import java.time.Duration;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-import java.util.function.Consumer;
-import java.util.function.Predicate;
-import java.util.stream.Collectors;
-
-import javax.annotation.Nullable;
-
-import org.junit.Ignore;
-import org.junit.Test;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import net.bytebuddy.ByteBuddy;
-import net.bytebuddy.dynamic.loading.ClassLoadingStrategy;
-import net.bytebuddy.implementation.MethodDelegation;
-import net.bytebuddy.implementation.bind.annotation.SuperCall;
-import net.bytebuddy.implementation.bind.annotation.This;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.compaction.CompactionManager;
-import org.apache.cassandra.db.marshal.Int32Type;
-import org.apache.cassandra.db.streaming.CassandraStreamReceiver;
-import org.apache.cassandra.dht.Murmur3Partitioner;
-import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.distributed.Cluster;
-import org.apache.cassandra.distributed.api.ConsistencyLevel;
-import org.apache.cassandra.distributed.api.Feature;
-import org.apache.cassandra.distributed.api.ICoordinator;
-import org.apache.cassandra.distributed.api.IInstanceInitializer;
-import org.apache.cassandra.distributed.api.IInvokableInstance;
-import org.apache.cassandra.distributed.api.IIsolatedExecutor;
-import org.apache.cassandra.distributed.shared.ClusterUtils;
-import org.apache.cassandra.distributed.shared.Uninterruptibles;
-import org.apache.cassandra.distributed.test.TestBaseImpl;
-import org.apache.cassandra.distributed.test.sai.SAIUtil;
-import org.apache.cassandra.exceptions.ReadTimeoutException;
-import org.apache.cassandra.io.IVersionedSerializer;
-import org.apache.cassandra.io.sstable.CQLSSTableWriter;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.util.File;
-import org.apache.cassandra.net.Message;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.replication.ActivatedTransfers;
-import org.apache.cassandra.replication.ImmutableCoordinatorLogOffsets;
-import org.apache.cassandra.replication.MutationSummary;
-import org.apache.cassandra.replication.MutationTrackingService;
-import org.apache.cassandra.replication.ShortMutationId;
-import org.apache.cassandra.replication.TransferActivation;
-import org.apache.cassandra.replication.UnknownShardException;
-import org.apache.cassandra.schema.TableId;
-import org.apache.cassandra.tcm.ClusterMetadata;
-import org.apache.cassandra.tcm.ownership.DataPlacement;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.assertj.core.api.Assertions;
-
-import static net.bytebuddy.matcher.ElementMatchers.named;
-import static net.bytebuddy.matcher.ElementMatchers.takesArguments;
-import static net.bytebuddy.matcher.ElementMatchers.takesNoArguments;
-import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows;
-import static org.apache.cassandra.distributed.shared.AssertUtils.row;
-import static org.apache.cassandra.replication.TransferActivation.Phase.COMMIT;
-import static org.apache.cassandra.replication.TransferActivation.Phase.PREPARE;
-
-/**
- * For now, tracked import with a replica down is not supported. The intention is to support this scenario by allowing
- * users to provide a {@link ConsistencyLevel} for tracked import operations, where the import will complete if
- * sufficient replicas acknowledge the transfer and activate it.
- */
-public class BulkTransfersTest extends TestBaseImpl
-{
- private static final Logger logger = LoggerFactory.getLogger(BulkTransfersTest.class);
-
- private static final String TABLE = "tbl";
- private static final String KEYSPACE_TABLE = String.format("%s.%s", KEYSPACE, TABLE);
- private static final String TABLE_SCHEMA_CQL = String.format(withKeyspace("CREATE TABLE %s." + TABLE + " (k int primary key, v int);"));
-
- private static final int IMPORT_PK = 1;
- private static final Token IMPORT_TOKEN = Murmur3Partitioner.instance.getToken(Int32Type.instance.decompose(IMPORT_PK));
- private static final int NODES = 3;
-
- private static final IIsolatedExecutor.SerializableConsumer<SSTableReader> TRANSFERS_EXIST = sstable -> {
- Assertions.assertThat(sstable.getCoordinatorLogOffsets().transfers())
- .isNotEmpty();
- Assertions.assertThat(sstable.isRepaired()).isFalse();
- };
- private static final IIsolatedExecutor.SerializableConsumer<SSTableReader> TRANSFERS_EMPTY = sstable -> {
- Assertions.assertThat(sstable.getCoordinatorLogOffsets().transfers())
- .isEmpty();
- Assertions.assertThat(sstable.isRepaired()).isTrue();
- };
- private static final IIsolatedExecutor.SerializableConsumer<SSTableReader> NOOP = sstable -> {};
-
- @Test
- public void importHappyPath() throws Throwable
- {
- try (Cluster cluster = cluster())
- {
- createSchema(cluster);
- doImport(cluster);
-
- // All pending/ dirs should be empty, should have no SSTables left if all the transfers completed
- assertPendingDirs(cluster, (File pendingUuidDir) -> {
- Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isEmpty();
- });
-
- // Verify transfer IDs exist before compaction, then compact, then verify they're removed
- assertCompaction(cluster, cluster, TRANSFERS_EXIST, TRANSFERS_EMPTY);
-
- // Run after compaction, to enforce offset persistence + broadcast
- assertSummary(cluster, summary -> {
- Assertions.assertThat(summary).satisfies(s -> {
- assert s.reconciledIds() == 1;
- assert s.unreconciledIds() == 0;
- });
- });
-
- assertLocalSelect(cluster, rows -> assertRows(rows, row(1, 1)));
- }
- }
-
- @Test
- public void importIndexAlreadyPresent() throws Throwable
- {
- try (Cluster cluster = cluster())
- {
- createSchema(cluster);
-
- String indexName = "v_idx";
- String indexCql = withKeyspace("CREATE INDEX " + indexName + " ON %s." + TABLE + " (v) USING 'sai'");
- cluster.schemaChange(indexCql);
-
- // This will add an SSTable that already has an SAI index, that needs to be distributed alongside the SSTable on transfer
- IInvokableInstance importer = cluster.get(1);
- long mark = importer.logs().mark();
- doImport(cluster, importer, indexCql);
- List<String> logs = importer.logs().grep(mark, "Submitting incremental index build of " + indexName).getResult();
- Assertions.assertThat(logs).isEmpty();
-
- // Index should exist and be queryable on all replicas after import
- SAIUtil.assertIndexQueryable(cluster, KEYSPACE, indexName);
-
- // Validate queries using the index
- cluster.forEach(instance -> {
- Object[][] rows = instance.executeInternal(withKeyspace("SELECT * FROM %s." + TABLE + " WHERE v = 1"));
- assertRows(rows, row(1, 1));
- });
- }
- }
-
- @Test
- public void importBuildsIndex() throws Throwable
- {
- try (Cluster cluster = cluster())
- {
- createSchema(cluster);
-
- String indexName = "v_idx";
- cluster.schemaChange(withKeyspace("CREATE INDEX " + indexName + " ON %s." + TABLE + " (v) USING 'sai'"));
-
- // This will add an SSTable that's missing an SAI index, and the index will be built during the import on
- // the coordinator
- IInvokableInstance importer = cluster.get(1);
- long mark = importer.logs().mark();
- doImport(cluster, importer);
- List<String> logs = importer.logs().watchFor(mark, Duration.ofMinutes(1), "Submitting incremental index build of " + indexName).getResult();
- Assertions.assertThat(logs).isNotEmpty();
-
- // Index should exist and be queryable on all replicas after import
- SAIUtil.assertIndexQueryable(cluster, KEYSPACE, indexName);
-
- // Validate queries using the index
- cluster.forEach(instance -> {
- Object[][] rows = instance.executeInternal(withKeyspace("SELECT * FROM %s." + TABLE + " WHERE v = 1"));
- assertRows(rows, row(1, 1));
- });
- }
- }
-
- @Test
- @Ignore
- public void importReplicaDown() throws Throwable
- {
- try (Cluster cluster = cluster())
- {
- createSchema(cluster);
-
- Iterable<IInvokableInstance> down = Collections.singleton(cluster.get(3));
- Iterable<IInvokableInstance> up = cluster.stream().filter(instance -> instance != down).collect(Collectors.toList());
- for (IInvokableInstance instance : down)
- instance.shutdown().get();
-
- doImport(cluster);
-
- cluster.get(3).startup();
-
- // Transfers did not complete, files should still exist on up replicas
- assertPendingDirs(up, (File pendingUuidDir) -> {
- Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isNotEmpty();
- });
- assertPendingDirs(down, (File pendingUuidDir) -> {
- Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isEmpty();
- });
-
- // Transfers did not complete, transfer IDs should not be removed
- assertCompaction(cluster, cluster, TRANSFERS_EXIST, TRANSFERS_EXIST);
-
- assertLocalSelect(up, rows -> assertRows(rows, row(1, 1)));
- }
- }
-
- @Test
- public void importMissedActivationPrepare() throws Throwable
- {
- importMissedActivation(PREPARE);
- }
-
- @Test
- public void importMissedActivationCommit() throws Throwable
- {
- importMissedActivation(COMMIT);
- }
-
- public void importMissedActivation(TransferActivation.Phase phase) throws Throwable
- {
- int MISSED_ACTIVATION = 2;
- try (Cluster cluster = cluster(ByteBuddyInjections.SkipActivation.install(MISSED_ACTIVATION)))
- {
- ByteBuddyInjections.SkipActivation.setup(cluster, phase);
- createSchema(cluster);
-
- Set<IInvokableInstance> missed = Collections.singleton(cluster.get(MISSED_ACTIVATION));
- Iterable<IInvokableInstance> received = cluster.stream().filter(instance -> !missed.contains(instance)).collect(Collectors.toList());
-
- Assertions.assertThatThrownBy(() -> doImport(cluster))
- .hasMessageContaining("Failed adding SSTables")
- .cause()
- .hasMessageContaining("Failed streaming on 1 instance(s):")
- .cause()
- .hasMessageMatching("Tracked import failed during " + phase + " on " + cluster.get(MISSED_ACTIVATION).broadcastAddress() + " due to TIMEOUT")
- .hasNoCause();
-
- assertSummary(received, summary -> {
- Assertions.assertThat(summary).satisfies(s -> {
- assert s.reconciledIds() == 0;
- assert s.unreconciledIds() == (phase == COMMIT ? 1 : 0);
- });
- });
- assertSummary(missed, summary -> {
- Assertions.assertThat(summary).satisfies(s -> {
- assert s.reconciledIds() == 0;
- assert s.unreconciledIds() == 0;
- });
- });
-
- switch (phase)
- {
- case PREPARE:
- // Activation did not start, files should be cleaned up
- Uninterruptibles.sleepUninterruptibly(5, TimeUnit.SECONDS);
- assertPendingDirs(cluster, (File pendingUuidDir) -> {
- Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isEmpty();
- });
- break;
- case COMMIT:
- // Activation did not complete, files should still exist on all replicas
- assertPendingDirs(cluster, (File pendingUuidDir) -> {
- Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isNotEmpty();
- });
- break;
- }
-
- // If the activation is not everywhere, it shouldn't be purged on compaction
- assertCompaction(cluster, received, TRANSFERS_EXIST, TRANSFERS_EXIST);
-
- if (phase == PREPARE)
- return;
-
- // Permit activation of missed commits during read reconciliation
- ByteBuddyInjections.SkipActivation.setup(cluster, null);
-
- // Use coordinated query rather to confirm read reconciliation triggers activation
- IInvokableInstance coordinator = cluster.get(3); // not initial transfer coordinator, but received activation
- assertCoordinatedRead(coordinator, rows -> {
- assertRows(rows, row(1, 1));
- });
-
- // Confirm others receive activation
- assertLocalSelect(missed, rows -> {
- assertRows(rows, row(1, 1));
- });
-
- assertCompaction(cluster, cluster, TRANSFERS_EXIST, TRANSFERS_EMPTY);
-
- // Activation completed, files should be removed
- assertPendingDirs(cluster, (File pendingUuidDir) -> {
- Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isEmpty();
- });
- }
- }
-
- /**
- * If a replica is missing an activation, executes a data read, then discovers the missing transfer during read
- * reconciliation, it needs to augment the client response with the new transfer.
- */
- @Test
- public void importMissingOnDataReplicaDuringAugment() throws Throwable
- {
- TransferActivation.Phase phase = COMMIT;
- int MISSED_ACTIVATION = 2;
- try (Cluster cluster = cluster(ByteBuddyInjections.SkipActivation.install(MISSED_ACTIVATION)))
- {
- ByteBuddyInjections.SkipActivation.setup(cluster, phase);
- createSchema(cluster);
-
- IInvokableInstance missed = cluster.get(MISSED_ACTIVATION);
-
- Assertions.assertThatThrownBy(() -> doImport(cluster))
- .hasMessageContaining("Failed adding SSTables")
- .cause()
- .hasMessageContaining("Failed streaming on 1 instance(s):")
- .cause()
- .hasMessageMatching("Tracked import failed during " + phase + " on " + missed.broadcastAddress() + " due to TIMEOUT")
- .hasNoCause();
-
- assertSummary(Collections.singleton(missed), summary -> {
- Assertions.assertThat(summary).satisfies(s -> {
- assert s.reconciledIds() == 0;
- assert s.unreconciledIds() == 0;
- });
- });
-
- // Permit activation of missed commits during read reconciliation
- ByteBuddyInjections.SkipActivation.setup(cluster, null);
-
- // First read will fail due to failure to augment with transfer
- long mark = missed.logs().mark();
- Assertions.assertThatThrownBy(() -> {
- assertCoordinatedRead(missed, rows -> {
- assertRows(rows, row(1, 1));
- });
- }).isInstanceOf(missed.callOnInstance(() -> ReadTimeoutException.class)); // use instance classloader
-
- List<String> logs = missed.logs().grep(mark, "Missing mutation ShortMutationId").getResult();
- Assertions.assertThat(logs).isNotEmpty();
-
- // Retry succeeds
- assertCoordinatedRead(missed, rows -> {
- assertRows(rows, row(1, 1));
- });
- }
- }
-
- /*
- * When an import fails, bounce must not move the pending SSTables into the live set.
- */
- @Test
- public void importBounceAfterPending() throws Throwable
- {
- IInstanceInitializer initializer = ByteBuddyInjections.SkipActivation.install(1, 2, 3);
- try (Cluster cluster = cluster(initializer))
- {
- ByteBuddyInjections.SkipActivation.setup(cluster, COMMIT);
- createSchema(cluster);
-
- Assertions.assertThatThrownBy(() -> doImport(cluster))
- .hasMessageContaining("Failed adding SSTables")
- .cause()
- .hasMessageContaining("Tracked import failed during COMMIT");
-
- Runnable assertEmpty = () -> {
- // Activation did not complete, files should still exist on all replicas
- assertPendingDirs(cluster, (File pendingUuidDir) -> {
- Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isNotEmpty();
- });
-
- // No one has activated, so should not be present in any summary
- assertSummary(cluster, summary -> {
- Assertions.assertThat(summary).satisfies(s -> {
- assert s.reconciledIds() == 0;
- assert s.unreconciledIds() == 0;
- });
- });
-
- assertLocalSelect(cluster, rows -> assertRows(rows, EMPTY_ROWS));
- };
-
- assertEmpty.run();
-
- bounce(cluster);
-
- assertEmpty.run();
- }
- }
-
- @Test
- public void importOutOfRange() throws Throwable
- {
- try (Cluster cluster = cluster())
- {
- createSchema(cluster, 1);
-
- Set<IInvokableInstance> inRange = new HashSet<>();
- Set<IInvokableInstance> outOfRange = new HashSet<>();
- cluster.forEach(instance -> {
- boolean importReplica = instance.callOnInstance(() -> {
- ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE);
- DataPlacement placement = ClusterMetadata.current().placements.get(cfs.keyspace.getMetadata().params.replication);
- return placement.writes.forToken(IMPORT_TOKEN).get().containsSelf();
- });
- (importReplica ? inRange : outOfRange).add(instance);
- });
- logger.info("inRange: {}, outOfRange: {}", inRange, outOfRange);
-
- Assertions.assertThat(inRange).hasSize(1);
- IInvokableInstance onlyInRange = inRange.iterator().next();
-
- // Reject import out of range
- for (IInvokableInstance instance : outOfRange)
- {
- long mark = instance.logs().mark();
- Consumer<List<String>> onResult = failedDirs -> Assertions.assertThat(failedDirs).hasSize(1);
- doImport(cluster, instance, onResult, null);
- instance.logs().grep(mark, "java.lang.RuntimeException: Key DecoratedKey(-4069959284402364209, 00000001) is not contained in the given ranges");
- }
-
- doImport(cluster, onlyInRange);
-
- assertSummary(Collections.singleton(onlyInRange), summary -> {
- Assertions.assertThat(summary).satisfies(s -> {
- assert s.reconciledIds() == 1;
- assert s.unreconciledIds() == 0;
- });
- });
-
- for (IInvokableInstance instance : outOfRange)
- {
- // Out of range shouldn't have any transfers
- assertCompaction(cluster, Collections.singleton(instance), TRANSFERS_EMPTY, TRANSFERS_EMPTY);
-
- // Run after compaction, to enforce offset persistence + broadcast
- assertSummary(Collections.singleton(instance), summary -> {
- Assertions.assertThat(summary).isNull();
- });
- }
- }
- }
-
- /*
- * Ensure that activation IDs attached to SSTables aren't spread across Token boundaries by compaction.
- *
- * For example:
- * IMPORT_TOKEN is owned by replicas (A, B)
- * OUTSIDE_IMPORT_TOKEN is owned by replicas (B, C)
- * Execute import so (A, B) have IMPORT_TOKEN
- * Execute plain write so (B, C) have OUTSIDE_IMPORT_TOKEN
- * Do a major compaction on B so IMPORT_TOKEN and OUTSIDE_IMPORT_TOKEN are compacted together into the same SSTable
- * Execute a data read for OUTSIDE_IMPORT_TOKEN against B, ensure it doesn't contain any activation IDs
- */
- @Test
- public void importActivationMergedByCompaction() throws Throwable
- {
- IInstanceInitializer initializer = (cl, tg, instance, gen) -> {
- ByteBuddyInjections.SkipPurgeTransfers.install().initialise(cl, tg, instance, gen);
- };
- try (Cluster cluster = cluster(initializer))
- {
- createSchema(cluster, 2);
-
- Set<IInvokableInstance> inImportRange = new HashSet<>();
- cluster.forEach(instance -> {
- logger.debug("Instance {} ring is {}", ClusterUtils.instanceId(instance), ClusterUtils.ring(instance));
- boolean isInRange = instance.callOnInstance(() -> {
- ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE);
- DataPlacement placement = ClusterMetadata.current().placements.get(cfs.keyspace.getMetadata().params.replication);
- return placement.writes.forToken(IMPORT_TOKEN).get().containsSelf();
- });
- if (isInRange)
- inImportRange.add(instance);
- });
- Assertions.assertThat(inImportRange).hasSize(2);
-
- // Find a partition key that's not owned by the same replicas as the import
- Murmur3Partitioner.LongToken NON_IMPORT_TOKEN = new Murmur3Partitioner.LongToken(IMPORT_TOKEN.getLongValue() * 3);
- int NON_IMPORT_PK = Int32Type.instance.compose(Murmur3Partitioner.LongToken.keyForToken(NON_IMPORT_TOKEN));
-
- Set<IInvokableInstance> inNonImportRange = new HashSet<>();
- cluster.forEach(instance -> {
- boolean isInRange = instance.callOnInstance(() -> {
- ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE);
- DataPlacement placement = ClusterMetadata.current().placements.get(cfs.keyspace.getMetadata().params.replication);
- return placement.writes.forToken(NON_IMPORT_TOKEN).get().containsSelf();
- });
- if (isInRange)
- inNonImportRange.add(instance);
- });
- Assertions.assertThat(inNonImportRange).hasSize(2);
- Assertions.assertThat(inNonImportRange).isNotEqualTo(inImportRange);
-
- // Import: (A, B)
- // Plain: (B, C)
- IInvokableInstance A = null;
- IInvokableInstance B = null;
- IInvokableInstance C = null;
- for (IInvokableInstance instance : cluster)
- {
- boolean isImport = inImportRange.contains(instance);
- boolean isNonImport = inNonImportRange.contains(instance);
- if (isImport && isNonImport)
- B = instance;
- else if (isImport)
- A = instance;
- else if (isNonImport)
- C = instance;
- }
- Assertions.assertThat(A).isNotNull();
- Assertions.assertThat(B).isNotNull();
- Assertions.assertThat(C).isNotNull();
-
- doImport(cluster, A);
- assertLocalSelect(List.of(A, B), (IIsolatedExecutor.SerializableConsumer<Object[][]>) rows -> {
- assertRows(rows, row(IMPORT_PK, IMPORT_PK));
- });
-
- ShortMutationId importTransferId = callSerialized(A, () -> ShortMutationId.serializer, () -> {
- ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE);
- for (SSTableReader sstable : cfs.getLiveSSTables())
- {
- ActivatedTransfers transfers = sstable.getCoordinatorLogOffsets().transfers();
- if (!transfers.isEmpty())
- return transfers.iterator().next();
- }
- return null;
- });
- Assertions.assertThat(importTransferId).isNotNull();
- C.coordinator().execute(withKeyspace("INSERT INTO %s." + TABLE + "(k, v) VALUES (?, ?)"), ConsistencyLevel.ALL, NON_IMPORT_PK, NON_IMPORT_PK);
- assertCompaction(cluster, Collections.singleton(B), NOOP, NOOP);
-
- // Reading from B for a range that doesn't include the import shouldn't include any transfer IDs, even though they've been compacted together
- long mark = B.logs().mark();
- Object[][] rows = B.coordinator().execute(withKeyspace("SELECT * FROM %s." + TABLE + " WHERE k = ?"), ConsistencyLevel.ALL, NON_IMPORT_PK);
- assertRows(rows, row(NON_IMPORT_PK, NON_IMPORT_PK));
- Assertions.assertThat(B.logs().grep(mark, "Found overlapping activation ID ").getResult()).isEmpty();
-
- // But if the read range does include a transfer ID, it should have been added
- mark = B.logs().mark();
- rows = B.coordinator().execute(withKeyspace("SELECT * FROM %s." + TABLE + " WHERE k = ?"), ConsistencyLevel.ALL, IMPORT_PK);
- assertRows(rows, row(IMPORT_PK, IMPORT_PK));
- Assertions.assertThat(B.logs().grep(mark, "Found overlapping activation ID ").getResult()).isNotEmpty();
- }
- }
-
- @Test
- public void importFailedStreamCleanup() throws Throwable
- {
- int FAILED_STREAM = 3;
- try (Cluster cluster = cluster(ByteBuddyInjections.FailIncomingStream.install(FAILED_STREAM)))
- {
- createSchema(cluster);
-
- IInvokableInstance importer = cluster.get(1);
- IInvokableInstance missed = cluster.get(FAILED_STREAM);
-
- long mark = importer.logs().mark();
- Assertions.assertThatThrownBy(() -> doImport(cluster, importer))
- .isInstanceOf(RuntimeException.class)
- .cause()
- .isInstanceOf(RuntimeException.class)
- .cause()
- .hasMessageContaining("Remote peer " + missed.broadcastAddress() + " failed stream session");
- List<String> logs = importer.logs().watchFor(mark, "Remote peer " + missed.broadcastAddress().toString() + " failed stream session").getResult();
- Assertions.assertThat(logs).isNotEmpty();
-
- // Await cleanup of failed stream
- Uninterruptibles.sleepUninterruptibly(5, TimeUnit.SECONDS);
-
- assertPendingDirs(cluster, (File pendingUuidDir) -> {
- Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isEmpty();
- });
-
- assertLocalSelect(cluster, rows -> {
- assertRows(rows); // empty
- });
- }
- }
-
- public static class ByteBuddyInjections
- {
- // Only skips direct transfer activation, not activation as part of read reconciliation
- public static class SkipActivation
- {
- // null to not skip
- public static volatile TransferActivation.Phase phase;
-
- public static IInstanceInitializer install(int...nodes)
- {
- return (ClassLoader cl, ThreadGroup tg, int num, int generation) -> {
- for (int node : nodes)
- if (node == num)
- new ByteBuddy().rebase(TransferActivation.VerbHandler.class)
- .method(named("doVerb"))
- .intercept(MethodDelegation.to(ByteBuddyInjections.SkipActivation.class))
- .make()
- .load(cl, ClassLoadingStrategy.Default.INJECTION);
- };
- }
-
- // Need to set phase in each instance's classloader, otherwise assignment won't be visible to injected method body
- public static void setup(Cluster cluster, TransferActivation.Phase phase)
- {
- logger.debug("Setting up phase {}", phase);
- cluster.forEach(instance -> instance.runOnInstance(() -> ByteBuddyInjections.SkipActivation.phase = phase));
- }
-
- @SuppressWarnings("unused")
- public static void doVerb(Message<TransferActivation> msg, @SuperCall Callable<?> zuper)
- {
- if (phase != null && msg.payload.phase == SkipActivation.phase)
- {
- logger.info("Skipping activation for test: {}", msg.payload);
- return;
- }
-
- logger.info("Test running activation as usual: {}", msg.payload);
-
- try
- {
- zuper.call();
- }
- catch (Exception e)
- {
- throw new RuntimeException(e);
- }
- }
- }
-
- // ImmutableCoordinatorLogOffsets.Builder.purgeTransfers(Predicate)
- public static class SkipPurgeTransfers
- {
- public static IInstanceInitializer install()
- {
- return (ClassLoader cl, ThreadGroup tg, int num, int generation) -> {
- new ByteBuddy().rebase(ImmutableCoordinatorLogOffsets.Builder.class)
- .method(named("purgeTransfers").and(takesArguments(Predicate.class)))
- .intercept(MethodDelegation.to(SkipPurgeTransfers.class))
- .make()
- .load(cl, ClassLoadingStrategy.Default.INJECTION);
- };
- }
-
- @SuppressWarnings("unused")
- public static void purgeTransfers()
- {
- logger.debug("Skipping purgeTransfers for test");
- }
- }
-
- // CassandraStreamReceiver.finished
- public static class FailIncomingStream
- {
- @SuppressWarnings("unused")
- private static volatile boolean enabled = true;
-
- public static IInstanceInitializer install(int... nodes)
- {
- return (ClassLoader cl, ThreadGroup tg, int num, int generation) -> {
- for (int node : nodes)
- if (node == num)
- new ByteBuddy().rebase(CassandraStreamReceiver.class)
- .method(named("finished").and(takesNoArguments()))
- .intercept(MethodDelegation.to(FailIncomingStream.class))
- .make()
- .load(cl, ClassLoadingStrategy.Default.INJECTION);
- };
- }
-
- @SuppressWarnings("unused")
- public static void finished(@This CassandraStreamReceiver self)
- {
- throw new RuntimeException("Failing incoming stream for test");
- }
-
- public static void toggle(Cluster cluster, boolean enable)
- {
- enabled = enable;
- cluster.forEach(instance -> instance.runOnInstance(() -> FailIncomingStream.enabled = enable));
- }
- }
- }
-
- private static Cluster cluster() throws IOException
- {
- return cluster((cl, tg, instance, generation) -> {});
- }
-
- private static Cluster cluster(IInstanceInitializer initializer) throws IOException
- {
- return Cluster.build(NODES)
- .withConfig(cfg -> cfg.with(Feature.NETWORK)
- .with(Feature.GOSSIP)
- .set("mutation_tracking_enabled", "true")
- .set("write_request_timeout", "1000ms")
- .set("autocompaction_on_startup_enabled", false)
- .set("repair_request_timeout", "2s")
- .set("stream_transfer_task_timeout", "10s"))
- .withInstanceInitializer(initializer)
- .start();
- }
-
- private static void createSchema(Cluster cluster)
- {
- createSchema(cluster, NODES);
- }
-
- private static void createSchema(Cluster cluster, int rf)
- {
- cluster.schemaChange(String.format(withKeyspace("CREATE KEYSPACE %s WITH replication = " +
- "{'class': 'SimpleStrategy', 'replication_factor': " + rf + "} " +
- "AND replication_type='tracked';")));
- cluster.schemaChange(TABLE_SCHEMA_CQL);
- }
-
- private static void doImport(Cluster cluster) throws IOException
- {
- doImport(cluster, cluster.get(1));
- }
-
- private static void doImport(Cluster cluster, IInvokableInstance target) throws IOException
- {
- doImport(cluster, target, failedDirs -> Assertions.assertThat(failedDirs).isEmpty(), null);
- }
-
- private static void doImport(Cluster cluster, IInvokableInstance target, @Nullable String createIndexCql) throws IOException
- {
- doImport(cluster, target, failedDirs -> Assertions.assertThat(failedDirs).isEmpty(), createIndexCql);
- }
-
- private static void doImport(Cluster cluster, IInvokableInstance target, Consumer<List<String>> onFailedDirs, @Nullable String createIndexCql) throws IOException
- {
- String file = Files.createTempDirectory(MutationTrackingTest.class.getSimpleName()).toString();
-
- // Needs to run outside of instance executor because creates schema
- CQLSSTableWriter.Builder builder = CQLSSTableWriter.builder()
- .forTable(TABLE_SCHEMA_CQL)
- .inDirectory(file)
- .using("INSERT INTO " + KEYSPACE_TABLE + " (k, v) " + "VALUES (?, ?)");
-
- if (createIndexCql != null)
- {
- builder.withIndexes(createIndexCql).withBuildIndexes(true);
- }
-
- try (CQLSSTableWriter writer = builder.build())
- {
- writer.addRow(IMPORT_PK, 1);
- }
-
- assertLocalSelect(cluster, rows -> {
- assertRows(rows); // empty
- });
-
- List<String> failed = target.callOnInstance(() -> {
- ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE);
- Set<String> paths = Set.of(file);
- logger.info("Importing SSTables {}", paths);
- return cfs.importNewSSTables(paths, true, true, true, true, true, true, true);
- });
-
- // Sleep for a while to make sure import completes
- Uninterruptibles.sleepUninterruptibly(3, TimeUnit.SECONDS);
- onFailedDirs.accept(failed);
- }
-
- private static void assertCoordinatedRead(IInvokableInstance instance, IIsolatedExecutor.SerializableConsumer<Object[][]> onRows)
- {
- ICoordinator coordinator = instance.coordinator();
- String cql = "SELECT * FROM %s." + TABLE + " WHERE k = 1";
- Object[][] rows = coordinator.execute(withKeyspace(cql), ConsistencyLevel.ALL);
- onRows.accept(rows);
- }
-
- private static void assertPendingDirs(Iterable<IInvokableInstance> validate, IIsolatedExecutor.SerializableConsumer<File> forPendingUuidDir)
- {
- for (IInvokableInstance instance : validate)
- {
- instance.runOnInstance(() -> {
- Set<File> allPendingDirs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE).getDirectories().getPendingLocations();
- for (File pendingDir : allPendingDirs)
- {
- File[] pendingUuidDirs = pendingDir.listUnchecked(File::isDirectory);
- for (File pendingUuidDir : pendingUuidDirs)
- {
- forPendingUuidDir.accept(pendingUuidDir);
- }
- }
- });
- }
- }
-
- private static void assertSummary(Iterable<IInvokableInstance> validate, IIsolatedExecutor.SerializableConsumer<MutationSummary> onSummary)
- {
- for (IInvokableInstance instance : validate)
- {
- instance.runOnInstance(() -> {
- DecoratedKey key = DatabaseDescriptor.getPartitioner().decorateKey(ByteBufferUtil.bytes(1));
- TableId tableId = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE).metadata().id;
- MutationSummary summary;
- try
- {
- summary = MutationTrackingService.instance.createSummaryForKey(key, tableId, false);
- }
- catch (UnknownShardException e)
- {
- summary = null;
- }
- logger.debug("Validating summary {}", summary);
- onSummary.accept(summary);
- });
- }
- }
-
- private static void assertCompaction(Cluster cluster, Iterable<IInvokableInstance> validate,
- IIsolatedExecutor.SerializableConsumer<SSTableReader> before,
- IIsolatedExecutor.SerializableConsumer<SSTableReader> after)
- {
- for (IInvokableInstance instance : validate)
- {
- instance.runOnInstance(() -> {
- ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE);
- for (SSTableReader sstable : cfs.getLiveSSTables())
- {
- logger.info("SSTable {} before compaction: {}", sstable.getFilename(), sstable.getCoordinatorLogOffsets());
- before.accept(sstable);
- }
- });
- }
-
- // Activation ID must be persisted and broadcast across all peers in the cluster for any to mark as persisted + reconciled
- cluster.forEach(i -> {
- i.runOnInstance(() -> {
- MutationTrackingService.instance.persistLogStateForTesting();
- MutationTrackingService.instance.broadcastOffsetsForTesting();
- });
- });
-
- // Broadcast is async, wait until completion
- Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
-
- for (IInvokableInstance instance : validate)
- {
- instance.runOnInstance(() -> {
- ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE);
- logger.info("Triggering compaction on instance {}", cfs.metadata.keyspace);
- CompactionManager.instance.performMaximal(cfs);
-
- for (SSTableReader sstable : cfs.getLiveSSTables())
- {
- logger.info("SSTable {} after compaction: {}", sstable.getFilename(), sstable.getCoordinatorLogOffsets());
- after.accept(sstable);
- }
- });
- }
- }
-
- private static void assertLocalSelect(Iterable<IInvokableInstance> validate, IIsolatedExecutor.SerializableConsumer<Object[][]> onRows)
- {
- for (IInvokableInstance instance : validate)
- {
- {
- Object[][] rows = instance.executeInternal(withKeyspace("SELECT * FROM %s." + TABLE + " WHERE k = 1"));
- onRows.accept(rows);
- }
- {
- Object[][] rows = instance.executeInternal(withKeyspace("SELECT * FROM %s." + TABLE));
- onRows.accept(rows);
- }
- }
- }
-
- private static void bounce(Cluster cluster)
- {
- cluster.forEach(instance -> {
- try
- {
- instance.shutdown().get();
- }
- catch (InterruptedException | ExecutionException e)
- {
- throw new RuntimeException(e);
- }
- instance.startup();
- });
- }
-
- private static <T> T callSerialized(IInvokableInstance instance, IIsolatedExecutor.SerializableSupplier<IVersionedSerializer<T>> serializer, IIsolatedExecutor.SerializableCallable<T> callable)
- {
- ByteBuffer serialized = instance.callOnInstance(() -> {
- T deserialized = callable.call();
- IVersionedSerializer<T> serialize = serializer.get();
- try
- {
- return serialize.serialize(deserialized, MessagingService.current_version);
- }
- catch (IOException e)
- {
- throw new RuntimeException(e);
- }
- });
- try
- {
- return serializer.get().deserialize(serialized, MessagingService.current_version);
- }
- catch (IOException e)
- {
- throw new RuntimeException(e);
- }
- }
-}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedImportFailureTest.java b/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedImportFailureTest.java
new file mode 100644
index 0000000000..b6e6ce290f
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedImportFailureTest.java
@@ -0,0 +1,409 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.tracking;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import org.assertj.core.api.Assertions;
+
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.api.ICoordinator;
+import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.apache.cassandra.distributed.api.IIsolatedExecutor;
+import org.apache.cassandra.distributed.shared.AssertUtils;
+import org.apache.cassandra.distributed.shared.ClusterUtils;
+import org.apache.cassandra.distributed.shared.Uninterruptibles;
+import org.apache.cassandra.exceptions.ReadTimeoutException;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.replication.ActivatedTransfers;
+import org.apache.cassandra.replication.ActivationRequest;
+import org.apache.cassandra.replication.ShortMutationId;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.ownership.DataPlacement;
+
+import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows;
+import static org.apache.cassandra.distributed.shared.AssertUtils.row;
+import static org.apache.cassandra.replication.ActivationRequest.Phase.COMMIT;
+import static org.apache.cassandra.replication.ActivationRequest.Phase.PREPARE;
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class TrackedImportFailureTest extends TrackedTransferTestBase
+{
+ private static final Logger logger = LoggerFactory.getLogger(TrackedImportFailureTest.class);
+
+ @Test
+ public void importMissedActivationPrepare() throws Throwable
+ {
+ importMissedActivation(PREPARE);
+ }
+
+ @Test
+ public void importMissedActivationCommit() throws Throwable
+ {
+ importMissedActivation(COMMIT);
+ }
+
+ public void importMissedActivation(ActivationRequest.Phase phase) throws Throwable
+ {
+ int MISSED_ACTIVATION = 2;
+ try (Cluster cluster = cluster(TrackedTransferTestBase.ByteBuddyInjections.SkipActivation.install(MISSED_ACTIVATION)))
+ {
+ TrackedTransferTestBase.ByteBuddyInjections.SkipActivation.setup(cluster, phase);
+ createSchema(cluster);
+
+ Set<IInvokableInstance> missed = Collections.singleton(cluster.get(MISSED_ACTIVATION));
+ Iterable<IInvokableInstance> received = cluster.stream().filter(instance -> !missed.contains(instance)).collect(Collectors.toList());
+
+ Assertions.assertThatThrownBy(() -> doImport(cluster))
+ .hasMessageContaining("Failed adding SSTables")
+ .cause()
+ .hasMessageContaining("Failed streaming on 1 instance(s):")
+ .cause()
+ .hasMessageMatching("Tracked transfer failed during " + phase + " on " + cluster.get(MISSED_ACTIVATION).broadcastAddress() + " due to TIMEOUT")
+ .hasNoCause();
+
+ assertSummary(received, summary -> {
+ Assertions.assertThat(summary).satisfies(s -> {
+ assert s.reconciledIds() == 0;
+ assert s.unreconciledIds() == (phase == COMMIT ? 1 : 0);
+ });
+ });
+ assertSummary(missed, summary -> {
+ Assertions.assertThat(summary).satisfies(s -> {
+ assert s.reconciledIds() == 0;
+ assert s.unreconciledIds() == 0;
+ });
+ });
+
+ switch (phase)
+ {
+ case PREPARE:
+ // Activation did not start, files should be cleaned up
+ Uninterruptibles.sleepUninterruptibly(5, TimeUnit.SECONDS);
+ assertPendingDirs(cluster, (File pendingUuidDir) -> {
+ Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isEmpty();
+ });
+ break;
+ case COMMIT:
+ // Activation did not complete, files should still exist on all replicas
+ assertPendingDirs(cluster, (File pendingUuidDir) -> {
+ Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isNotEmpty();
+ });
+ break;
+ }
+
+ // If the activation is not everywhere, it shouldn't be purged on compaction
+ assertCompaction(cluster, received, TRANSFERS_EXIST, TRANSFERS_EXIST);
+
+ if (phase == PREPARE)
+ return;
+
+ // Permit activation of missed commits during read reconciliation
+ TrackedTransferTestBase.ByteBuddyInjections.SkipActivation.setup(cluster, null);
+
+ // Use coordinated query rather to confirm read reconciliation triggers activation
+ IInvokableInstance coordinator = cluster.get(3); // not initial transfer coordinator, but received activation
+ assertCoordinatedRead(coordinator, rows -> {
+ assertRows(rows, row(1, 1));
+ });
+
+ // Confirm others receive activation
+ assertLocalSelect(missed, rows -> {
+ assertRows(rows, row(1, 1));
+ });
+
+ assertCompaction(cluster, cluster, TRANSFERS_EXIST, TRANSFERS_EMPTY);
+
+ // Activation completed, files should be removed
+ assertPendingDirs(cluster, (File pendingUuidDir) -> {
+ Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isEmpty();
+ });
+ }
+ }
+
+ /**
+ * If a replica is missing an activation, executes a data read, then discovers the missing transfer during read
+ * reconciliation, it needs to augment the client response with the new transfer.
+ */
+ @Test
+ public void importMissingOnDataReplicaDuringAugment() throws Throwable
+ {
+ ActivationRequest.Phase phase = COMMIT;
+ int MISSED_ACTIVATION = 2;
+ try (Cluster cluster = cluster(TrackedTransferTestBase.ByteBuddyInjections.SkipActivation.install(MISSED_ACTIVATION)))
+ {
+ TrackedTransferTestBase.ByteBuddyInjections.SkipActivation.setup(cluster, phase);
+ createSchema(cluster);
+
+ IInvokableInstance missed = cluster.get(MISSED_ACTIVATION);
+
+ Assertions.assertThatThrownBy(() -> doImport(cluster))
+ .hasMessageContaining("Failed adding SSTables")
+ .cause()
+ .hasMessageContaining("Failed streaming on 1 instance(s):")
+ .cause()
+ .hasMessageMatching("Tracked transfer failed during " + phase + " on " + missed.broadcastAddress() + " due to TIMEOUT")
+ .hasNoCause();
+
+ assertSummary(Collections.singleton(missed), summary -> {
+ Assertions.assertThat(summary).satisfies(s -> {
+ assert s.reconciledIds() == 0;
+ assert s.unreconciledIds() == 0;
+ });
+ });
+
+ // Permit activation of missed commits during read reconciliation
+ TrackedTransferTestBase.ByteBuddyInjections.SkipActivation.setup(cluster, null);
+
+ // First read will fail due to failure to augment with transfer
+ long mark = missed.logs().mark();
+ Assertions.assertThatThrownBy(() -> {
+ assertCoordinatedRead(missed, rows -> {
+ assertRows(rows, row(1, 1));
+ });
+ }).isInstanceOf(missed.callOnInstance(() -> ReadTimeoutException.class)); // use instance classloader
+
+ List<String> logs = missed.logs().grep(mark, "Missing mutation ShortMutationId").getResult();
+ Assertions.assertThat(logs).isNotEmpty();
+
+ // Retry succeeds
+ assertCoordinatedRead(missed, rows -> {
+ assertRows(rows, row(1, 1));
+ });
+ }
+ }
+
+ /*
+ * Ensure that activation IDs attached to SSTables aren't spread across Token boundaries by compaction.
+ *
+ * For example:
+ * IMPORT_TOKEN is owned by replicas (A, B)
+ * OUTSIDE_IMPORT_TOKEN is owned by replicas (B, C)
+ * Execute import so (A, B) have IMPORT_TOKEN
+ * Execute plain write so (B, C) have OUTSIDE_IMPORT_TOKEN
+ * Do a major compaction on B so IMPORT_TOKEN and OUTSIDE_IMPORT_TOKEN are compacted together into the same SSTable
+ * Execute a data read for OUTSIDE_IMPORT_TOKEN against B, ensure it doesn't contain any activation IDs
+ */
+ @Test
+ public void importActivationMergedByCompaction() throws Throwable
+ {
+ try (Cluster cluster = cluster((cl, tg, instance, gen) -> ByteBuddyInjections.SkipPurgeTransfers.install().initialise(cl, tg, instance, gen)))
+ {
+ createSchema(cluster, 2);
+
+ Set<IInvokableInstance> inImportRange = new HashSet<>();
+ cluster.forEach(instance -> {
+ logger.debug("Instance {} ring is {}", ClusterUtils.instanceId(instance), ClusterUtils.ring(instance));
+ boolean isInRange = instance.callOnInstance(() -> {
+ ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE);
+ DataPlacement placement = ClusterMetadata.current().placements.get(cfs.keyspace.getMetadata().params.replication);
+ return placement.writes.forToken(IMPORT_TOKEN).get().containsSelf();
+ });
+ if (isInRange)
+ inImportRange.add(instance);
+ });
+ Assertions.assertThat(inImportRange).hasSize(2);
+
+ // Find a partition key that's not owned by the same replicas as the import
+ Murmur3Partitioner.LongToken NON_IMPORT_TOKEN = new Murmur3Partitioner.LongToken(IMPORT_TOKEN.getLongValue() * 3);
+ int NON_IMPORT_PK = Int32Type.instance.compose(Murmur3Partitioner.LongToken.keyForToken(NON_IMPORT_TOKEN));
+
+ Set<IInvokableInstance> inNonImportRange = new HashSet<>();
+ cluster.forEach(instance -> {
+ boolean isInRange = instance.callOnInstance(() -> {
+ ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE);
+ DataPlacement placement = ClusterMetadata.current().placements.get(cfs.keyspace.getMetadata().params.replication);
+ return placement.writes.forToken(NON_IMPORT_TOKEN).get().containsSelf();
+ });
+ if (isInRange)
+ inNonImportRange.add(instance);
+ });
+ Assertions.assertThat(inNonImportRange).hasSize(2);
+ Assertions.assertThat(inNonImportRange).isNotEqualTo(inImportRange);
+
+ // Import: (A, B)
+ // Plain: (B, C)
+ IInvokableInstance A = null;
+ IInvokableInstance B = null;
+ IInvokableInstance C = null;
+ for (IInvokableInstance instance : cluster)
+ {
+ boolean isImport = inImportRange.contains(instance);
+ boolean isNonImport = inNonImportRange.contains(instance);
+ if (isImport && isNonImport)
+ B = instance;
+ else if (isImport)
+ A = instance;
+ else if (isNonImport)
+ C = instance;
+ }
+ Assertions.assertThat(A).isNotNull();
+ Assertions.assertThat(B).isNotNull();
+ Assertions.assertThat(C).isNotNull();
+
+ doImport(cluster, A);
+ assertLocalSelect(List.of(A, B), (IIsolatedExecutor.SerializableConsumer<Object[][]>) rows -> {
+ assertRows(rows, row(IMPORT_PK, IMPORT_PK));
+ });
+
+ ShortMutationId importTransferId = callSerialized(A, () -> ShortMutationId.serializer, () -> {
+ ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE);
+ for (SSTableReader sstable : cfs.getLiveSSTables())
+ {
+ ActivatedTransfers transfers = sstable.getCoordinatorLogOffsets().transfers();
+ if (!transfers.isEmpty())
+ return transfers.iterator().next();
+ }
+ return null;
+ });
+ Assertions.assertThat(importTransferId).isNotNull();
+ C.coordinator().execute(withKeyspace("INSERT INTO %s." + TABLE + "(k, v) VALUES (?, ?)"), ConsistencyLevel.ALL, NON_IMPORT_PK, NON_IMPORT_PK);
+ assertCompaction(cluster, Collections.singleton(B), NOOP, NOOP);
+
+ // Reading from B for a range that doesn't include the import shouldn't include any transfer IDs, even though they've been compacted together
+ long mark = B.logs().mark();
+ Object[][] rows = B.coordinator().execute(withKeyspace("SELECT * FROM %s." + TABLE + " WHERE k = ?"), ConsistencyLevel.ALL, NON_IMPORT_PK);
+ assertRows(rows, row(NON_IMPORT_PK, NON_IMPORT_PK));
+ Assertions.assertThat(B.logs().grep(mark, "Found overlapping activation ID ").getResult()).isEmpty();
+
+ // But if the read range does include a transfer ID, it should have been added
+ mark = B.logs().mark();
+ rows = B.coordinator().execute(withKeyspace("SELECT * FROM %s." + TABLE + " WHERE k = ?"), ConsistencyLevel.ALL, IMPORT_PK);
+ assertRows(rows, row(IMPORT_PK, IMPORT_PK));
+ Assertions.assertThat(B.logs().grep(mark, "Found overlapping activation ID ").getResult()).isNotEmpty();
+ }
+ }
+
+ @Test
+ public void importFailedStreamCleanup() throws Throwable
+ {
+ int FAILED_STREAM = 3;
+ try (Cluster cluster = cluster(ByteBuddyInjections.FailIncomingStream.install(FAILED_STREAM)))
+ {
+ createSchema(cluster);
+
+ IInvokableInstance importer = cluster.get(1);
+ IInvokableInstance missed = cluster.get(FAILED_STREAM);
+
+ long mark = importer.logs().mark();
+ Assertions.assertThatThrownBy(() -> doImport(cluster, importer))
+ .isInstanceOf(RuntimeException.class)
+ .cause()
+ .isInstanceOf(RuntimeException.class)
+ .cause()
+ .hasMessageContaining("Remote peer " + missed.broadcastAddress() + " failed stream session");
+ List<String> logs = importer.logs().watchFor(mark, "Remote peer " + missed.broadcastAddress().toString() + " failed stream session").getResult();
+ Assertions.assertThat(logs).isNotEmpty();
+
+ // Await cleanup of failed stream
+ Uninterruptibles.sleepUninterruptibly(5, TimeUnit.SECONDS);
+
+ assertPendingDirs(cluster, (File pendingUuidDir) -> {
+ Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isEmpty();
+ });
+
+ // empty
+ assertLocalSelect(cluster, AssertUtils::assertRows);
+ }
+ }
+
+ @Test
+ @Ignore("Reactivate this when we support CL < ALL for tracked imports")
+ public void importReplicaDown() throws Throwable
+ {
+ try (Cluster cluster = cluster())
+ {
+ String keyspace = "replica_down";
+ createSchema(cluster, keyspace);
+
+ Iterable<IInvokableInstance> down = Collections.singleton(cluster.get(3));
+ Iterable<IInvokableInstance> up = cluster.stream().filter(instance -> instance != down).collect(Collectors.toList());
+ for (IInvokableInstance instance : down)
+ instance.shutdown().get();
+
+ doImport(cluster, keyspace);
+
+ cluster.get(3).startup();
+
+ // Transfers did not complete, files should still exist on up replicas
+ assertPendingDirs(up, keyspace, (File pendingUuidDir) -> {
+ assertThat(pendingUuidDir.listUnchecked(File::isFile)).isNotEmpty();
+ });
+ assertPendingDirs(down, keyspace, (File pendingUuidDir) -> {
+ assertThat(pendingUuidDir.listUnchecked(File::isFile)).isEmpty();
+ });
+
+ // Transfers did not complete, transfer IDs should not be removed
+ assertCompaction(cluster, keyspace, cluster, TRANSFERS_EXIST, TRANSFERS_EXIST);
+
+ assertLocalSelect(up, keyspace, rows -> assertRows(rows, row(1, 1)));
+ }
+ }
+
+ private static void assertCoordinatedRead(IInvokableInstance instance, IIsolatedExecutor.SerializableConsumer<Object[][]> onRows)
+ {
+ ICoordinator coordinator = instance.coordinator();
+ String cql = "SELECT * FROM %s." + TABLE + " WHERE k = 1";
+ Object[][] rows = coordinator.execute(withKeyspace(cql), ConsistencyLevel.ALL);
+ onRows.accept(rows);
+ }
+
+ private static <T> T callSerialized(IInvokableInstance instance, IIsolatedExecutor.SerializableSupplier<IVersionedSerializer<T>> serializer, IIsolatedExecutor.SerializableCallable<T> callable)
+ {
+ ByteBuffer serialized = instance.callOnInstance(() -> {
+ T deserialized = callable.call();
+ IVersionedSerializer<T> serialize = serializer.get();
+ try
+ {
+ return serialize.serialize(deserialized, MessagingService.current_version);
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException(e);
+ }
+ });
+ try
+ {
+ return serializer.get().deserialize(serialized, MessagingService.current_version);
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedImportTransferTest.java b/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedImportTransferTest.java
new file mode 100644
index 0000000000..f311381ab1
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedImportTransferTest.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.tracking;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.function.Consumer;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.apache.cassandra.distributed.test.sai.SAIUtil;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.ownership.DataPlacement;
+
+import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows;
+import static org.apache.cassandra.distributed.shared.AssertUtils.row;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * For now, tracked import with a replica down is not supported. The intention is to support this scenario by allowing
+ * users to provide a {@link ConsistencyLevel} for tracked import operations, where the import will complete if
+ * sufficient replicas acknowledge the transfer and activate it.
+ */
+public class TrackedImportTransferTest extends TrackedTransferTestBase
+{
+ private static final Logger logger = LoggerFactory.getLogger(TrackedImportTransferTest.class);
+
+ private static Cluster cluster;
+
+ @BeforeClass
+ public static void setup() throws IOException
+ {
+ cluster = cluster();
+ }
+
+ @AfterClass
+ public static void teardown()
+ {
+ if (cluster != null)
+ cluster.close();
+ }
+
+ @Test
+ public void importHappyPath() throws Throwable
+ {
+ String keyspace = "happy_path";
+ createSchema(cluster, keyspace);
+ doImport(cluster, keyspace);
+
+ // All pending/ dirs should be empty, should have no SSTables left if all the transfers completed
+ assertPendingDirs(cluster, keyspace, (File pendingUuidDir) -> {
+ assertThat(pendingUuidDir.listUnchecked(File::isFile)).isEmpty();
+ });
+
+ // Verify transfer IDs exist before compaction, then compact, then verify they're removed
+ assertCompaction(cluster, keyspace, cluster, TRANSFERS_EXIST, TRANSFERS_EMPTY);
+
+ // Run after compaction, to enforce offset persistence + broadcast
+ assertSummary(cluster, keyspace, summary -> {
+ assertThat(summary).satisfies(s -> {
+ assert s.reconciledIds() == 1;
+ assert s.unreconciledIds() == 0;
+ });
+ });
+
+ assertLocalSelect(cluster, keyspace, rows -> assertRows(rows, row(1, 1)));
+ }
+
+ @Test
+ public void importIndexAlreadyPresent() throws Throwable
+ {
+ String keyspace = "index_present";
+ createSchema(cluster, keyspace);
+
+ String indexName = "v_idx";
+ String indexCql = withKeyspace("CREATE INDEX " + indexName + " ON %s." + TABLE + " (v) USING 'sai'", keyspace);
+ cluster.schemaChange(indexCql);
+
+ // This will add an SSTable that already has an SAI index, that needs to be distributed alongside the SSTable on transfer
+ IInvokableInstance importer = cluster.get(1);
+ long mark = importer.logs().mark();
+ doImport(cluster, importer, keyspace, indexCql);
+ List<String> logs = importer.logs().grep(mark, "Submitting incremental index build of " + indexName).getResult();
+ assertThat(logs).isEmpty();
+
+ // Index should exist and be queryable on all replicas after import
+ SAIUtil.assertIndexQueryable(cluster, keyspace, indexName);
+
+ // Validate queries using the index
+ cluster.forEach(instance -> {
+ Object[][] rows = instance.executeInternal(withKeyspace("SELECT * FROM %s." + TABLE + " WHERE v = 1", keyspace));
+ assertRows(rows, row(1, 1));
+ });
+ }
+
+ @Test
+ public void importBuildsIndex() throws Throwable
+ {
+ String keyspace = "import_builds";
+ createSchema(cluster, keyspace);
+
+ String indexName = "v_idx";
+ cluster.schemaChange(withKeyspace("CREATE INDEX " + indexName + " ON %s." + TABLE + " (v) USING 'sai'", keyspace));
+
+ // This will add an SSTable that's missing an SAI index, and the index will be built during the import on
+ // the coordinator
+ IInvokableInstance importer = cluster.get(1);
+ long mark = importer.logs().mark();
+ doImport(cluster, importer, keyspace);
+ List<String> logs = importer.logs().watchFor(mark, Duration.ofMinutes(1), "Submitting incremental index build of " + indexName).getResult();
+ assertThat(logs).isNotEmpty();
+
+ // Index should exist and be queryable on all replicas after import
+ SAIUtil.assertIndexQueryable(cluster, keyspace, indexName);
+
+ // Validate queries using the index
+ cluster.forEach(instance -> {
+ Object[][] rows = instance.executeInternal(withKeyspace("SELECT * FROM %s." + TABLE + " WHERE v = 1", keyspace));
+ assertRows(rows, row(1, 1));
+ });
+ }
+
+ @Test
+ public void importOutOfRange() throws Throwable
+ {
+ String keyspace = "out_of_range";
+ createSchema(cluster, keyspace, 1);
+
+ Set<IInvokableInstance> inRange = new HashSet<>();
+ Set<IInvokableInstance> outOfRange = new HashSet<>();
+ cluster.forEach(instance -> {
+ boolean importReplica = instance.callOnInstance(() -> {
+ ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(keyspace, TABLE);
+ DataPlacement placement = ClusterMetadata.current().placements.get(cfs.keyspace.getMetadata().params.replication);
+ return placement.writes.forToken(IMPORT_TOKEN).get().containsSelf();
+ });
+ (importReplica ? inRange : outOfRange).add(instance);
+ });
+ logger.info("inRange: {}, outOfRange: {}", inRange, outOfRange);
+
+ assertThat(inRange).hasSize(1);
+ IInvokableInstance onlyInRange = inRange.iterator().next();
+
+ // Reject import out of range
+ for (IInvokableInstance instance : outOfRange)
+ {
+ long mark = instance.logs().mark();
+ Consumer<List<String>> onResult = failedDirs -> assertThat(failedDirs).hasSize(1);
+ doImport(cluster, instance, onResult, keyspace, null);
+ instance.logs().grep(mark, "java.lang.RuntimeException: Key DecoratedKey(-4069959284402364209, 00000001) is not contained in the given ranges");
+ }
+
+ doImport(cluster, onlyInRange, keyspace);
+
+ assertSummary(Collections.singleton(onlyInRange), keyspace, summary -> {
+ assertThat(summary).satisfies(s -> {
+ assert s.reconciledIds() == 1;
+ assert s.unreconciledIds() == 0;
+ });
+ });
+
+ for (IInvokableInstance instance : outOfRange)
+ {
+ // Out of range shouldn't have any transfers
+ assertCompaction(cluster, keyspace, Collections.singleton(instance), TRANSFERS_EMPTY, TRANSFERS_EMPTY);
+
+ // Run after compaction, to enforce offset persistence + broadcast
+ assertSummary(Collections.singleton(instance), keyspace, summary -> {
+ assertThat(summary).isNull();
+ });
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/repair/CompletableRemoteSyncTask.java b/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedNonZeroCopyRepairTransferTest.java
similarity index 71%
copy from src/java/org/apache/cassandra/repair/CompletableRemoteSyncTask.java
copy to test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedNonZeroCopyRepairTransferTest.java
index c4fe6c8b20..710e4e01f2 100644
--- a/src/java/org/apache/cassandra/repair/CompletableRemoteSyncTask.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedNonZeroCopyRepairTransferTest.java
@@ -16,13 +16,17 @@
* limitations under the License.
*/
-package org.apache.cassandra.repair;
+package org.apache.cassandra.distributed.test.tracking;
-import java.util.List;
+import java.io.IOException;
-import org.apache.cassandra.streaming.SessionSummary;
+import org.junit.BeforeClass;
-public interface CompletableRemoteSyncTask
+public class TrackedNonZeroCopyRepairTransferTest extends TrackedRepairTransferSuccessTestBase
{
- void syncComplete(boolean success, List<SessionSummary> summaries);
+ @BeforeClass
+ public static void setup() throws IOException
+ {
+ cluster = cluster(NON_ZCS_CONFIG);
+ }
}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedRepairFailureTest.java b/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedRepairFailureTest.java
new file mode 100644
index 0000000000..9748968887
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedRepairFailureTest.java
@@ -0,0 +1,390 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.tracking;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import net.bytebuddy.ByteBuddy;
+import net.bytebuddy.dynamic.loading.ClassLoadingStrategy;
+import net.bytebuddy.implementation.bind.annotation.SuperCall;
+import org.awaitility.Awaitility;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.streaming.CassandraStreamReceiver;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.apache.cassandra.distributed.api.NodeToolResult;
+import org.apache.cassandra.distributed.shared.ClusterUtils;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.replication.ActivationRequest;
+
+import static net.bytebuddy.implementation.MethodDelegation.to;
+import static net.bytebuddy.matcher.ElementMatchers.named;
+import static net.bytebuddy.matcher.ElementMatchers.takesNoArguments;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import static org.apache.cassandra.distributed.api.ConsistencyLevel.ALL;
+import static org.apache.cassandra.distributed.api.ConsistencyLevel.QUORUM;
+import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows;
+import static org.apache.cassandra.distributed.shared.AssertUtils.row;
+
+public class TrackedRepairFailureTest extends TrackedRepairTransferTestBase
+{
+ @Test
+ public void testFullRepairPartiallyCompleteAnomaly() throws IOException, ExecutionException, InterruptedException, TimeoutException
+ {
+ try (Cluster cluster = cluster(StreamReceiverFailureHelper::install))
+ {
+ cluster.schemaChange("CREATE KEYSPACE " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3} AND replication_type='tracked';");
+ String TABLE_SCHEMA_CQL = "CREATE TABLE " + tableWithKeyspace(KEYSPACE) + " (k INT PRIMARY KEY, v INT)";
+ cluster.schemaChange(TABLE_SCHEMA_CQL);
+
+ IInvokableInstance COORDINATING = cluster.get(1);
+ IInvokableInstance RECEIVING = cluster.get(2);
+ IInvokableInstance MISSING = cluster.get(3);
+
+ /*
+ If we were to start this process with a normal write, that write would be added to the log. When repair
+ validation runs and finds the mismatching range, it streams the log. MISSING then receives the log and
+ applies the new mutation to the memtable, where it's visible to reads.
+
+ We want to emulate a situation where one node has a mutation that's not present in the log, hence the
+ roundabout write path. Once the mutation has completed and is repaired on the coordinator, it's been
+ (durably) reconciled on all replicas. Then, drop this SSTable on the other peers and we should have a full
+ repair digest mismatch.
+
+ The idea here is to emulate logical data corruption, where SSTables mismatch but the logs are in agreement.
+ */
+ COORDINATING.coordinator().execute("INSERT INTO " + tableWithKeyspace(KEYSPACE) + " (k, v) " + "VALUES (?, ?)", ALL, 1, 1);
+ COORDINATING.flush(KEYSPACE);
+ Awaitility.waitAtMost(1, TimeUnit.MINUTES).pollDelay(1, TimeUnit.SECONDS)
+ .until(() -> {
+ boolean isRepaired = COORDINATING.callOnInstance(() -> {
+ ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE);
+ Set<SSTableReader> sstables = cfs.getLiveSSTables();
+ if (sstables.size() != 1)
+ return false;
+ SSTableReader sstable = sstables.iterator().next();
+ return sstable.isRepaired();
+ });
+ if (!isRepaired)
+ COORDINATING.forceCompact(KEYSPACE, TABLE);
+ return isRepaired;
+ });
+ List.of(RECEIVING, MISSING).forEach(instance -> instance.runOnInstance(() -> {
+ ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE);
+ cfs.truncateBlockingWithoutSnapshot();
+ }));
+ cluster.forEach(instance -> {
+ // Before repair, peers should have no data
+ Object[][] rows = instance.executeInternal("SELECT * FROM " + KEYSPACE + ".tbl WHERE k = 1");
+ if (instance == COORDINATING)
+ assertRows(rows, row(1, 1));
+ else
+ assertRows(rows);
+ });
+
+ // Prevent repair stream from completing
+ MISSING.runOnInstance(() -> StreamReceiverFailureHelper.shouldWait.set(true));
+
+ // Run full repair from COORDINATING
+ {
+ ExecutorService repairExecutor = Executors.newSingleThreadExecutor();
+ Future<NodeToolResult> repair = repairExecutor.submit(() -> COORDINATING.nodetoolResult("repair", "--full", KEYSPACE));
+ Awaitility.waitAtMost(10, TimeUnit.SECONDS).pollDelay(1, TimeUnit.SECONDS)
+ .until(() -> {
+ int finished = StreamReceiverFailureHelper.getFinishedRepairs(RECEIVING);
+ return finished > 0;
+ });
+ MISSING.runOnInstance(() -> StreamReceiverFailureHelper.shouldThrow.set(true));
+ repair.get(10, TimeUnit.SECONDS).asserts().failure();
+ repairExecutor.shutdown();
+ }
+
+ // Even after partial repair, RECEIVED should not move its SSTable to the live set, since the repair failed
+ cluster.forEach(instance -> {
+ Object[][] rows = instance.executeInternal("SELECT * FROM " + KEYSPACE + ".tbl WHERE k = 1");
+ if (instance == COORDINATING)
+ assertRows(rows, row(1, 1));
+ else
+ assertRows(rows);
+ });
+
+ /*
+ At this point, the repair is complete and partially applied. RECEIVED has an SSTable it received from
+ repair, and MISSING has no SSTables. If we were to do a tracked data read against RECEIVED, we'd have an
+ emptpy summary but rows, and if we were to execute the same read against MISSING we'd have an entirely
+ empty response. This would break monotonicity if a client executes a QUORUM read against RECEIVED then
+ against missing, because the empty summaries lead to no reconciliation happening.
+
+ To provide monotonicity in this scenario, we integrate the full repair with bulk transfer machinery and
+ tag the SSTables with transfer IDs that can be included in summaries and reconciled. Then, the initial data
+ read against RECEIVED includes transfer IDs that are reconciled. Reconciliation detects that RECEIVED has an
+ SSTable that isn't present on MISSING and streams them, so the subsequent read against MISSING is up to
+ date.
+ */
+
+ {
+ MISSING.runOnInstance(() -> StreamReceiverFailureHelper.shouldWait.set(false));
+ MISSING.runOnInstance(() -> StreamReceiverFailureHelper.shouldThrow.set(false));
+ // Don't let coordinating act as a replica for the read
+ cluster.filters().inbound().to(ClusterUtils.instanceId(COORDINATING)).drop();
+ MutationTrackingReadReconciliationTest.awaitNodeDead(RECEIVING, COORDINATING);
+ }
+ // Repair did not succeed sync, so it did not proceed to activation, so it's not visible on RECEIVING.
+ {
+ Object[][] rows = RECEIVING.coordinator().execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE k = 1", QUORUM);
+ assertRows(rows); // empty
+ }
+ cluster.filters().reset();
+ MutationTrackingReadReconciliationTest.awaitNodeAlive(RECEIVING, COORDINATING);
+
+ // Another repair succeeds, all peers should now agree on the local data
+ long mark = COORDINATING.logs().mark();
+ COORDINATING.nodetoolResult("repair", "--full", KEYSPACE).asserts().success();
+ List<String> logs = COORDINATING.logs().grep(mark, "Activating .* for ").getResult();
+ assertThat(logs).isNotEmpty();
+ cluster.forEach(instance -> {
+ Object[][] rows = instance.executeInternal("SELECT * FROM " + KEYSPACE + ".tbl WHERE k = 1");
+ assertRows(rows, row(1, 1));
+ });
+
+ // Make sure all instances can successfully coordinate
+ cluster.forEach(instance -> {
+ Object[][] rows = instance.coordinator().execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE k = 1", ALL);
+ assertRows(rows, row(1, 1));
+ });
+ }
+ }
+
+ @Test
+ public void testFullRepairCleanupOnFailure() throws IOException, ExecutionException, InterruptedException, TimeoutException
+ {
+ try (Cluster cluster = cluster(StreamReceiverFailureHelper::install))
+ {
+ cluster.schemaChange("CREATE KEYSPACE " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3} AND replication_type='tracked';");
+ String TABLE_SCHEMA_CQL = "CREATE TABLE " + KEYSPACE + '.' + TABLE + " (k INT PRIMARY KEY, v INT)";
+ cluster.schemaChange(TABLE_SCHEMA_CQL);
+
+ IInvokableInstance COORDINATING = cluster.get(1);
+ IInvokableInstance RECEIVING = cluster.get(2);
+ IInvokableInstance MISSING = cluster.get(3);
+
+ // Write a single row to COORDINATING node only
+ COORDINATING.executeInternal("INSERT INTO " + tableWithKeyspace(KEYSPACE) + " (k, v) VALUES (?, ?)", 1, 100);
+
+ // Before repair, only COORDINATING has data
+ for (IInvokableInstance instance : cluster)
+ {
+ Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE k = 1");
+ if (instance == COORDINATING)
+ assertRows(rows, row(1, 100));
+ else
+ assertRows(rows); // empty
+ }
+
+ // Prevent repair stream from completing
+ MISSING.runOnInstance(() -> StreamReceiverFailureHelper.shouldWait.set(true));
+
+ // Run full repair from COORDINATING
+ {
+ ExecutorService repairExecutor = Executors.newSingleThreadExecutor();
+ Future<NodeToolResult> repair = repairExecutor.submit(() -> COORDINATING.nodetoolResult("repair", "--full", KEYSPACE));
+ Awaitility.waitAtMost(10, TimeUnit.SECONDS).pollDelay(1, TimeUnit.SECONDS)
+ .until(() -> {
+ int finished = StreamReceiverFailureHelper.getFinishedRepairs(RECEIVING);
+ return finished > 0;
+ });
+
+ // Repair completed against RECEIVING, so it should have pending SSTables
+ {
+ List<String> pending = getPendingSSTablePaths(RECEIVING);
+ assertThat(pending).isNotEmpty();
+ }
+
+ MISSING.runOnInstance(() -> StreamReceiverFailureHelper.shouldThrow.set(true));
+ repair.get(10, TimeUnit.SECONDS).asserts().failure();
+ repairExecutor.shutdown();
+ }
+
+ // No pending SSTables on COORDINATING because they're streamed from the live set
+ // No pending SSTables on MISSING because of stream failure injection
+ // No pending SSTables on RECEIVING because they were cleaned up when the repair failed
+ for (IInvokableInstance instance : cluster)
+ {
+ Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE k = 1");
+ if (instance == COORDINATING)
+ assertRows(rows, row(1, 100));
+ else
+ assertRows(rows); // empty
+
+ List<String> pending = getPendingSSTablePaths(instance);
+ assertThat(pending).isEmpty();
+ }
+ }
+ }
+
+ @Test
+ public void testRepairFailsOnMissedActivation() throws IOException
+ {
+ try (Cluster cluster = cluster(ByteBuddyInjections.SkipActivation.install(2, 3)))
+ {
+ cluster.schemaChange("CREATE KEYSPACE " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3} AND replication_type='tracked';");
+ cluster.schemaChange("CREATE TABLE " + tableWithKeyspace(KEYSPACE) + " (k INT PRIMARY KEY, v INT)");
+
+ IInvokableInstance COORDINATING = cluster.get(1);
+
+ COORDINATING.executeInternal("INSERT INTO " + tableWithKeyspace(KEYSPACE) + " (k, v) VALUES (?, ?)", 1, 100);
+
+ // Before repair, only instance 1 has data
+ for (IInvokableInstance instance : cluster)
+ {
+ Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE k = 1");
+ if (instance == COORDINATING)
+ assertRows(rows, row(1, 100));
+ else
+ assertRows(rows); // empty
+ }
+
+ // Repair fails because activation is blocked on nodes 2 and 3
+ ByteBuddyInjections.SkipActivation.setup(cluster, ActivationRequest.Phase.COMMIT, true);
+ {
+ NodeToolResult repair = COORDINATING.nodetoolResult("repair", "--full", KEYSPACE);
+ repair.asserts().failure();
+ }
+ for (IInvokableInstance instance : cluster)
+ {
+ Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE k = 1");
+ List<String> pending = getPendingSSTablePaths(instance);
+ if (instance == COORDINATING)
+ {
+ assertRows(rows, row(1, 100));
+ assertThat(pending).isEmpty();
+ }
+ else
+ {
+ // Nodes 2 and 3 should only have pending SSTables
+ assertRows(rows);
+ assertThat(pending).isNotEmpty();
+ }
+ }
+
+ // Re-enable activation; read reconciliation at ALL should activate the pending SSTables on nodes 2 and 3
+ ByteBuddyInjections.SkipActivation.setup(cluster, null);
+ COORDINATING.coordinator().execute("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE k = 1", ALL);
+
+ for (IInvokableInstance instance : cluster)
+ {
+ Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE k = 1");
+ assertRows(rows, row(1, 100));
+ }
+
+ // Pending directory cleanup is scheduled, not synchronous with read reconciliation
+ for (IInvokableInstance instance : cluster)
+ {
+ Awaitility.waitAtMost(10, TimeUnit.SECONDS).pollDelay(1, TimeUnit.SECONDS)
+ .until(() -> {
+ List<String> pending = getPendingSSTablePaths(instance);
+ return pending.isEmpty();
+ });
+ }
+
+ // Make sure all instances can successfully coordinate
+ cluster.forEach(instance -> {
+ Object[][] rows = instance.coordinator().execute("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE k = 1", ALL);
+ assertRows(rows, row(1, 100));
+ });
+ }
+ }
+
+ private static List<String> getPendingSSTablePaths(IInvokableInstance instance)
+ {
+ return instance.callOnInstance(() -> {
+ ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE);
+ Set<File> pendingLocations = cfs.getDirectories().getPendingLocations();
+
+ List<String> pendingUuidDirs = new ArrayList<>();
+ for (File pendingDir : pendingLocations)
+ {
+ File[] uuidDirs = pendingDir.listUnchecked(File::isDirectory);
+ for (File dir : uuidDirs)
+ pendingUuidDirs.add(dir.absolutePath());
+ }
+ return pendingUuidDirs;
+ });
+ }
+
+ public static class StreamReceiverFailureHelper
+ {
+ private static final Logger logger = LoggerFactory.getLogger(StreamReceiverFailureHelper.class);
+
+ static AtomicBoolean shouldThrow = new AtomicBoolean(false);
+ static AtomicBoolean shouldWait = new AtomicBoolean(false);
+ static AtomicInteger count = new AtomicInteger(0);
+
+ /**
+ * {@link CassandraStreamReceiver#finished}
+ */
+ @SuppressWarnings("resource")
+ public static void install(ClassLoader classLoader, Integer instanceNum)
+ {
+ new ByteBuddy().rebase(CassandraStreamReceiver.class)
+ .method(named("finished").and(takesNoArguments()))
+ .intercept(to(StreamReceiverFailureHelper.class))
+ .make()
+ .load(classLoader, ClassLoadingStrategy.Default.INJECTION);
+ }
+
+ public static void finished(@SuperCall Callable<Void> zuper) throws Exception
+ {
+ while (shouldWait.get())
+ {
+ if (shouldThrow.get())
+ throw new RuntimeException("Test: failing stream session");
+
+ logger.info("Test: blocking finish of stream session");
+ Thread.sleep(1_000); // TODO: Look for a way to do this without sleeping
+ }
+ zuper.call();
+ logger.info("Test: finished stream session");
+ count.incrementAndGet();
+ }
+
+ protected static int getFinishedRepairs(IInvokableInstance instance)
+ {
+ return instance.callOnInstance(() -> StreamReceiverFailureHelper.count.get());
+ }
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedRepairTransferSuccessTestBase.java b/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedRepairTransferSuccessTestBase.java
new file mode 100644
index 0000000000..81413325ee
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedRepairTransferSuccessTestBase.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.tracking;
+
+import java.io.IOException;
+
+import org.junit.AfterClass;
+import org.junit.Test;
+
+import org.apache.cassandra.distributed.Cluster;
+
+public abstract class TrackedRepairTransferSuccessTestBase extends TrackedRepairTransferTestBase
+{
+ protected static Cluster cluster;
+
+ @AfterClass
+ public static void teardown()
+ {
+ if (cluster != null)
+ cluster.close();
+ }
+
+ @Test
+ public void testFullRepairShardAlignedSinglePlan() throws IOException
+ {
+ String keyspace = "full_repair_shard_aligned_single_plan";
+ testFullRepairSinglePlan(keyspace, cluster, KEY_200, 2, false, "repair", "--start-token", SHARD_ALIGNED_RANGE_2.left.toString(), "--end-token", SHARD_ALIGNED_RANGE_2.right.toString(), "--full", keyspace);
+ }
+
+ @Test
+ public void testFullRepairAcrossShardsSinglePlan() throws IOException
+ {
+ String keyspace = "full_repair_single_plan";
+ testFullRepairSinglePlan(keyspace, cluster, KEY_200, 2, false, "repair", "--full", keyspace);
+ }
+
+ @Test
+ public void testFullRepairShardAlignedSinglePlanOptimized() throws IOException
+ {
+ String keyspace = "full_repair_shard_aligned_single_plan_optimized";
+ testFullRepairSinglePlan(keyspace, cluster, KEY_200, 3, true, "repair", "--optimise-streams", "--start-token", SHARD_ALIGNED_RANGE_2.left.toString(), "--end-token", SHARD_ALIGNED_RANGE_2.right.toString(), "--full", keyspace);
+ }
+
+ @Test
+ public void testFullRepairAcrossShardsSinglePlanOptimized() throws IOException
+ {
+ String keyspace = "full_repair_single_plan_optimized";
+ testFullRepairSinglePlan(keyspace, cluster, KEY_200, 3, true, "repair", "--optimise-streams", "--full", keyspace);
+ }
+
+ @Test
+ public void testFullRepairShardAlignedRemoteSender() throws IOException
+ {
+ String keyspace = "full_repair_shard_aligned_remote_sender";
+ testFullRepairRemoteSender(keyspace, cluster, 2, false, "repair", "--start-token", SHARD_ALIGNED_RANGE_2.left.toString(), "--end-token", SHARD_ALIGNED_RANGE_2.right.toString(), "--full", keyspace);
+ }
+
+ @Test
+ public void testFullRepairAcrossShardsRemoteSender() throws IOException
+ {
+ String keyspace = "full_repair_remote_sender";
+ testFullRepairRemoteSender(keyspace, cluster, 2, false, "repair", "--full", keyspace);
+ }
+
+ @Test
+ public void testFullRepairShardAlignedRemoteSenderOptimized() throws IOException
+ {
+ String keyspace = "full_repair_aligned_optimized_remote_sender";
+ testFullRepairRemoteSender(keyspace, cluster, 3, true, "repair", "--optimise-streams", "--start-token", SHARD_ALIGNED_RANGE_2.left.toString(), "--end-token", SHARD_ALIGNED_RANGE_2.right.toString(), "--full", keyspace);
+ }
+
+ @Test
+ public void testFullRepairAcrossShardsRemoteSenderOptimized() throws IOException
+ {
+ String keyspace = "full_repair_optimized_remote_sender";
+ testFullRepairRemoteSender(keyspace, cluster, 3, true, "repair", "--optimise-streams", "--full", keyspace);
+ }
+
+ @Test
+ public void testFullRepairShardAlignedDuplicateSender() throws IOException
+ {
+ String keyspace = "full_repair_shard_aligned_duplicate_sender";
+ testFullRepairDuplicateSender(keyspace, cluster, KEY_100, 3, false, "repair", "--start-token", SHARD_ALIGNED_RANGE_1.left.toString(), "--end-token", SHARD_ALIGNED_RANGE_1.right.toString(), "--full", keyspace);
+ }
+
+ @Test
+ public void testFullRepairAcrossShardsDuplicateSender() throws IOException
+ {
+ String keyspace = "full_repair_duplicate_sender";
+ testFullRepairDuplicateSender(keyspace, cluster, KEY_100, 3, false, "repair", "--full", keyspace);
+ }
+
+ @Test
+ public void testFullRepairShardAlignedDuplicateSenderOptimized() throws IOException
+ {
+ String keyspace = "full_repair_optimized_aligned_duplicate_sender";
+ testFullRepairDuplicateSender(keyspace, cluster, KEY_100, 6, true, "repair", "--optimise-streams", "--start-token", SHARD_ALIGNED_RANGE_1.left.toString(), "--end-token", SHARD_ALIGNED_RANGE_1.right.toString(), "--full", keyspace);
+ }
+
+ @Test
+ public void testFullRepairAcrossShardsDuplicateSenderOptimized() throws IOException
+ {
+ String keyspace = "full_repair_optimized_duplicate_sender";
+ testFullRepairDuplicateSender(keyspace, cluster, KEY_100, 6, true, "repair", "--optimise-streams", "--full", keyspace);
+ }
+
+ @Test
+ public void testFullRepairShardAlignedMultiSender() throws IOException
+ {
+ String keyspace = "full_repair_shard_aligned_multi_sender";
+ testFullRepairMultiSender(keyspace, cluster, 3, false, "repair", "--start-token", SHARD_ALIGNED_RANGE_2.left.toString(), "--end-token", SHARD_ALIGNED_RANGE_2.right.toString(), "--full", keyspace);
+ }
+
+ @Test
+ public void testFullRepairAcrossShardsMultiSender() throws IOException
+ {
+ String keyspace = "full_repair_multi_sender";
+ testFullRepairMultiSender(keyspace, cluster, 3, false, "repair", "--full", keyspace);
+ }
+
+ @Test
+ public void testFullRepairShardAlignedMultiSenderOptimized() throws IOException
+ {
+ String keyspace = "full_repair_aligned_optimized_multi_sender";
+ testFullRepairMultiSender(keyspace, cluster, 6, true, "repair", "--optimise-streams", "--start-token", SHARD_ALIGNED_RANGE_2.left.toString(), "--end-token", SHARD_ALIGNED_RANGE_2.right.toString(), "--full", keyspace);
+ }
+
+ @Test
+ public void testFullRepairAcrossShardsMultiSenderOptimized() throws IOException
+ {
+ String keyspace = "full_repair_optimized_multi_sender";
+ testFullRepairMultiSender(keyspace, cluster, 6, true, "repair", "--optimise-streams", "--full", keyspace);
+ }
+
+ @Test
+ public void testFullRepairMultiSenderSameTokenShardAligned() throws IOException
+ {
+ String keyspace = "full_repair_multi_sender_same_token_aligned";
+ testFullRepairMultiSenderSameToken(keyspace, cluster, 3, false, "repair", "--start-token", SHARD_ALIGNED_RANGE_2.left.toString(), "--end-token", SHARD_ALIGNED_RANGE_2.right.toString(), "--full", keyspace);
+ }
+
+ @Test
+ public void testFullRepairMultiSenderSameTokenAcrossShards() throws IOException
+ {
+ String keyspace = "full_repair_multi_sender_same_token";
+ testFullRepairMultiSenderSameToken(keyspace, cluster, 3, false, "repair", "--full", keyspace);
+ }
+
+ @Test
+ public void testFullRepairMultiSenderSameTokenShardAlignedOptimized() throws IOException
+ {
+ String keyspace = "full_multi_sender_same_token_aligned_optimized";
+ testFullRepairMultiSenderSameToken(keyspace, cluster, 6, true, "repair", "--optimise-streams", "--start-token", SHARD_ALIGNED_RANGE_2.left.toString(), "--end-token", SHARD_ALIGNED_RANGE_2.right.toString(), "--full", keyspace);
+ }
+
+ @Test
+ public void testFullRepairMultiSenderSameTokenAcrossShardsOptimized() throws IOException
+ {
+ String keyspace = "full_multi_sender_same_token_optimized";
+ testFullRepairMultiSenderSameToken(keyspace, cluster, 6, true, "repair", "--optimise-streams", "--full", keyspace);
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedRepairTransferTestBase.java b/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedRepairTransferTestBase.java
new file mode 100644
index 0000000000..759b57a1fb
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedRepairTransferTestBase.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.tracking;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.List;
+
+import org.assertj.core.api.Assertions;
+
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.apache.cassandra.distributed.api.NodeToolResult;
+import org.apache.cassandra.distributed.shared.ClusterUtils;
+
+import static org.apache.cassandra.distributed.api.ConsistencyLevel.ALL;
+import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows;
+import static org.apache.cassandra.distributed.shared.AssertUtils.row;
+
+public abstract class TrackedRepairTransferTestBase extends TrackedTransferTestBase
+{
+ public void testFullRepairSinglePlan(String keyspace, Cluster cluster, ByteBuffer key, int syncs, boolean optimized, String... repairCommandAndArgs) throws IOException
+ {
+ cluster.schemaChange("CREATE KEYSPACE " + keyspace + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3} AND replication_type='tracked';");
+ cluster.schemaChange("CREATE TABLE " + tableWithKeyspace(keyspace) + " (pk BLOB PRIMARY KEY, v INT)");
+
+ IInvokableInstance coordinator = cluster.get(1);
+ coordinator.executeInternal("INSERT INTO " + tableWithKeyspace(keyspace) + " (pk, v) VALUES (?, 1)", key);
+
+ // Write should only be present on instance 1
+ cluster.forEach(instance -> {
+ Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", key);
+ if (ClusterUtils.instanceId(instance) == 1)
+ assertRows(rows, row(key, 1));
+ else
+ assertRows(rows); // empty
+ });
+
+ long mark = coordinator.logs().mark();
+ NodeToolResult result = coordinator.nodetoolResult(repairCommandAndArgs);
+ result.asserts().success();
+ List<String> logs = coordinator.logs().grep(mark, "Created " + syncs + (optimized ? " optimised" : "") + " sync tasks based on 3 merkle tree responses").getResult();
+ Assertions.assertThat(logs).isNotEmpty();
+
+ // Write visible on all instances after repair
+ cluster.forEach(instance -> {
+ Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", key);
+ assertRows(rows, row(key, 1));
+ });
+
+ // Make sure all instances can successfully coordinate
+ cluster.forEach(instance -> {
+ Object[][] rows = instance.coordinator().execute("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", ALL, key);
+ assertRows(rows, row(key, 1));
+ });
+ }
+
+ public void testFullRepairRemoteSender(String keyspace, Cluster cluster, int syncs, boolean optimized, String... repairCommandAndArgs) throws IOException
+ {
+ cluster.schemaChange("CREATE KEYSPACE " + keyspace + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3} AND replication_type='tracked';");
+ cluster.schemaChange("CREATE TABLE " + tableWithKeyspace(keyspace) + " (pk BLOB PRIMARY KEY, v INT)");
+
+ IInvokableInstance coordinator = cluster.get(1);
+ cluster.get(2).executeInternal("INSERT INTO " + tableWithKeyspace(keyspace) + " (pk, v) VALUES (?, 2)", KEY_201);
+
+ // Second key should only be present on instance 2
+ cluster.forEach(instance -> {
+ Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", KEY_201);
+ if (ClusterUtils.instanceId(instance) == 2)
+ assertRows(rows, row(KEY_201, 2));
+ else
+ assertRows(rows); // empty
+ });
+
+ long mark = coordinator.logs().mark();
+ NodeToolResult result = coordinator.nodetoolResult(repairCommandAndArgs);
+ result.asserts().success();
+ List<String> logs = coordinator.logs().grep(mark, "Created " + syncs + (optimized ? " optimised" : "") + " sync tasks based on 3 merkle tree responses").getResult();
+ Assertions.assertThat(logs).isNotEmpty();
+
+ // Write visible on all instances after repair
+ cluster.forEach(instance -> {
+ Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", KEY_201);
+ assertRows(rows, row(KEY_201, 2));
+ });
+
+ // Make sure all instances can successfully coordinate
+ cluster.forEach(instance -> {
+ Object[][] rows = instance.coordinator().execute("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", ALL, KEY_201);
+ assertRows(rows, row(KEY_201, 2));
+ });
+ }
+
+ public void testFullRepairDuplicateSender(String keyspace, Cluster cluster, ByteBuffer key, int syncs, boolean optimized, String... repairCommandAndArgs) throws IOException
+ {
+ cluster.schemaChange("CREATE KEYSPACE " + keyspace + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3} AND replication_type='tracked';");
+ cluster.schemaChange("CREATE TABLE " + tableWithKeyspace(keyspace) + " (pk BLOB PRIMARY KEY, v INT)");
+
+ IInvokableInstance coordinator = cluster.get(1);
+ coordinator.executeInternal("INSERT INTO " + tableWithKeyspace(keyspace) + " (pk, v) VALUES (?, 1)", key);
+ cluster.get(2).executeInternal("INSERT INTO " + tableWithKeyspace(keyspace) + " (pk, v) VALUES (?, 1)", key);
+
+ // Write should be missing on node 3
+ cluster.forEach(instance -> {
+ Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", key);
+ if (ClusterUtils.instanceId(instance) != 3)
+ assertRows(rows, row(key, 1));
+ else
+ assertRows(rows); // empty
+ });
+
+ long mark = coordinator.logs().mark();
+ coordinator.nodetoolResult(repairCommandAndArgs).asserts().success();
+
+ List<String> logs = coordinator.logs().grep(mark, "Created " + syncs + (optimized ? " optimised" : "") + " sync tasks based on 3 merkle tree responses").getResult();
+ Assertions.assertThat(logs).isNotEmpty();
+
+ // Write visible on all instances after repair
+ cluster.forEach(instance -> {
+ Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", key);
+ assertRows(rows, row(key, 1));
+ });
+
+ // Make sure all instances can successfully coordinate
+ cluster.forEach(instance -> {
+ Object[][] rows = instance.coordinator().execute("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", ALL, key);
+ assertRows(rows, row(key, 1));
+ });
+ }
+
+ public void testFullRepairMultiSender(String keyspace, Cluster cluster, int syncs, boolean optimized, String... repairCommandAndArgs) throws IOException
+ {
+ cluster.schemaChange("CREATE KEYSPACE " + keyspace + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3} AND replication_type='tracked';");
+ cluster.schemaChange("CREATE TABLE " + tableWithKeyspace(keyspace) + " (pk BLOB PRIMARY KEY, v INT)");
+
+ IInvokableInstance coordinator = cluster.get(1);
+ coordinator.executeInternal("INSERT INTO " + tableWithKeyspace(keyspace) + " (pk, v) VALUES (?, 1)", KEY_200);
+ cluster.get(2).executeInternal("INSERT INTO " + tableWithKeyspace(keyspace) + " (pk, v) VALUES (?, 2)", KEY_201);
+
+ // First key should only be present on instance 1
+ cluster.forEach(instance -> {
+ Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", KEY_200);
+ if (ClusterUtils.instanceId(instance) == 1)
+ assertRows(rows, row(KEY_200, 1));
+ else
+ assertRows(rows); // empty
+ });
+
+ // Second key should only be present on instance 2
+ cluster.forEach(instance -> {
+ Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", KEY_201);
+ if (ClusterUtils.instanceId(instance) == 2)
+ assertRows(rows, row(KEY_201, 2));
+ else
+ assertRows(rows); // empty
+ });
+
+ long mark = coordinator.logs().mark();
+ NodeToolResult result = coordinator.nodetoolResult(repairCommandAndArgs);
+ result.asserts().success();
+ List<String> logs = coordinator.logs().grep(mark, "Created " + syncs + (optimized ? " optimised" : "") + " sync tasks based on 3 merkle tree responses").getResult();
+ Assertions.assertThat(logs).isNotEmpty();
+
+ // Writes visible on all instances after repair
+ cluster.forEach(instance -> {
+ Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", KEY_200);
+ assertRows(rows, row(KEY_200, 1));
+ rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", KEY_201);
+ assertRows(rows, row(KEY_201, 2));
+ });
+
+ // Make sure all instances can successfully coordinate
+ cluster.forEach(instance -> {
+ Object[][] rows = instance.coordinator().execute("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", ALL, KEY_200);
+ assertRows(rows, row(KEY_200, 1));
+ rows = instance.coordinator().execute("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", ALL, KEY_201);
+ assertRows(rows, row(KEY_201, 2));
+ });
+ }
+
+ public void testFullRepairMultiSenderSameToken(String keyspace, Cluster cluster, int syncs, boolean optimized, String... repairCommandAndArgs) throws IOException
+ {
+ cluster.schemaChange("CREATE KEYSPACE " + keyspace + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3} AND replication_type='tracked';");
+ cluster.schemaChange("CREATE TABLE " + tableWithKeyspace(keyspace) + " (pk BLOB PRIMARY KEY, v INT)");
+
+ IInvokableInstance coordinator = cluster.get(1);
+ coordinator.executeInternal("INSERT INTO " + tableWithKeyspace(keyspace) + " (pk, v) VALUES (?, 1) USING TIMESTAMP 1", KEY_200);
+ cluster.get(2).executeInternal("INSERT INTO " + tableWithKeyspace(keyspace) + " (pk, v) VALUES (?, 2) USING TIMESTAMP 2", KEY_200);
+
+ cluster.forEach(instance -> {
+ Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", KEY_200);
+ if (ClusterUtils.instanceId(instance) == 1)
+ assertRows(rows, row(KEY_200, 1));
+ else if (ClusterUtils.instanceId(instance) == 2)
+ assertRows(rows, row(KEY_200, 2));
+ else
+ assertRows(rows); // empty
+ });
+
+ long mark = coordinator.logs().mark();
+ NodeToolResult result = coordinator.nodetoolResult(repairCommandAndArgs);
+ result.asserts().success();
+ List<String> logs = coordinator.logs().grep(mark, "Created " + syncs + (optimized ? " optimised" : "") + " sync tasks based on 3 merkle tree responses").getResult();
+ Assertions.assertThat(logs).isNotEmpty();
+
+ // Writes visible on all instances after repair
+ cluster.forEach(instance -> {
+ Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", KEY_200);
+ assertRows(rows, row(KEY_200, 2));
+ });
+
+ // Make sure all instances can successfully coordinate
+ cluster.forEach(instance -> {
+ Object[][] rows = instance.coordinator().execute("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", ALL, KEY_200);
+ assertRows(rows, row(KEY_200, 2));
+ });
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedTransferBounceTest.java b/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedTransferBounceTest.java
new file mode 100644
index 0000000000..2da449eead
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedTransferBounceTest.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.tracking;
+
+import java.io.IOException;
+import java.util.function.Consumer;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.IInstanceConfig;
+import org.apache.cassandra.distributed.api.IInstanceInitializer;
+
+import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows;
+import static org.apache.cassandra.distributed.shared.AssertUtils.row;
+import static org.apache.cassandra.replication.ActivationRequest.Phase.COMMIT;
+
+public class TrackedTransferBounceTest extends TrackedTransferTestBase
+{
+ /*
+ * When an import fails, bounce must not move the pending SSTables into the live set.
+ */
+ @Test
+ public void testBounceAfterPendingImport() throws Throwable
+ {
+ IInstanceInitializer initializer = ByteBuddyInjections.SkipActivation.install(1, 2, 3);
+ try (Cluster cluster = cluster(initializer))
+ {
+ ByteBuddyInjections.SkipActivation.setup(cluster, COMMIT);
+ createSchema(cluster);
+
+ Assertions.assertThatThrownBy(() -> doImport(cluster))
+ .hasMessageContaining("Failed adding SSTables")
+ .cause()
+ .hasMessageContaining("Tracked transfer failed during COMMIT");
+
+ assertPendingActivation(cluster);
+ assertLocalSelect(cluster, rows -> assertRows(rows, EMPTY_ROWS));
+
+ bounce(cluster);
+
+ assertPendingActivation(cluster);
+ assertLocalSelect(cluster, rows -> assertRows(rows, EMPTY_ROWS));
+ }
+ }
+
+ @Test
+ public void testBounceAfterPendingShardAlignedZeroCopy() throws IOException
+ {
+ testBounceAfterPendingRepair(ZCS_CONFIG, "repair", "--start-token", SHARD_ALIGNED_RANGE_1.left.toString(), "--end-token", SHARD_ALIGNED_RANGE_1.right.toString(), "--full", KEYSPACE);
+ }
+
+ @Test
+ public void testBounceAfterPendingAcrossShardsZeroCopy() throws IOException
+ {
+ testBounceAfterPendingRepair(ZCS_CONFIG, "repair", "--full", KEYSPACE);
+ }
+
+ @Test
+ public void testBounceAfterPendingShardAlignedNonZeroCopy() throws IOException
+ {
+ testBounceAfterPendingRepair(NON_ZCS_CONFIG, "repair", "--start-token", SHARD_ALIGNED_RANGE_1.left.toString(), "--end-token", SHARD_ALIGNED_RANGE_1.right.toString(), "--full", KEYSPACE);
+ }
+
+ @Test
+ public void testBounceAfterPendingAcrossShardsNonZeroCopy() throws IOException
+ {
+ testBounceAfterPendingRepair(NON_ZCS_CONFIG, "repair", "--full", KEYSPACE);
+ }
+
+ /*
+ * When a repair fails, bounce must not move the pending SSTables into the live set.
+ */
+ private static void testBounceAfterPendingRepair(Consumer<IInstanceConfig> config, String... repairCommandAndArgs) throws IOException
+ {
+ IInstanceInitializer initializer = ByteBuddyInjections.SkipActivation.install(1, 2, 3);
+
+ try (Cluster cluster = cluster(config, initializer))
+ {
+ // Make sure we fail on commit...
+ ByteBuddyInjections.SkipActivation.setup(cluster, COMMIT);
+
+ cluster.schemaChange("CREATE KEYSPACE " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3} AND replication_type='tracked';");
+ cluster.schemaChange("CREATE TABLE " + tableWithKeyspace(KEYSPACE) + " (pk BLOB PRIMARY KEY, v INT)");
+
+ cluster.get(1).executeInternal("INSERT INTO " + tableWithKeyspace(KEYSPACE) + " (pk, v) VALUES (?, 1)", KEY_100);
+
+ assertRows(cluster.get(1).executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE pk = ?", KEY_100), row(KEY_100, 1));
+ assertRows(cluster.get(2).executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE pk = ?", KEY_100));
+ assertRows(cluster.get(3).executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE pk = ?", KEY_100));
+
+ cluster.get(1).nodetoolResult(repairCommandAndArgs).asserts().failure().errorContains("Tracked transfer failed during COMMIT");
+
+ assertPendingActivation(cluster);
+ assertRows(cluster.get(1).executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE pk = ?", KEY_100), row(KEY_100, 1));
+ assertRows(cluster.get(2).executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE pk = ?", KEY_100));
+ assertRows(cluster.get(3).executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE pk = ?", KEY_100));
+
+ bounce(cluster);
+
+ assertPendingActivation(cluster);
+ assertRows(cluster.get(1).executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE pk = ?", KEY_100), row(KEY_100, 1));
+ assertRows(cluster.get(2).executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE pk = ?", KEY_100));
+ assertRows(cluster.get(3).executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE pk = ?", KEY_100));
+ }
+ }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedTransferTestBase.java b/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedTransferTestBase.java
new file mode 100644
index 0000000000..5b5a3b5a45
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedTransferTestBase.java
@@ -0,0 +1,546 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.tracking;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+import java.util.function.Predicate;
+
+import javax.annotation.Nullable;
+
+import org.assertj.core.api.Assertions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import net.bytebuddy.ByteBuddy;
+import net.bytebuddy.dynamic.loading.ClassLoadingStrategy;
+import net.bytebuddy.implementation.MethodDelegation;
+import net.bytebuddy.implementation.bind.annotation.SuperCall;
+import net.bytebuddy.implementation.bind.annotation.This;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.streaming.CassandraStreamReceiver;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.Feature;
+import org.apache.cassandra.distributed.api.IInstanceConfig;
+import org.apache.cassandra.distributed.api.IInstanceInitializer;
+import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.apache.cassandra.distributed.api.IIsolatedExecutor;
+import org.apache.cassandra.distributed.shared.AssertUtils;
+import org.apache.cassandra.distributed.shared.Uninterruptibles;
+import org.apache.cassandra.distributed.test.TestBaseImpl;
+import org.apache.cassandra.io.sstable.CQLSSTableWriter;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.replication.ActivationRequest;
+import org.apache.cassandra.replication.ImmutableCoordinatorLogOffsets;
+import org.apache.cassandra.replication.MutationSummary;
+import org.apache.cassandra.replication.MutationTrackingService;
+import org.apache.cassandra.replication.UnknownShardException;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+import static net.bytebuddy.matcher.ElementMatchers.named;
+import static net.bytebuddy.matcher.ElementMatchers.takesArguments;
+import static net.bytebuddy.matcher.ElementMatchers.takesNoArguments;
+import static org.junit.Assert.assertEquals;
+
+public abstract class TrackedTransferTestBase extends TestBaseImpl
+{
+ private static final Logger logger = LoggerFactory.getLogger(TrackedTransferTestBase.class);
+
+ protected static final Consumer<IInstanceConfig> CONFIG = cfg -> cfg.with(Feature.NETWORK)
+ .with(Feature.GOSSIP)
+ .set("mutation_tracking_enabled", true)
+ .set("write_request_timeout", "1000ms")
+ .set("autocompaction_on_startup_enabled", false)
+ .set("repair_request_timeout", "2s")
+ .set("stream_transfer_task_timeout", "10s");
+
+ protected static final Consumer<IInstanceConfig> ZCS_CONFIG = CONFIG.andThen(cfg -> cfg.set("stream_entire_sstables", true));
+ protected static final Consumer<IInstanceConfig> NON_ZCS_CONFIG = CONFIG.andThen(cfg -> cfg.set("stream_entire_sstables", false));
+
+ protected static final IIsolatedExecutor.SerializableConsumer<SSTableReader> TRANSFERS_EXIST = sstable -> {
+ Assertions.assertThat(sstable.getCoordinatorLogOffsets().transfers()).isNotEmpty();
+ Assertions.assertThat(sstable.isRepaired()).isFalse();
+ };
+ protected static final IIsolatedExecutor.SerializableConsumer<SSTableReader> TRANSFERS_EMPTY = sstable -> {
+ Assertions.assertThat(sstable.getCoordinatorLogOffsets().transfers()).isEmpty();
+ Assertions.assertThat(sstable.isRepaired()).isTrue();
+ };
+ protected static final IIsolatedExecutor.SerializableConsumer<SSTableReader> NOOP = sstable -> {};
+
+ protected static final int NODES = 3;
+
+ protected static final String TABLE = "tbl";
+
+ protected static final int IMPORT_PK = 1;
+ protected static final Token IMPORT_TOKEN = Murmur3Partitioner.instance.getToken(Int32Type.instance.decompose(IMPORT_PK));
+
+ // This should be aligned to a single shard: (min, -3074457345618258603]
+ protected final static long TOKEN_VALUE_100 = -4074457345618258601L;
+ protected final static Token TOKEN_100 = new Murmur3Partitioner.LongToken(TOKEN_VALUE_100);
+ protected final static ByteBuffer KEY_100 = Murmur3Partitioner.LongToken.keyForToken(TOKEN_100.getLongValue());
+
+ protected final static Range<Token> SHARD_ALIGNED_RANGE_1 = new Range<>(new Murmur3Partitioner.LongToken(TOKEN_VALUE_100 - 10), new Murmur3Partitioner.LongToken(TOKEN_VALUE_100 + 10));
+
+ // This should be aligned to a single shard: (-3074457345618258603,3074457345618258601]
+ protected final static long TOKEN_VALUE_200 = 1;
+ protected final static Token TOKEN_200 = new Murmur3Partitioner.LongToken(TOKEN_VALUE_200);
+ protected final static ByteBuffer KEY_200 = Murmur3Partitioner.LongToken.keyForToken(TOKEN_200.getLongValue());
+
+ protected final static long TOKEN_VALUE_201 = 2;
+ protected final static Token TOKEN_201 = new Murmur3Partitioner.LongToken(TOKEN_VALUE_201);
+ protected final static ByteBuffer KEY_201 = Murmur3Partitioner.LongToken.keyForToken(TOKEN_201.getLongValue());
+
+ protected final static Range<Token> SHARD_ALIGNED_RANGE_2 = new Range<>(new Murmur3Partitioner.LongToken(TOKEN_VALUE_200 - 10), new Murmur3Partitioner.LongToken(TOKEN_VALUE_200 + 10));
+
+ static
+ {
+ DecoratedKey reversed = Murmur3Partitioner.instance.decorateKey(KEY_100);
+ Assertions.assertThat(reversed.getToken()).isEqualTo(TOKEN_100);
+
+ reversed = Murmur3Partitioner.instance.decorateKey(KEY_200);
+ Assertions.assertThat(reversed.getToken()).isEqualTo(TOKEN_200);
+
+ reversed = Murmur3Partitioner.instance.decorateKey(KEY_201);
+ Assertions.assertThat(reversed.getToken()).isEqualTo(TOKEN_201);
+ }
+
+ protected static Cluster cluster() throws IOException
+ {
+ return cluster((cl, tg, instance, generation) -> {});
+ }
+
+ protected static Cluster cluster(Consumer<IInstanceConfig> config) throws IOException
+ {
+ return Cluster.build(NODES).withConfig(config).start();
+ }
+
+ protected static Cluster cluster(Consumer<IInstanceConfig> config, IInstanceInitializer initializer) throws IOException
+ {
+ return Cluster.build(NODES).withConfig(config).withInstanceInitializer(initializer).start();
+ }
+
+ protected static Cluster cluster(IInstanceInitializer initializer) throws IOException
+ {
+ return Cluster.build(NODES).withConfig(CONFIG).withInstanceInitializer(initializer).start();
+ }
+
+ protected static Cluster cluster(BiConsumer<ClassLoader, Integer> initializer) throws IOException
+ {
+ return Cluster.build(NODES).withConfig(CONFIG).withInstanceInitializer(initializer).start();
+ }
+
+ protected static void assertPendingActivation(Cluster cluster)
+ {
+ assertPendingActivation(cluster, KEYSPACE);
+ }
+
+ protected static void assertPendingActivation(Cluster cluster, String keyspace)
+ {
+ // Activation did not complete, files should still exist on all replicas
+ assertPendingDirs(cluster, keyspace, (File pendingUuidDir) -> {
+ Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isNotEmpty();
+ });
+
+ // No one has activated, so should not be present in any summary
+ assertSummary(cluster, keyspace, summary -> {
+ Assertions.assertThat(summary).satisfies(s -> {
+ assertEquals(0, s.reconciledIds());
+ assertEquals(0, s.unreconciledIds());
+ });
+ });
+ }
+
+ protected static void assertPendingDirs(Iterable<IInvokableInstance> validate, IIsolatedExecutor.SerializableConsumer<File> forPendingUuidDir)
+ {
+ assertPendingDirs(validate, KEYSPACE, forPendingUuidDir);
+ }
+
+ protected static void assertPendingDirs(Iterable<IInvokableInstance> validate, String keysapce, IIsolatedExecutor.SerializableConsumer<File> forPendingUuidDir)
+ {
+ for (IInvokableInstance instance : validate)
+ {
+ instance.runOnInstance(() -> {
+ Set<File> allPendingDirs = ColumnFamilyStore.getIfExists(keysapce, TABLE).getDirectories().getPendingLocations();
+ for (File pendingDir : allPendingDirs)
+ {
+ File[] pendingUuidDirs = pendingDir.listUnchecked(File::isDirectory);
+ for (File pendingUuidDir : pendingUuidDirs)
+ {
+ forPendingUuidDir.accept(pendingUuidDir);
+ }
+ }
+ });
+ }
+ }
+
+ protected static void assertSummary(Iterable<IInvokableInstance> validate, IIsolatedExecutor.SerializableConsumer<MutationSummary> onSummary)
+ {
+ assertSummary(validate, KEYSPACE, onSummary);
+ }
+
+ protected static void assertSummary(Iterable<IInvokableInstance> validate, String keyspace, IIsolatedExecutor.SerializableConsumer<MutationSummary> onSummary)
+ {
+ for (IInvokableInstance instance : validate)
+ {
+ instance.runOnInstance(() -> {
+ DecoratedKey key = DatabaseDescriptor.getPartitioner().decorateKey(ByteBufferUtil.bytes(1));
+ TableId tableId = ColumnFamilyStore.getIfExists(keyspace, TABLE).metadata().id;
+ MutationSummary summary;
+ try
+ {
+ summary = MutationTrackingService.instance.createSummaryForKey(key, tableId, false);
+ }
+ catch (UnknownShardException e)
+ {
+ summary = null;
+ }
+ onSummary.accept(summary);
+ });
+ }
+ }
+
+ protected static void bounce(Cluster cluster)
+ {
+ cluster.forEach(instance -> {
+ try
+ {
+ instance.shutdown().get();
+ }
+ catch (InterruptedException | ExecutionException e)
+ {
+ throw new RuntimeException(e);
+ }
+ instance.startup();
+ });
+ }
+
+ protected static String withKeyspace(String replaceIn, String keyspace)
+ {
+ return String.format(replaceIn, keyspace);
+ }
+
+ protected static String tableSchema(String keyspace)
+ {
+ return String.format("CREATE TABLE %s.%s (k int PRIMARY KEY, v int)", keyspace, TABLE);
+ }
+
+ protected static String tableWithKeyspace(String keyspace)
+ {
+ return String.format("%s.%s", keyspace, TABLE);
+ }
+
+ protected static void createSchema(Cluster cluster)
+ {
+ createSchema(cluster, KEYSPACE, NODES);
+ }
+
+ protected static void createSchema(Cluster cluster, String keyspace)
+ {
+ createSchema(cluster, keyspace, NODES);
+ }
+
+ protected static void createSchema(Cluster cluster, int rf)
+ {
+ createSchema(cluster, KEYSPACE, rf);
+ }
+
+ protected static void createSchema(Cluster cluster, String keyspace, int rf)
+ {
+ cluster.schemaChange(String.format(withKeyspace("CREATE KEYSPACE %s WITH replication = " +
+ "{'class': 'SimpleStrategy', 'replication_factor': " + rf + "} " +
+ "AND replication_type='tracked'", keyspace)));
+ cluster.schemaChange(tableSchema(keyspace));
+ }
+
+ protected static void doImport(Cluster cluster) throws IOException
+ {
+ doImport(cluster, KEYSPACE);
+ }
+
+ protected static void doImport(Cluster cluster, String keyspace) throws IOException
+ {
+ doImport(cluster, cluster.get(1), keyspace);
+ }
+
+ protected static void doImport(Cluster cluster, IInvokableInstance target) throws IOException
+ {
+ doImport(cluster, target, KEYSPACE);
+ }
+
+ protected static void doImport(Cluster cluster, IInvokableInstance target, String keyspace) throws IOException
+ {
+ doImport(cluster, target, failedDirs -> Assertions.assertThat(failedDirs).isEmpty(), keyspace, null);
+ }
+
+ protected static void doImport(Cluster cluster, IInvokableInstance target, String keyspace, @Nullable String createIndexCql) throws IOException
+ {
+ doImport(cluster, target, failedDirs -> Assertions.assertThat(failedDirs).isEmpty(), keyspace, createIndexCql);
+ }
+
+ protected static void doImport(Cluster cluster, IInvokableInstance target, Consumer<List<String>> onFailedDirs, String keyspace, @Nullable String createIndexCql) throws IOException
+ {
+ String file = Files.createTempDirectory(MutationTrackingTest.class.getSimpleName()).toString();
+
+ // Needs to run outside of instance executor because creates schema
+ CQLSSTableWriter.Builder builder = CQLSSTableWriter.builder()
+ .forTable(tableSchema(keyspace))
+ .inDirectory(file)
+ .using("INSERT INTO " + tableWithKeyspace(keyspace) + " (k, v) " + "VALUES (?, ?)");
+
+ if (createIndexCql != null)
+ {
+ builder.withIndexes(createIndexCql).withBuildIndexes(true);
+ }
+
+ try (CQLSSTableWriter writer = builder.build())
+ {
+ writer.addRow(IMPORT_PK, 1);
+ }
+
+ // empty
+ assertLocalSelect(cluster, keyspace, AssertUtils::assertRows);
+
+ List<String> failed = target.callOnInstance(() -> {
+ ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(keyspace, TABLE);
+ Set<String> paths = Set.of(file);
+ logger.info("Importing SSTables {}", paths);
+ return cfs.importNewSSTables(paths, true, true, true, true, true, true, true);
+ });
+
+ // Sleep for a while to make sure import completes
+ Uninterruptibles.sleepUninterruptibly(3, TimeUnit.SECONDS);
+ onFailedDirs.accept(failed);
+ }
+
+ protected static void assertLocalSelect(Iterable<IInvokableInstance> validate, IIsolatedExecutor.SerializableConsumer<Object[][]> onRows)
+ {
+ assertLocalSelect(validate, KEYSPACE, onRows);
+ }
+
+ protected static void assertLocalSelect(Iterable<IInvokableInstance> validate, String keyspace, IIsolatedExecutor.SerializableConsumer<Object[][]> onRows)
+ {
+ for (IInvokableInstance instance : validate)
+ {
+ {
+ Object[][] rows = instance.executeInternal(withKeyspace("SELECT * FROM %s." + TABLE + " WHERE k = 1", keyspace));
+ onRows.accept(rows);
+ }
+ {
+ Object[][] rows = instance.executeInternal(withKeyspace("SELECT * FROM %s." + TABLE, keyspace));
+ onRows.accept(rows);
+ }
+ }
+ }
+
+ protected static void assertCompaction(Cluster cluster,
+ Iterable<IInvokableInstance> validate,
+ IIsolatedExecutor.SerializableConsumer<SSTableReader> before,
+ IIsolatedExecutor.SerializableConsumer<SSTableReader> after)
+ {
+ assertCompaction(cluster, KEYSPACE, validate, before, after);
+ }
+ protected static void assertCompaction(Cluster cluster,
+ String keyspace,
+ Iterable<IInvokableInstance> validate,
+ IIsolatedExecutor.SerializableConsumer<SSTableReader> before,
+ IIsolatedExecutor.SerializableConsumer<SSTableReader> after)
+ {
+ for (IInvokableInstance instance : validate)
+ {
+ instance.runOnInstance(() -> {
+ ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(keyspace, TABLE);
+ for (SSTableReader sstable : cfs.getLiveSSTables())
+ {
+ logger.info("SSTable {} before compaction: {}", sstable.getFilename(), sstable.getCoordinatorLogOffsets());
+ before.accept(sstable);
+ }
+ });
+ }
+
+ // Activation ID must be persisted and broadcast across all peers in the cluster for any to mark as persisted + reconciled
+ cluster.forEach(i -> {
+ i.runOnInstance(() -> {
+ MutationTrackingService.instance.persistLogStateForTesting();
+ MutationTrackingService.instance.broadcastOffsetsForTesting();
+ });
+ });
+
+ // Broadcast is async, wait until completion
+ Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
+
+ for (IInvokableInstance instance : validate)
+ {
+ instance.runOnInstance(() -> {
+ ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(keyspace, TABLE);
+ logger.info("Triggering compaction on instance {}", cfs.metadata.keyspace);
+ CompactionManager.instance.performMaximal(cfs);
+
+ for (SSTableReader sstable : cfs.getLiveSSTables())
+ {
+ logger.info("SSTable {} after compaction: {}", sstable.getFilename(), sstable.getCoordinatorLogOffsets());
+ after.accept(sstable);
+ }
+ });
+ }
+ }
+
+ public static class ByteBuddyInjections
+ {
+ // Only skips direct transfer activation, not activation as part of read reconciliation
+ public static class SkipActivation
+ {
+ // null to not skip
+ public static volatile ActivationRequest.Phase phase;
+
+ public static volatile boolean throwOnActivation = false;
+
+ @SuppressWarnings("resource")
+ public static IInstanceInitializer install(int...nodes)
+ {
+ return (ClassLoader cl, ThreadGroup tg, int num, int generation) -> {
+ for (int node : nodes)
+ if (node == num)
+ new ByteBuddy().rebase(ActivationRequest.VerbHandler.class)
+ .method(named("doVerb"))
+ .intercept(MethodDelegation.to(ByteBuddyInjections.SkipActivation.class))
+ .make()
+ .load(cl, ClassLoadingStrategy.Default.INJECTION);
+ };
+ }
+
+ // Need to set phase in each instance's classloader, otherwise assignment won't be visible to injected method body
+ public static void setup(Cluster cluster, ActivationRequest.Phase phase)
+ {
+ setup(cluster, phase, false);
+ }
+
+ public static void setup(Cluster cluster, ActivationRequest.Phase phase, boolean throwOnActivation)
+ {
+ logger.debug("Setting up phase {}, throwOnActivation {}", phase, throwOnActivation);
+ cluster.forEach(instance -> instance.runOnInstance(() -> {
+ SkipActivation.phase = phase;
+ SkipActivation.throwOnActivation = throwOnActivation;
+ }));
+ }
+
+ @SuppressWarnings("unused")
+ public static void doVerb(Message<ActivationRequest> msg, @SuperCall Callable<?> zuper)
+ {
+ if (phase != null && msg.payload.phase == SkipActivation.phase)
+ {
+ if (throwOnActivation)
+ {
+ // Avoid spamming logs with retries
+ Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
+ throw new RuntimeException("Throwing on activation for test");
+ }
+ else
+ {
+ logger.info("Skipping activation for test: {}", msg.payload);
+ return;
+ }
+ }
+
+ logger.info("Test running activation as usual: {}", msg.payload);
+
+ try
+ {
+ zuper.call();
+ }
+ catch (Exception e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ // ImmutableCoordinatorLogOffsets.Builder.purgeTransfers(Predicate)
+ public static class SkipPurgeTransfers
+ {
+ @SuppressWarnings("resource")
+ public static IInstanceInitializer install()
+ {
+ return (ClassLoader cl, ThreadGroup tg, int num, int generation) -> {
+ new ByteBuddy().rebase(ImmutableCoordinatorLogOffsets.Builder.class)
+ .method(named("purgeTransfers").and(takesArguments(Predicate.class)))
+ .intercept(MethodDelegation.to(SkipPurgeTransfers.class))
+ .make()
+ .load(cl, ClassLoadingStrategy.Default.INJECTION);
+ };
+ }
+
+ @SuppressWarnings("unused")
+ public static void purgeTransfers()
+ {
+ logger.debug("Skipping purgeTransfers for test");
+ }
+ }
+
+ // CassandraStreamReceiver.finished
+ public static class FailIncomingStream
+ {
+ @SuppressWarnings("unused")
+ private static volatile boolean enabled = true;
+
+ @SuppressWarnings("resource")
+ public static IInstanceInitializer install(int... nodes)
+ {
+ return (ClassLoader cl, ThreadGroup tg, int num, int generation) -> {
+ for (int node : nodes)
+ if (node == num)
+ new ByteBuddy().rebase(CassandraStreamReceiver.class)
+ .method(named("finished").and(takesNoArguments()))
+ .intercept(MethodDelegation.to(FailIncomingStream.class))
+ .make()
+ .load(cl, ClassLoadingStrategy.Default.INJECTION);
+ };
+ }
+
+ @SuppressWarnings("unused")
+ public static void finished(@This CassandraStreamReceiver self)
+ {
+ throw new RuntimeException("Failing incoming stream for test");
+ }
+
+ public static void toggle(Cluster cluster, boolean enable)
+ {
+ enabled = enable;
+ cluster.forEach(instance -> instance.runOnInstance(() -> FailIncomingStream.enabled = enable));
+ }
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/repair/CompletableRemoteSyncTask.java b/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedZeroCopyRepairTransferTest.java
similarity index 71%
copy from src/java/org/apache/cassandra/repair/CompletableRemoteSyncTask.java
copy to test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedZeroCopyRepairTransferTest.java
index c4fe6c8b20..75c95b4e3f 100644
--- a/src/java/org/apache/cassandra/repair/CompletableRemoteSyncTask.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedZeroCopyRepairTransferTest.java
@@ -16,13 +16,17 @@
* limitations under the License.
*/
-package org.apache.cassandra.repair;
+package org.apache.cassandra.distributed.test.tracking;
-import java.util.List;
+import java.io.IOException;
-import org.apache.cassandra.streaming.SessionSummary;
+import org.junit.BeforeClass;
-public interface CompletableRemoteSyncTask
+public class TrackedZeroCopyRepairTransferTest extends TrackedRepairTransferSuccessTestBase
{
- void syncComplete(boolean success, List<SessionSummary> summaries);
+ @BeforeClass
+ public static void setup() throws IOException
+ {
+ cluster = cluster(ZCS_CONFIG);
+ }
}
diff --git a/test/unit/org/apache/cassandra/repair/LocalSyncTaskTest.java b/test/unit/org/apache/cassandra/repair/LocalSyncTaskTest.java
index c03fe2aac7..7594fe6b47 100644
--- a/test/unit/org/apache/cassandra/repair/LocalSyncTaskTest.java
+++ b/test/unit/org/apache/cassandra/repair/LocalSyncTaskTest.java
@@ -133,7 +133,7 @@ public class LocalSyncTaskTest extends AbstractRepairTest
TreeResponse r1 = new TreeResponse(local, tree1);
TreeResponse r2 = new TreeResponse(InetAddressAndPort.getByName("127.0.0.2"), tree2);
LocalSyncTask task = new LocalSyncTask(SharedContext.Global.instance, desc, r1.endpoint, r2.endpoint, MerkleTrees.difference(r1.trees, r2.trees),
- NO_PENDING_REPAIR, true, true, PreviewKind.NONE);
+ NO_PENDING_REPAIR, true, true, PreviewKind.NONE, null);
NettyStreamingConnectionFactory.MAX_CONNECT_ATTEMPTS = 1;
try
{
@@ -159,7 +159,7 @@ public class LocalSyncTaskTest extends AbstractRepairTest
TreeResponse r2 = new TreeResponse(PARTICIPANT2, createInitialTree(desc, DatabaseDescriptor.getPartitioner()));
LocalSyncTask task = new LocalSyncTask(SharedContext.Global.instance, desc, r1.endpoint, r2.endpoint, MerkleTrees.difference(r1.trees, r2.trees),
- NO_PENDING_REPAIR, true, true, PreviewKind.NONE);
+ NO_PENDING_REPAIR, true, true, PreviewKind.NONE, null);
StreamPlan plan = task.createStreamPlan();
assertEquals(NO_PENDING_REPAIR, plan.getPendingRepair());
diff --git a/test/unit/org/apache/cassandra/repair/RepairJobTest.java b/test/unit/org/apache/cassandra/repair/RepairJobTest.java
index b01b26bd83..69610934cd 100644
--- a/test/unit/org/apache/cassandra/repair/RepairJobTest.java
+++ b/test/unit/org/apache/cassandra/repair/RepairJobTest.java
@@ -284,7 +284,7 @@ public class RepairJobTest
// block syncComplete execution until test has verified session still retains the trees
CompletableFuture<?> future = new CompletableFuture<>();
session.registerSyncCompleteCallback(future::get);
- ListenableFuture<List<SyncStat>> syncResults = job.executeTasks(syncTasks);
+ ListenableFuture<List<SyncStat>> syncResults = job.executeTasks(SyncTasks.untracked(syncTasks));
// Immediately following execution the internal execution queue should still retain the trees
long sizeDuringExecution = ObjectSizes.measureDeep(session);
@@ -903,7 +903,7 @@ public class RepairJobTest
break;
case SYNC_REQ:
SyncRequest syncRequest = (SyncRequest) message.payload;
- session.syncComplete(sessionJobDesc, Message.builder(Verb.SYNC_RSP, new SyncResponse(sessionJobDesc, new SyncNodePair(syncRequest.src, syncRequest.dst), true, Collections.emptyList())).from(to).build());
+ session.syncComplete(sessionJobDesc, Message.builder(Verb.SYNC_RSP, new SyncResponse(sessionJobDesc, new SyncNodePair(syncRequest.src, syncRequest.dst), true, Collections.emptyList(), null, null)).from(to).build());
break;
default:
break;
diff --git a/test/unit/org/apache/cassandra/repair/StreamingRepairTaskTest.java b/test/unit/org/apache/cassandra/repair/StreamingRepairTaskTest.java
index 8aa39d06d9..7cbc26b803 100644
--- a/test/unit/org/apache/cassandra/repair/StreamingRepairTaskTest.java
+++ b/test/unit/org/apache/cassandra/repair/StreamingRepairTaskTest.java
@@ -68,8 +68,8 @@ public class StreamingRepairTaskTest extends AbstractRepairTest
ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance().getParentRepairSession(sessionID);
RepairJobDesc desc = new RepairJobDesc(sessionID, nextTimeUUID(), ks, tbl, prs.getRanges());
- SyncRequest request = new SyncRequest(desc, PARTICIPANT1, PARTICIPANT2, PARTICIPANT3, prs.getRanges(), PreviewKind.NONE, false);
- StreamingRepairTask task = new StreamingRepairTask(SharedContext.Global.instance, new SyncState(Clock.Global.clock(), desc, PARTICIPANT1, PARTICIPANT2, PARTICIPANT3), desc, request.initiator, request.src, request.dst, request.ranges, desc.sessionId, PreviewKind.NONE, false);
+ SyncRequest request = new SyncRequest(desc, PARTICIPANT1, PARTICIPANT2, PARTICIPANT3, prs.getRanges(), PreviewKind.NONE, false, null);
+ StreamingRepairTask task = new StreamingRepairTask(SharedContext.Global.instance, new SyncState(Clock.Global.clock(), desc, PARTICIPANT1, PARTICIPANT2, PARTICIPANT3, null), desc, request.initiator, request.src, request.dst, request.ranges, desc.sessionId, PreviewKind.NONE, false, null);
StreamPlan plan = task.createStreamPlan(request.dst);
Assert.assertFalse(plan.getFlushBeforeTransfer());
@@ -81,8 +81,9 @@ public class StreamingRepairTaskTest extends AbstractRepairTest
TimeUUID sessionID = registerSession(cfs, false, true);
ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance().getParentRepairSession(sessionID);
RepairJobDesc desc = new RepairJobDesc(sessionID, nextTimeUUID(), ks, tbl, prs.getRanges());
- SyncRequest request = new SyncRequest(desc, PARTICIPANT1, PARTICIPANT2, PARTICIPANT3, prs.getRanges(), PreviewKind.NONE, false);
- StreamingRepairTask task = new StreamingRepairTask(SharedContext.Global.instance, new SyncState(Clock.Global.clock(), desc, PARTICIPANT1, PARTICIPANT2, PARTICIPANT3), desc, request.initiator, request.src, request.dst, request.ranges, null, PreviewKind.NONE, false);
+
+ SyncRequest request = new SyncRequest(desc, PARTICIPANT1, PARTICIPANT2, PARTICIPANT3, prs.getRanges(), PreviewKind.NONE, false, null);
+ StreamingRepairTask task = new StreamingRepairTask(SharedContext.Global.instance, new SyncState(Clock.Global.clock(), desc, PARTICIPANT1, PARTICIPANT2, PARTICIPANT3, null), desc, request.initiator, request.src, request.dst, request.ranges, null, PreviewKind.NONE, false, null);
StreamPlan plan = task.createStreamPlan(request.dst);
Assert.assertTrue(plan.getFlushBeforeTransfer());
diff --git a/test/unit/org/apache/cassandra/repair/SymmetricRemoteSyncTaskTest.java b/test/unit/org/apache/cassandra/repair/SymmetricRemoteSyncTaskTest.java
index 37d4e09e78..ff840391c2 100644
--- a/test/unit/org/apache/cassandra/repair/SymmetricRemoteSyncTaskTest.java
+++ b/test/unit/org/apache/cassandra/repair/SymmetricRemoteSyncTaskTest.java
@@ -41,7 +41,7 @@ public class SymmetricRemoteSyncTaskTest extends AbstractRepairTest
{
public InstrumentedSymmetricRemoteSyncTask(InetAddressAndPort e1, InetAddressAndPort e2)
{
- super(SharedContext.Global.instance, DESC, e1, e2, RANGE_LIST, PreviewKind.NONE);
+ super(SharedContext.Global.instance, DESC, e1, e2, RANGE_LIST, PreviewKind.NONE, null);
}
RepairMessage sentMessage = null;
diff --git a/test/unit/org/apache/cassandra/repair/messages/RepairMessageSerializationsTest.java b/test/unit/org/apache/cassandra/repair/messages/RepairMessageSerializationsTest.java
index f15b2a6182..22f7cf8810 100644
--- a/test/unit/org/apache/cassandra/repair/messages/RepairMessageSerializationsTest.java
+++ b/test/unit/org/apache/cassandra/repair/messages/RepairMessageSerializationsTest.java
@@ -169,7 +169,8 @@ public class RepairMessageSerializationsTest extends CassandraTestBase
InetAddressAndPort src = InetAddressAndPort.getByName("127.0.0.2");
InetAddressAndPort dst = InetAddressAndPort.getByName("127.0.0.3");
- SyncRequest msg = new SyncRequest(buildRepairJobDesc(), initiator, src, dst, buildTokenRanges(), PreviewKind.NONE, false);
+ // TODO: Do we want to test with a transfer ID?
+ SyncRequest msg = new SyncRequest(buildRepairJobDesc(), initiator, src, dst, buildTokenRanges(), PreviewKind.NONE, false, null);
serializeRoundTrip(msg, SyncRequest.serializer);
}
@@ -183,7 +184,7 @@ public class RepairMessageSerializationsTest extends CassandraTestBase
Lists.newArrayList(new StreamSummary(TableId.fromUUID(UUID.randomUUID()), emptyList(), 5, 100)),
Lists.newArrayList(new StreamSummary(TableId.fromUUID(UUID.randomUUID()), emptyList(), 500, 10))
));
- SyncResponse msg = new SyncResponse(buildRepairJobDesc(), new SyncNodePair(src, dst), true, summaries);
+ SyncResponse msg = new SyncResponse(buildRepairJobDesc(), new SyncNodePair(src, dst), true, summaries, null, null);
serializeRoundTrip(msg, SyncResponse.serializer);
}
diff --git a/test/unit/org/apache/cassandra/replication/TransferActivationSerializationTest.java b/test/unit/org/apache/cassandra/replication/ActivationRequestSerializationTest.java
similarity index 50%
rename from test/unit/org/apache/cassandra/replication/TransferActivationSerializationTest.java
rename to test/unit/org/apache/cassandra/replication/ActivationRequestSerializationTest.java
index 57a51f87b8..424b57ddb9 100644
--- a/test/unit/org/apache/cassandra/replication/TransferActivationSerializationTest.java
+++ b/test/unit/org/apache/cassandra/replication/ActivationRequestSerializationTest.java
@@ -22,16 +22,24 @@ import java.io.IOException;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.apache.cassandra.CassandraTestBase;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.dht.ByteOrderedPartitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper;
import org.apache.cassandra.io.Serializers;
import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.streaming.StreamOperation;
import org.apache.cassandra.tcm.membership.NodeId;
+import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.TimeUUID;
import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
-public class TransferActivationSerializationTest
+public class ActivationRequestSerializationTest extends CassandraTestBase
{
private static final int VERSION = MessagingService.current_version;
@@ -39,37 +47,44 @@ public class TransferActivationSerializationTest
public static void setUpClass()
{
DatabaseDescriptor.daemonInitialization();
+ ClusterMetadataTestHelper.setInstanceForTest();
}
@Test
public void testRoundtripPreparePhase() throws IOException
{
+ Pair<InetAddressAndPort, InetAddressAndPort> pair = Pair.create(InetAddressAndPort.getLocalHost(), InetAddressAndPort.getLocalHost());
TimeUUID planId = nextTimeUUID();
- MutationId transferId = new MutationId(1L, 100, 200);
+ ShortMutationId transferId = new ShortMutationId(1L, 100);
NodeId coordinatorId = new NodeId(1);
- TransferActivation.Phase phase = TransferActivation.Phase.PREPARE;
+ ActivationRequest.Phase phase = ActivationRequest.Phase.PREPARE;
+ String keyspace = "test_ks";
+ Range<Token> range = new Range<>(new ByteOrderedPartitioner.BytesToken("key1".getBytes()), new ByteOrderedPartitioner.BytesToken("key100".getBytes()));
- TransferActivation activation = new TransferActivation(planId, transferId, coordinatorId, phase);
+ ActivationRequest activation = new ActivationRequest(StreamOperation.IMPORT, pair, phase, transferId, coordinatorId, range, keyspace, planId);
try (DataOutputBuffer output = new DataOutputBuffer())
{
- Serializers.testSerde(output, TransferActivation.serializer, activation, VERSION);
+ Serializers.testSerde(output, ActivationRequest.serializer, activation, VERSION);
}
}
@Test
public void testRoundtripCommitPhase() throws IOException
{
+ Pair<InetAddressAndPort, InetAddressAndPort> pair = Pair.create(InetAddressAndPort.getLocalHost(), InetAddressAndPort.getLocalHost());
TimeUUID planId = nextTimeUUID();
- MutationId transferId = new MutationId(2L, 300, 400);
+ ShortMutationId transferId = new ShortMutationId(2L, 300);
NodeId coordinatorId = new NodeId(2);
- TransferActivation.Phase phase = TransferActivation.Phase.COMMIT;
+ ActivationRequest.Phase phase = ActivationRequest.Phase.COMMIT;
+ String keyspace = "test_ks";
+ Range<Token> range = new Range<>(new ByteOrderedPartitioner.BytesToken("key1".getBytes()), new ByteOrderedPartitioner.BytesToken("key100".getBytes()));
- TransferActivation activation = new TransferActivation(planId, transferId, coordinatorId, phase);
+ ActivationRequest activation = new ActivationRequest(StreamOperation.IMPORT, pair, phase, transferId, coordinatorId, range, keyspace, planId);
try (DataOutputBuffer output = new DataOutputBuffer())
{
- Serializers.testSerde(output, TransferActivation.serializer, activation, VERSION);
+ Serializers.testSerde(output, ActivationRequest.serializer, activation, VERSION);
}
}
}
diff --git a/test/unit/org/apache/cassandra/replication/ActivationResponseSerializationTest.java b/test/unit/org/apache/cassandra/replication/ActivationResponseSerializationTest.java
new file mode 100644
index 0000000000..14cc384e7f
--- /dev/null
+++ b/test/unit/org/apache/cassandra/replication/ActivationResponseSerializationTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.replication;
+
+import java.io.IOException;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.io.Serializers;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.utils.Pair;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class ActivationResponseSerializationTest
+{
+ private static final int VERSION = MessagingService.current_version;
+
+ @BeforeClass
+ public static void setUpClass()
+ {
+ DatabaseDescriptor.daemonInitialization();
+ }
+
+ @Test
+ public void testRoundtripSuccess() throws IOException
+ {
+ Pair<InetAddressAndPort, InetAddressAndPort> pair = Pair.create(InetAddressAndPort.getLocalHost(), InetAddressAndPort.getLocalHost());
+ ActivationResponse response = new ActivationResponse(pair);
+
+ try (DataOutputBuffer output = new DataOutputBuffer())
+ {
+ Serializers.testSerde(output, ActivationResponse.serializer, response, VERSION);
+ }
+ }
+
+ @Test
+ public void testRoundtripDifferentAddresses() throws IOException
+ {
+ InetAddressAndPort coordinator = InetAddressAndPort.getByName("127.0.0.1");
+ InetAddressAndPort peer = InetAddressAndPort.getByName("127.0.0.2");
+ Pair<InetAddressAndPort, InetAddressAndPort> pair = Pair.create(coordinator, peer);
+ ActivationResponse response = new ActivationResponse(pair);
+
+ try (DataOutputBuffer output = new DataOutputBuffer())
+ {
+ Serializers.testSerde(output, ActivationResponse.serializer, response, VERSION);
+ }
+ }
+
+ @Test
+ public void testEqualsAndHashCode() throws Exception
+ {
+ InetAddressAndPort addr1 = InetAddressAndPort.getByName("127.0.0.1");
+ InetAddressAndPort addr2 = InetAddressAndPort.getByName("127.0.0.2");
+ Pair<InetAddressAndPort, InetAddressAndPort> pair1 = Pair.create(addr1, addr2);
+ Pair<InetAddressAndPort, InetAddressAndPort> pair2 = Pair.create(addr1, addr2);
+
+ ActivationResponse resp1 = new ActivationResponse(pair1);
+ ActivationResponse resp2 = new ActivationResponse(pair2);
+
+ assertThat(resp1).isEqualTo(resp2);
+ assertThat(resp1.hashCode()).isEqualTo(resp2.hashCode());
+ }
+}
diff --git a/test/unit/org/apache/cassandra/replication/LocalTransfersTest.java b/test/unit/org/apache/cassandra/replication/LocalTransfersTest.java
deleted file mode 100644
index 465f280c49..0000000000
--- a/test/unit/org/apache/cassandra/replication/LocalTransfersTest.java
+++ /dev/null
@@ -1,344 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.replication;
-
-import java.util.Collection;
-import java.util.Collections;
-
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.dht.Murmur3Partitioner;
-import org.apache.cassandra.dht.Range;
-import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.utils.TimeUUID;
-import org.assertj.core.api.Assertions;
-
-import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.COMMITTED;
-import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.COMMITTING;
-import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.PREPARE_FAILED;
-import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.PREPARING;
-import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.STREAM_COMPLETE;
-import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.STREAM_FAILED;
-import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.STREAM_NOOP;
-import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
-import static org.mockito.Mockito.mock;
-
-public class LocalTransfersTest
-{
- private LocalTransfers localTransfers;
- private ShortMutationId transferId;
- private TimeUUID planId;
-
- @BeforeClass
- public static void setUpClass()
- {
- DatabaseDescriptor.daemonInitialization();
- DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
- }
-
- @Before
- public void setUp() throws Exception
- {
- localTransfers = new LocalTransfers();
- transferId = new ShortMutationId(1, 100);
- planId = nextTimeUUID();
- }
-
- private CoordinatedTransfer coordinatedTransfer(ShortMutationId transferId)
- {
- return coordinatedTransfer(transferId, new Range<>(tk(0), tk(1000)));
- }
-
- private CoordinatedTransfer coordinatedTransfer(ShortMutationId transferId, Range<Token> range)
- {
- MutationId mutationId = transferId != null
- ? new MutationId(transferId.logId(), transferId.offset(), (int) System.currentTimeMillis())
- : null;
- return new CoordinatedTransfer(range, mutationId);
- }
-
- private PendingLocalTransfer pendingTransfer(TimeUUID planId)
- {
- SSTableReader mockSSTable = mock(SSTableReader.class);
- Collection<SSTableReader> sstables = Collections.singletonList(mockSSTable);
-
- return new PendingLocalTransfer(planId, sstables);
- }
-
- private static Token tk(long token)
- {
- return new Murmur3Partitioner.LongToken(token);
- }
-
- @Test
- public void testSaveCoordinatedTransfer()
- {
- CoordinatedTransfer transfer = coordinatedTransfer(transferId);
-
- localTransfers.save(transfer);
- localTransfers.activating(transfer);
-
- CoordinatedTransfer loaded = localTransfers.getActivatedTransfer(transferId);
- Assertions.assertThat(loaded).isEqualTo(transfer);
-
- assertThatThrownBy(() -> localTransfers.save(transfer))
- .isInstanceOf(IllegalStateException.class);
- }
-
- @Test
- public void testActivatingTransfer()
- {
- CoordinatedTransfer transfer = coordinatedTransfer(transferId);
-
- localTransfers.save(transfer);
- localTransfers.activating(transfer);
-
- CoordinatedTransfer retrieved = localTransfers.getActivatedTransfer(transferId);
- assertThat(retrieved).isEqualTo(transfer);
- }
-
- @Test
- public void testReceivedTransfer()
- {
- TimeUUID planId = nextTimeUUID();
- PendingLocalTransfer transfer = pendingTransfer(planId);
- localTransfers.received(transfer);
- PendingLocalTransfer retrieved = localTransfers.getPendingTransfer(planId);
- assertThat(retrieved).isEqualTo(transfer);
- }
-
- @Test
- public void testReceivedEmptyTransferThrows()
- {
- assertThatThrownBy(() -> new PendingLocalTransfer(planId, Collections.emptyList()))
- .isInstanceOf(IllegalStateException.class);
- }
-
- @Test
- public void testGetPendingTransferNotFound()
- {
- PendingLocalTransfer retrieved = localTransfers.getPendingTransfer(planId);
- assertThat(retrieved).isNull();
- }
-
- @Test
- public void testGetActivatedTransferNotFound()
- {
- CoordinatedTransfer retrieved = localTransfers.getActivatedTransfer(transferId);
- assertThat(retrieved).isNull();
- }
-
- @Test
- public void testPurgingTransferNotStarted()
- {
- CoordinatedTransfer transfer = coordinatedTransfer(transferId);
-
- // All streams in INIT state - should NOT be purgeable (stream hasn't started yet)
- CoordinatedTransfer.SingleTransferResult result = CoordinatedTransfer.SingleTransferResult.Init();
- transfer.streamResults.put(mock(InetAddressAndPort.class), result);
-
- Assertions.assertThat(localTransfers.purger.test(transfer)).isFalse();
- }
-
- @Test
- public void testPurgingTransferAllStreamsComplete()
- {
- CoordinatedTransfer transfer = coordinatedTransfer(transferId);
-
- // All streams in STREAM_COMPLETE state - should NOT be purgeable (no failures)
- CoordinatedTransfer.SingleTransferResult result1 = CoordinatedTransfer.SingleTransferResult.StreamComplete(nextTimeUUID());
- CoordinatedTransfer.SingleTransferResult result2 = CoordinatedTransfer.SingleTransferResult.StreamComplete(nextTimeUUID());
-
- transfer.streamResults.put(mock(InetAddressAndPort.class), result1);
- transfer.streamResults.put(mock(InetAddressAndPort.class), result2);
-
- Assertions.assertThat(localTransfers.purger.test(transfer)).isFalse();
- }
-
- @Test
- public void testPurgingTransferPrepareFailed()
- {
- CoordinatedTransfer transfer = coordinatedTransfer(transferId);
-
- CoordinatedTransfer.SingleTransferResult result1 = new CoordinatedTransfer.SingleTransferResult(PREPARE_FAILED, planId);
- CoordinatedTransfer.SingleTransferResult result2 = new CoordinatedTransfer.SingleTransferResult(PREPARING, planId);
-
- transfer.streamResults.put(mock(InetAddressAndPort.class), result1);
- transfer.streamResults.put(mock(InetAddressAndPort.class), result2);
-
- Assertions.assertThat(localTransfers.purger.test(transfer)).isTrue();
- }
-
- @Test
- public void testPurgingTransferAllActivationCommitted()
- {
- CoordinatedTransfer transfer = coordinatedTransfer(transferId);
-
- // All streams in ACTIVATE_COMMITTED state - should be purgeable (allComplete = true)
- CoordinatedTransfer.SingleTransferResult result1 = new CoordinatedTransfer.SingleTransferResult(COMMITTED, planId);
- CoordinatedTransfer.SingleTransferResult result2 = new CoordinatedTransfer.SingleTransferResult(COMMITTED, planId);
-
- transfer.streamResults.put(mock(InetAddressAndPort.class), result1);
- transfer.streamResults.put(mock(InetAddressAndPort.class), result2);
-
- Assertions.assertThat(localTransfers.purger.test(transfer)).isTrue();
- }
-
- @Test
- public void testPurgingTransferMixedCommittedAndNoop()
- {
- CoordinatedTransfer transfer = coordinatedTransfer(transferId);
-
- // Mix of ACTIVATE_COMMITTED and STREAM_NOOP - should be purgeable (allComplete = true)
- CoordinatedTransfer.SingleTransferResult result1 = new CoordinatedTransfer.SingleTransferResult(COMMITTED, planId);
- CoordinatedTransfer.SingleTransferResult result2 = new CoordinatedTransfer.SingleTransferResult(STREAM_NOOP, null);
-
- transfer.streamResults.put(mock(InetAddressAndPort.class), result1);
- transfer.streamResults.put(mock(InetAddressAndPort.class), result2);
-
- Assertions.assertThat(localTransfers.purger.test(transfer)).isTrue();
- }
-
- @Test
- public void testPurgingTransferActivationPartialCommitted()
- {
- CoordinatedTransfer transfer = coordinatedTransfer(transferId);
-
- // One stream in ACTIVATE_PREPARING - should NOT be purgeable
- CoordinatedTransfer.SingleTransferResult result1 = new CoordinatedTransfer.SingleTransferResult(PREPARING, planId);
- CoordinatedTransfer.SingleTransferResult result2 = new CoordinatedTransfer.SingleTransferResult(COMMITTING, planId);
-
- transfer.streamResults.put(mock(InetAddressAndPort.class), result1);
- transfer.streamResults.put(mock(InetAddressAndPort.class), result2);
-
- Assertions.assertThat(localTransfers.purger.test(transfer)).isFalse();
- }
-
- @Test
- public void testPurgingTransferAllStreamsFailed()
- {
- CoordinatedTransfer transfer = coordinatedTransfer(transferId);
-
- // All streams in STREAM_FAILED state - should be purgeable (noneActivated = true)
- CoordinatedTransfer.SingleTransferResult result1 = new CoordinatedTransfer.SingleTransferResult(STREAM_FAILED, planId);
- CoordinatedTransfer.SingleTransferResult result2 = new CoordinatedTransfer.SingleTransferResult(STREAM_FAILED, planId);
-
- transfer.streamResults.put(mock(InetAddressAndPort.class), result1);
- transfer.streamResults.put(mock(InetAddressAndPort.class), result2);
-
- Assertions.assertThat(localTransfers.purger.test(transfer)).isTrue();
- }
-
- @Test
- public void testPurgingTransferMixedInitAndFailed()
- {
- CoordinatedTransfer transfer = coordinatedTransfer(transferId);
-
- // Mix of INIT and STREAM_FAILED - should be purgeable (has failure, none activated)
- CoordinatedTransfer.SingleTransferResult result1 = CoordinatedTransfer.SingleTransferResult.Init();
- CoordinatedTransfer.SingleTransferResult result2 = CoordinatedTransfer.SingleTransferResult.Init().streamFailed(nextTimeUUID());
-
- transfer.streamResults.put(mock(InetAddressAndPort.class), result1);
- transfer.streamResults.put(mock(InetAddressAndPort.class), result2);
-
- Assertions.assertThat(localTransfers.purger.test(transfer)).isTrue();
- }
-
- @Test
- public void testPurgingTransferMixedCompleteAndFailed()
- {
- CoordinatedTransfer transfer = coordinatedTransfer(transferId);
-
- // Mix of STREAM_COMPLETE and STREAM_FAILED - should be purgeable (has failure, none activated)
- CoordinatedTransfer.SingleTransferResult result1 = CoordinatedTransfer.SingleTransferResult.StreamComplete(nextTimeUUID());
- CoordinatedTransfer.SingleTransferResult result2 = CoordinatedTransfer.SingleTransferResult.Init().streamFailed(nextTimeUUID());
-
- transfer.streamResults.put(mock(InetAddressAndPort.class), result1);
- transfer.streamResults.put(mock(InetAddressAndPort.class), result2);
-
- Assertions.assertThat(localTransfers.purger.test(transfer)).isTrue();
- }
-
- @Test
- public void testPurgingTransferMixedStreamingCompleteAndPreparing()
- {
- CoordinatedTransfer transfer = coordinatedTransfer(transferId);
-
- // Mix of STREAM_COMPLETE and ACTIVATE_PREPARING - should NOT be purgeable
- // (noneActivated = false because of ACTIVATE_PREPARING, allComplete = false)
- CoordinatedTransfer.SingleTransferResult result1 = new CoordinatedTransfer.SingleTransferResult(STREAM_COMPLETE, planId);
- CoordinatedTransfer.SingleTransferResult result2 = new CoordinatedTransfer.SingleTransferResult(PREPARING, planId);
-
- transfer.streamResults.put(mock(InetAddressAndPort.class), result1);
- transfer.streamResults.put(mock(InetAddressAndPort.class), result2);
-
- Assertions.assertThat(localTransfers.purger.test(transfer)).isFalse();
- }
-
- @Test
- public void testPurgingTransferMixedCommittingCommitted()
- {
- CoordinatedTransfer transfer = coordinatedTransfer(transferId);
-
- CoordinatedTransfer.SingleTransferResult result1 = new CoordinatedTransfer.SingleTransferResult(COMMITTING, planId);
- CoordinatedTransfer.SingleTransferResult result2 = new CoordinatedTransfer.SingleTransferResult(COMMITTED, planId);
-
- transfer.streamResults.put(mock(InetAddressAndPort.class), result1);
- transfer.streamResults.put(mock(InetAddressAndPort.class), result2);
-
- Assertions.assertThat(localTransfers.purger.test(transfer)).isFalse();
- }
-
- @Test
- public void testPurgingTransferWithNullTransferId()
- {
- CoordinatedTransfer transfer = coordinatedTransfer(null);
-
- // All streams complete but transferId is null - should NOT be purgeable
- CoordinatedTransfer.SingleTransferResult result1 = new CoordinatedTransfer.SingleTransferResult(STREAM_COMPLETE, null);
- CoordinatedTransfer.SingleTransferResult result2 = new CoordinatedTransfer.SingleTransferResult(STREAM_COMPLETE, null);
-
- transfer.streamResults.put(mock(InetAddressAndPort.class), result1);
- transfer.streamResults.put(mock(InetAddressAndPort.class), result2);
-
- // allComplete = true, but transferId is null, so should not purge
- Assertions.assertThat(localTransfers.purger.test(transfer)).isFalse();
- }
-
- @Test
- public void testPurgingTransferNoopOnly()
- {
- CoordinatedTransfer transfer = coordinatedTransfer(transferId);
-
- // All streams in STREAM_NOOP - should be purgeable (both noneActivated and allComplete are true)
- CoordinatedTransfer.SingleTransferResult result1 = CoordinatedTransfer.SingleTransferResult.Noop();
- CoordinatedTransfer.SingleTransferResult result2 = CoordinatedTransfer.SingleTransferResult.Noop();
-
- transfer.streamResults.put(mock(InetAddressAndPort.class), result1);
- transfer.streamResults.put(mock(InetAddressAndPort.class), result2);
-
- Assertions.assertThat(localTransfers.purger.test(transfer)).isTrue();
- }
-}
diff --git a/test/unit/org/apache/cassandra/replication/MutationTrackingServiceTest.java b/test/unit/org/apache/cassandra/replication/MutationTrackingServiceTest.java
new file mode 100644
index 0000000000..7d53a3f29b
--- /dev/null
+++ b/test/unit/org/apache/cassandra/replication/MutationTrackingServiceTest.java
@@ -0,0 +1,265 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.replication;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.dht.ByteOrderedPartitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.repair.RepairJobDesc;
+import org.apache.cassandra.repair.SharedContext;
+import org.apache.cassandra.repair.SyncTask;
+import org.apache.cassandra.repair.SyncTasks;
+import org.apache.cassandra.repair.SymmetricRemoteSyncTask;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.ReplicationType;
+import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.TimeUUID;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+public class MutationTrackingServiceTest
+{
+ private static final String TEST_KEYSPACE = "test_ks";
+ private static final String TEST_TABLE = "test_table";
+ private static final InetAddressAndPort LOCAL = InetAddressAndPort.getByNameUnchecked("127.0.0.1");
+ private static final InetAddressAndPort REMOTE = InetAddressAndPort.getByNameUnchecked("127.0.0.2");
+
+ @BeforeClass
+ public static void setup() throws IOException
+ {
+ DatabaseDescriptor.daemonInitialization();
+ SchemaLoader.prepareServer();
+ SchemaLoader.createKeyspace(TEST_KEYSPACE, KeyspaceParams.simple(1, ReplicationType.tracked), SchemaLoader.standardCFMD(TEST_KEYSPACE, TEST_TABLE));
+ }
+
+ @Test
+ public void testAlignToShardBoundariesSingleTaskWithinSingleShard()
+ {
+ MutationTrackingService service = MutationTrackingService.TestAccess.create();
+
+ // Create a single shard covering a-z
+ Set<Range<Token>> shardRanges = new HashSet<>();
+ shardRanges.add(range("a", "z"));
+ MutationTrackingService.KeyspaceShards shards =
+ MutationTrackingService.TestAccess.createTestKeyspaceShards(TEST_KEYSPACE, shardRanges);
+ MutationTrackingService.TestAccess.setKeyspaceShards(service, TEST_KEYSPACE, shards);
+
+ // Input task completely within the shard
+ List<SyncTask> inputTasks = Collections.singletonList(createSyncTask(range("d", "m")));
+
+ Keyspace keyspace = Keyspace.open(TEST_KEYSPACE);
+ SyncTasks result = service.alignToShardBoundaries(keyspace, inputTasks);
+
+ // Should have one shard entry
+ AtomicInteger entries = new AtomicInteger(0);
+ result.apply((shardedTask) -> entries.incrementAndGet());
+ assertEquals(1, entries.get());
+
+ // Get all tasks from the result
+ List<SyncTask> allTasks = new ArrayList<>();
+ result.apply((shardedTask) -> allTasks.add(shardedTask.task));
+
+ assertEquals(1, allTasks.size());
+
+ SyncTask resultTask = allTasks.get(0);
+ assertEquals(1, resultTask.rangesToSync.size());
+ assertTrue(resultTask.rangesToSync.contains(range("d", "m")));
+
+ // Should have a transfer ID assigned
+ assertNotNull(resultTask.getTransferId());
+ }
+
+ @Test
+ public void testAlignToShardBoundariesSingleTaskSpanningMultipleShards()
+ {
+ MutationTrackingService service = MutationTrackingService.TestAccess.create();
+
+ // Create two shards
+ Set<Range<Token>> shardRanges = new HashSet<>();
+ shardRanges.add(range("a", "m"));
+ shardRanges.add(range("m", "z"));
+ MutationTrackingService.KeyspaceShards shards =
+ MutationTrackingService.TestAccess.createTestKeyspaceShards(TEST_KEYSPACE, shardRanges);
+ MutationTrackingService.TestAccess.setKeyspaceShards(service, TEST_KEYSPACE, shards);
+
+ // Input task spans both shards
+ List<SyncTask> inputTasks = Collections.singletonList(createSyncTask(range("d", "s")));
+
+ Keyspace keyspace = Keyspace.open(TEST_KEYSPACE);
+ SyncTasks result = service.alignToShardBoundaries(keyspace, inputTasks);
+
+ // Should be split into two shard entries
+ AtomicInteger entries = new AtomicInteger(0);
+ result.apply((shardedTask) -> entries.incrementAndGet());
+ assertEquals(2, entries.get());
+
+ // Collect all ranges from all tasks
+ Set<Range<Token>> allRanges = new HashSet<>();
+ result.apply((shardedTask) -> allRanges.addAll(shardedTask.task.rangesToSync));
+
+ // Should contain the two split pieces
+ assertEquals(2, allRanges.size());
+ assertTrue(allRanges.contains(range("d", "m")));
+ assertTrue(allRanges.contains(range("m", "s")));
+ }
+
+ @Test
+ public void testAlignToShardBoundariesMultipleTasksAcrossMultipleShards()
+ {
+ MutationTrackingService service = MutationTrackingService.TestAccess.create();
+
+ // Create three shards
+ Set<Range<Token>> shardRanges = new HashSet<>();
+ shardRanges.add(range("a", "h"));
+ shardRanges.add(range("h", "p"));
+ shardRanges.add(range("p", "z"));
+ MutationTrackingService.KeyspaceShards shards =
+ MutationTrackingService.TestAccess.createTestKeyspaceShards(TEST_KEYSPACE, shardRanges);
+ MutationTrackingService.TestAccess.setKeyspaceShards(service, TEST_KEYSPACE, shards);
+
+ // Multiple tasks, some spanning shards
+ List<SyncTask> inputTasks = Arrays.asList(
+ createSyncTask(range("b", "e")), // Within shard 1
+ createSyncTask(range("f", "j")), // Spans shard 1 and 2
+ createSyncTask(range("q", "s")) // Within shard 3
+ );
+
+ Keyspace keyspace = Keyspace.open(TEST_KEYSPACE);
+ SyncTasks result = service.alignToShardBoundaries(keyspace, inputTasks);
+
+ // Should have 4 entries (one per sync task)
+ AtomicInteger entries = new AtomicInteger(0);
+ result.apply((shardedTask) -> entries.incrementAndGet());
+ assertEquals(4, entries.get());
+
+ // Collect all ranges from all tasks
+ Set<Range<Token>> allRanges = new HashSet<>();
+ result.apply((shardedTask) -> allRanges.addAll(shardedTask.task.rangesToSync));
+
+ // Should have split ranges: b-e, f-h (shard 1), h-j (shard 2), q-s (shard 3)
+ assertTrue(allRanges.contains(range("b", "e")));
+ assertTrue(allRanges.contains(range("f", "h")));
+ assertTrue(allRanges.contains(range("h", "j")));
+ assertTrue(allRanges.contains(range("q", "s")));
+ }
+
+ @Test
+ public void testAlignToShardBoundariesTaskWithMultipleRangesSpanningShards()
+ {
+ MutationTrackingService service = MutationTrackingService.TestAccess.create();
+
+ // Create two shards
+ Set<Range<Token>> shardRanges = new HashSet<>();
+ shardRanges.add(range("a", "m"));
+ shardRanges.add(range("m", "z"));
+ MutationTrackingService.KeyspaceShards shards =
+ MutationTrackingService.TestAccess.createTestKeyspaceShards(TEST_KEYSPACE, shardRanges);
+ MutationTrackingService.TestAccess.setKeyspaceShards(service, TEST_KEYSPACE, shards);
+
+ // Single task with multiple ranges spanning both shards
+ List<Range<Token>> ranges = Arrays.asList(
+ range("b", "e"), // Shard 1
+ range("f", "p") // Spans both shards
+ );
+
+ SyncTask task = createSyncTask(ranges, LOCAL, REMOTE);
+ List<SyncTask> inputTasks = Collections.singletonList(task);
+
+ Keyspace keyspace = Keyspace.open(TEST_KEYSPACE);
+ SyncTasks result = service.alignToShardBoundaries(keyspace, inputTasks);
+
+ // Should be split into two shard entries
+ AtomicInteger entries = new AtomicInteger(0);
+ result.forEach((shardedTask) -> entries.incrementAndGet());
+ assertEquals(2, entries.get());
+
+ // Collect all ranges
+ Set<Range<Token>> allRanges = new HashSet<>();
+ result.forEach((shardedTask) -> allRanges.addAll(shardedTask.rangesToSync));
+
+ // Should have: b-e, f-m (shard 1), m-p (shard 2)
+ assertTrue(allRanges.contains(range("b", "e")));
+ assertTrue(allRanges.contains(range("f", "m")));
+ assertTrue(allRanges.contains(range("m", "p")));
+ }
+
+ @Test
+ public void testAlignToShardBoundariesPreservesTaskType()
+ {
+ MutationTrackingService service = MutationTrackingService.TestAccess.create();
+
+ // Create two shards
+ Set<Range<Token>> shardRanges = new HashSet<>();
+ shardRanges.add(range("a", "m"));
+ shardRanges.add(range("m", "z"));
+ MutationTrackingService.KeyspaceShards shards =
+ MutationTrackingService.TestAccess.createTestKeyspaceShards(TEST_KEYSPACE, shardRanges);
+ MutationTrackingService.TestAccess.setKeyspaceShards(service, TEST_KEYSPACE, shards);
+
+ // Task spanning both shards
+ List<SyncTask> inputTasks = Collections.singletonList(createSyncTask(range("d", "s")));
+
+ Keyspace keyspace = Keyspace.open(TEST_KEYSPACE);
+ SyncTasks result = service.alignToShardBoundaries(keyspace, inputTasks);
+
+ // All resulting tasks should be the same type as the input
+ result.apply((shardedTask) -> assertTrue("Task should be SymmetricRemoteSyncTask", shardedTask.task instanceof SymmetricRemoteSyncTask));
+ }
+
+ private static Token tk(String key)
+ {
+ return new ByteOrderedPartitioner.BytesToken(ByteBufferUtil.bytes(key));
+ }
+
+ private static Range<Token> range(String left, String right)
+ {
+ return new Range<>(tk(left), tk(right));
+ }
+
+ private static SyncTask createSyncTask(Range<Token> range)
+ {
+ return createSyncTask(Collections.singletonList(range), LOCAL, REMOTE);
+ }
+
+ private static SyncTask createSyncTask(List<Range<Token>> ranges, InetAddressAndPort local, InetAddressAndPort remote)
+ {
+ SharedContext ctx = SharedContext.Global.instance;
+ TimeUUID sessionId = TimeUUID.Generator.nextTimeUUID();
+ RepairJobDesc desc = new RepairJobDesc(sessionId, TimeUUID.Generator.nextTimeUUID(),TEST_KEYSPACE, TEST_TABLE, ranges);
+ return new SymmetricRemoteSyncTask(ctx, desc, local, remote, ranges, PreviewKind.NONE, null);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/replication/TransferTrackingServiceTest.java b/test/unit/org/apache/cassandra/replication/TransferTrackingServiceTest.java
new file mode 100644
index 0000000000..159dd3cffe
--- /dev/null
+++ b/test/unit/org/apache/cassandra/replication/TransferTrackingServiceTest.java
@@ -0,0 +1,342 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.replication;
+
+import java.util.Collection;
+import java.util.Collections;
+
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.TimeUUID;
+import org.assertj.core.api.Assertions;
+
+import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.COMMITTED;
+import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.COMMITTING;
+import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.PREPARE_FAILED;
+import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.PREPARING;
+import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.STREAM_COMPLETE;
+import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.STREAM_FAILED;
+import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.STREAM_NOOP;
+import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.Mockito.mock;
+
+public class TransferTrackingServiceTest
+{
+ private TransferTrackingService transferTrackingService;
+ private ShortMutationId transferId;
+ private TimeUUID planId;
+
+ @BeforeClass
+ public static void setUpClass()
+ {
+ DatabaseDescriptor.daemonInitialization();
+ DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
+ }
+
+ @Before
+ public void setUp() throws Exception
+ {
+ transferTrackingService = new TransferTrackingService();
+ transferId = new ShortMutationId(1, 100);
+ planId = nextTimeUUID();
+ }
+
+ private TrackedImportTransfer coordinatedTransfer(ShortMutationId transferId)
+ {
+ return coordinatedTransfer(transferId, new Range<>(tk(0), tk(1000)));
+ }
+
+ private TrackedImportTransfer coordinatedTransfer(ShortMutationId transferId, Range<Token> range)
+ {
+ MutationId mutationId = transferId != null
+ ? new MutationId(transferId.logId(), transferId.offset(), (int) System.currentTimeMillis())
+ : null;
+ return new TrackedImportTransfer(range, mutationId);
+ }
+
+ private PendingLocalTransfer pendingTransfer(TimeUUID planId)
+ {
+ SSTableReader mockSSTable = mock(SSTableReader.class);
+ Collection<SSTableReader> sstables = Collections.singletonList(mockSSTable);
+
+ return new PendingLocalTransfer(planId, sstables);
+ }
+
+ private static Token tk(long token)
+ {
+ return new Murmur3Partitioner.LongToken(token);
+ }
+
+ @Test
+ public void testSaveCoordinatedTransfer()
+ {
+ TrackedImportTransfer transfer = coordinatedTransfer(transferId);
+
+ transferTrackingService.save(transfer);
+
+ CoordinatedTransfer loaded = transferTrackingService.getActivatedTransfer(transferId);
+ Assertions.assertThat(loaded).isEqualTo(transfer);
+
+ assertThatThrownBy(() -> transferTrackingService.save(transfer))
+ .isInstanceOf(IllegalStateException.class);
+ }
+
+ @Test
+ public void testActivatingTransfer()
+ {
+ TrackedImportTransfer transfer = coordinatedTransfer(transferId);
+
+ transferTrackingService.save(transfer);
+
+ CoordinatedTransfer retrieved = transferTrackingService.getActivatedTransfer(transferId);
+ assertThat(retrieved).isEqualTo(transfer);
+ }
+
+ @Test
+ public void testReceivedTransfer()
+ {
+ TimeUUID planId = nextTimeUUID();
+ PendingLocalTransfer transfer = pendingTransfer(planId);
+ transferTrackingService.received(transfer);
+ PendingLocalTransfer retrieved = transferTrackingService.getPendingTransfer(planId);
+ assertThat(retrieved).isEqualTo(transfer);
+ }
+
+ @Test
+ public void testReceivedEmptyTransferThrows()
+ {
+ assertThatThrownBy(() -> new PendingLocalTransfer(planId, Collections.emptyList()))
+ .isInstanceOf(IllegalStateException.class);
+ }
+
+ @Test
+ public void testGetPendingTransferNotFound()
+ {
+ PendingLocalTransfer retrieved = transferTrackingService.getPendingTransfer(planId);
+ assertThat(retrieved).isNull();
+ }
+
+ @Test
+ public void testGetActivatedTransferNotFound()
+ {
+ CoordinatedTransfer retrieved = transferTrackingService.getActivatedTransfer(transferId);
+ assertThat(retrieved).isNull();
+ }
+
+ @Test
+ public void testPurgingTransferNotStarted()
+ {
+ TrackedImportTransfer transfer = coordinatedTransfer(transferId);
+
+ // All streams in INIT state - should NOT be purgeable (stream hasn't started yet)
+ TrackedImportTransfer.SingleTransferResult result = TrackedImportTransfer.SingleTransferResult.Init();
+ transfer.streamResults.put(mock(Pair.class), result);
+
+ Assertions.assertThat(transferTrackingService.purger.test(transfer)).isFalse();
+ }
+
+ @Test
+ public void testPurgingTransferAllStreamsComplete()
+ {
+ TrackedImportTransfer transfer = coordinatedTransfer(transferId);
+
+ // All streams in STREAM_COMPLETE state - should NOT be purgeable (no failures)
+ TrackedImportTransfer.SingleTransferResult result1 = TrackedImportTransfer.SingleTransferResult.StreamComplete(nextTimeUUID());
+ TrackedImportTransfer.SingleTransferResult result2 = TrackedImportTransfer.SingleTransferResult.StreamComplete(nextTimeUUID());
+
+ transfer.streamResults.put(mock(Pair.class), result1);
+ transfer.streamResults.put(mock(Pair.class), result2);
+
+ Assertions.assertThat(transferTrackingService.purger.test(transfer)).isFalse();
+ }
+
+ @Test
+ public void testPurgingTransferPrepareFailed()
+ {
+ TrackedImportTransfer transfer = coordinatedTransfer(transferId);
+
+ TrackedImportTransfer.SingleTransferResult result1 = new TrackedImportTransfer.SingleTransferResult(PREPARE_FAILED, planId);
+ TrackedImportTransfer.SingleTransferResult result2 = new TrackedImportTransfer.SingleTransferResult(PREPARING, planId);
+
+ transfer.streamResults.put(mock(Pair.class), result1);
+ transfer.streamResults.put(mock(Pair.class), result2);
+
+ Assertions.assertThat(transferTrackingService.purger.test(transfer)).isTrue();
+ }
+
+ @Test
+ public void testPurgingTransferAllActivationCommitted()
+ {
+ TrackedImportTransfer transfer = coordinatedTransfer(transferId);
+
+ // All streams in ACTIVATE_COMMITTED state - should be purgeable (allComplete = true)
+ TrackedImportTransfer.SingleTransferResult result1 = new TrackedImportTransfer.SingleTransferResult(COMMITTED, planId);
+ TrackedImportTransfer.SingleTransferResult result2 = new TrackedImportTransfer.SingleTransferResult(COMMITTED, planId);
+
+ transfer.streamResults.put(mock(Pair.class), result1);
+ transfer.streamResults.put(mock(Pair.class), result2);
+
+ Assertions.assertThat(transferTrackingService.purger.test(transfer)).isTrue();
+ }
+
+ @Test
+ public void testPurgingTransferMixedCommittedAndNoop()
+ {
+ TrackedImportTransfer transfer = coordinatedTransfer(transferId);
+
+ // Mix of ACTIVATE_COMMITTED and STREAM_NOOP - should be purgeable (allComplete = true)
+ TrackedImportTransfer.SingleTransferResult result1 = new TrackedImportTransfer.SingleTransferResult(COMMITTED, planId);
+ TrackedImportTransfer.SingleTransferResult result2 = new TrackedImportTransfer.SingleTransferResult(STREAM_NOOP, null);
+
+ transfer.streamResults.put(mock(Pair.class), result1);
+ transfer.streamResults.put(mock(Pair.class), result2);
+
+ Assertions.assertThat(transferTrackingService.purger.test(transfer)).isTrue();
+ }
+
+ @Test
+ public void testPurgingTransferActivationPartialCommitted()
+ {
+ TrackedImportTransfer transfer = coordinatedTransfer(transferId);
+
+ // One stream in ACTIVATE_PREPARING - should NOT be purgeable
+ TrackedImportTransfer.SingleTransferResult result1 = new TrackedImportTransfer.SingleTransferResult(PREPARING, planId);
+ TrackedImportTransfer.SingleTransferResult result2 = new TrackedImportTransfer.SingleTransferResult(COMMITTING, planId);
+
+ transfer.streamResults.put(mock(Pair.class), result1);
+ transfer.streamResults.put(mock(Pair.class), result2);
+
+ Assertions.assertThat(transferTrackingService.purger.test(transfer)).isFalse();
+ }
+
+ @Test
+ public void testPurgingTransferAllStreamsFailed()
+ {
+ TrackedImportTransfer transfer = coordinatedTransfer(transferId);
+
+ // All streams in STREAM_FAILED state - should be purgeable (noneActivated = true)
+ TrackedImportTransfer.SingleTransferResult result1 = new TrackedImportTransfer.SingleTransferResult(STREAM_FAILED, planId);
+ TrackedImportTransfer.SingleTransferResult result2 = new TrackedImportTransfer.SingleTransferResult(STREAM_FAILED, planId);
+
+ transfer.streamResults.put(mock(Pair.class), result1);
+ transfer.streamResults.put(mock(Pair.class), result2);
+
+ Assertions.assertThat(transferTrackingService.purger.test(transfer)).isTrue();
+ }
+
+ @Test
+ public void testPurgingTransferMixedInitAndFailed()
+ {
+ TrackedImportTransfer transfer = coordinatedTransfer(transferId);
+
+ // Mix of INIT and STREAM_FAILED - should be purgeable (has failure, none activated)
+ TrackedImportTransfer.SingleTransferResult result1 = TrackedImportTransfer.SingleTransferResult.Init();
+ TrackedImportTransfer.SingleTransferResult result2 = TrackedImportTransfer.SingleTransferResult.Init().streamFailed(nextTimeUUID());
+
+ transfer.streamResults.put(mock(Pair.class), result1);
+ transfer.streamResults.put(mock(Pair.class), result2);
+
+ Assertions.assertThat(transferTrackingService.purger.test(transfer)).isTrue();
+ }
+
+ @Test
+ public void testPurgingTransferMixedCompleteAndFailed()
+ {
+ TrackedImportTransfer transfer = coordinatedTransfer(transferId);
+
+ // Mix of STREAM_COMPLETE and STREAM_FAILED - should be purgeable (has failure, none activated)
+ TrackedImportTransfer.SingleTransferResult result1 = TrackedImportTransfer.SingleTransferResult.StreamComplete(nextTimeUUID());
+ TrackedImportTransfer.SingleTransferResult result2 = TrackedImportTransfer.SingleTransferResult.Init().streamFailed(nextTimeUUID());
+
+ transfer.streamResults.put(mock(Pair.class), result1);
+ transfer.streamResults.put(mock(Pair.class), result2);
+
+ Assertions.assertThat(transferTrackingService.purger.test(transfer)).isTrue();
+ }
+
+ @Test
+ public void testPurgingTransferMixedStreamingCompleteAndPreparing()
+ {
+ TrackedImportTransfer transfer = coordinatedTransfer(transferId);
+
+ // Mix of STREAM_COMPLETE and ACTIVATE_PREPARING - should NOT be purgeable
+ // (noneActivated = false because of ACTIVATE_PREPARING, allComplete = false)
+ TrackedImportTransfer.SingleTransferResult result1 = new TrackedImportTransfer.SingleTransferResult(STREAM_COMPLETE, planId);
+ TrackedImportTransfer.SingleTransferResult result2 = new TrackedImportTransfer.SingleTransferResult(PREPARING, planId);
+
+ transfer.streamResults.put(mock(Pair.class), result1);
+ transfer.streamResults.put(mock(Pair.class), result2);
+
+ Assertions.assertThat(transferTrackingService.purger.test(transfer)).isFalse();
+ }
+
+ @Test
+ public void testPurgingTransferMixedCommittingCommitted()
+ {
+ TrackedImportTransfer transfer = coordinatedTransfer(transferId);
+
+ TrackedImportTransfer.SingleTransferResult result1 = new TrackedImportTransfer.SingleTransferResult(COMMITTING, planId);
+ TrackedImportTransfer.SingleTransferResult result2 = new TrackedImportTransfer.SingleTransferResult(COMMITTED, planId);
+
+ transfer.streamResults.put(mock(Pair.class), result1);
+ transfer.streamResults.put(mock(Pair.class), result2);
+
+ Assertions.assertThat(transferTrackingService.purger.test(transfer)).isFalse();
+ }
+
+ @Test
+ public void testPurgingTransferWithNullTransferId()
+ {
+ TrackedImportTransfer transfer = coordinatedTransfer(null);
+
+ // All streams complete but transferId is null - should NOT be purgeable
+ TrackedImportTransfer.SingleTransferResult result1 = new TrackedImportTransfer.SingleTransferResult(STREAM_COMPLETE, null);
+ TrackedImportTransfer.SingleTransferResult result2 = new TrackedImportTransfer.SingleTransferResult(STREAM_COMPLETE, null);
+
+ transfer.streamResults.put(mock(Pair.class), result1);
+ transfer.streamResults.put(mock(Pair.class), result2);
+
+ // allComplete = true, but transferId is null, so should not purge
+ Assertions.assertThat(transferTrackingService.purger.test(transfer)).isFalse();
+ }
+
+ @Test
+ public void testPurgingTransferNoopOnly()
+ {
+ TrackedImportTransfer transfer = coordinatedTransfer(transferId);
+
+ // All streams in STREAM_NOOP - should be purgeable (both noneActivated and allComplete are true)
+ TrackedImportTransfer.SingleTransferResult result1 = TrackedImportTransfer.SingleTransferResult.Noop();
+ TrackedImportTransfer.SingleTransferResult result2 = TrackedImportTransfer.SingleTransferResult.Noop();
+
+ transfer.streamResults.put(mock(Pair.class), result1);
+ transfer.streamResults.put(mock(Pair.class), result2);
+
+ Assertions.assertThat(transferTrackingService.purger.test(transfer)).isTrue();
+ }
+}
diff --git a/test/unit/org/apache/cassandra/schema/MockSchema.java b/test/unit/org/apache/cassandra/schema/MockSchema.java
index b7720e051c..c64d55f46a 100644
--- a/test/unit/org/apache/cassandra/schema/MockSchema.java
+++ b/test/unit/org/apache/cassandra/schema/MockSchema.java
@@ -222,15 +222,7 @@ public class MockSchema
first.retainable().getKey().slice(),
last.retainable().getKey().slice());
StatsMetadata statsMetadata = (StatsMetadata) metadataComponents.get(MetadataType.STATS);
- try (DataOutputStreamPlus out = descriptor.fileFor(Components.STATS).newOutputStream(File.WriteMode.OVERWRITE))
- {
- new StatsComponent(metadataComponents).save(descriptor);
- out.flush();
- }
- catch (IOException ioe)
- {
- throw new RuntimeException(ioe);
- }
+ writeStatsMetadata(descriptor, metadataComponents);
BigTableReader reader = new BigTableReader.Builder(descriptor).setComponents(components)
.setTableMetadataRef(cfs.metadata)
.setDataFile(fileHandle.sharedCopy())
@@ -253,7 +245,7 @@ public class MockSchema
}
else if (BtiFormat.is(format))
{
- Set<Component> components = ImmutableSet.of(Components.DATA, BtiFormat.Components.PARTITION_INDEX, BtiFormat.Components.ROW_INDEX, Components.FILTER, Components.TOC);
+ Set<Component> components = ImmutableSet.of(Components.DATA, BtiFormat.Components.PARTITION_INDEX, BtiFormat.Components.ROW_INDEX, Components.FILTER, Components.TOC, Components.STATS);
for (Component component : components)
{
File file = descriptor.fileFor(component);
@@ -268,9 +260,17 @@ public class MockSchema
collector.update(DeletionTime.build(timestamp, minLocalDeletionTime));
BufferDecoratedKey first = readerBounds(firstToken);
BufferDecoratedKey last = readerBounds(lastToken);
- StatsMetadata metadata = (StatsMetadata) collector.sstableLevel(level)
- .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, UNREPAIRED_SSTABLE, null, ImmutableCoordinatorLogOffsets.NONE, header, first.retainable().getKey(), last.retainable().getKey())
- .get(MetadataType.STATS);
+ Map<MetadataType, MetadataComponent> metadataComponents = collector.sstableLevel(level)
+ .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(),
+ 0.01f,
+ UNREPAIRED_SSTABLE,
+ null,
+ ImmutableCoordinatorLogOffsets.NONE,
+ header,
+ first.retainable().getKey(),
+ last.retainable().getKey());
+ StatsMetadata statsMetadata = (StatsMetadata) metadataComponents.get(MetadataType.STATS);
+ writeStatsMetadata(descriptor, metadataComponents);
BtiTableReader reader = new BtiTableReader.Builder(descriptor).setComponents(components)
.setTableMetadataRef(cfs.metadata)
.setDataFile(fileHandle.sharedCopy())
@@ -278,7 +278,7 @@ public class MockSchema
.setRowIndexFile(fileHandle.sharedCopy())
.setFilter(FilterFactory.AlwaysPresent)
.setMaxDataAge(1L)
- .setStatsMetadata(metadata)
+ .setStatsMetadata(statsMetadata)
.setOpenReason(SSTableReader.OpenReason.NORMAL)
.setSerializationHeader(header)
.setFirst(readerBounds(firstToken))
@@ -295,6 +295,19 @@ public class MockSchema
}
}
+ private static void writeStatsMetadata(Descriptor descriptor, Map<MetadataType, MetadataComponent> metadataComponents)
+ {
+ try (DataOutputStreamPlus out = descriptor.fileFor(Components.STATS).newOutputStream(File.WriteMode.OVERWRITE))
+ {
+ new StatsComponent(metadataComponents).save(descriptor);
+ out.flush();
+ }
+ catch (IOException ioe)
+ {
+ throw new RuntimeException(ioe);
+ }
+ }
+
private static void maybeSetDataLength(Descriptor descriptor, long size)
{
if (size > 0)
diff --git a/test/unit/org/apache/cassandra/service/SerializationsTest.java b/test/unit/org/apache/cassandra/service/SerializationsTest.java
index 00864f5a34..89de9c2fcd 100644
--- a/test/unit/org/apache/cassandra/service/SerializationsTest.java
+++ b/test/unit/org/apache/cassandra/service/SerializationsTest.java
@@ -192,7 +192,8 @@ public class SerializationsTest extends AbstractSerializationsTester
InetAddressAndPort src = InetAddressAndPort.getByNameOverrideDefaults("127.0.0.2", PORT);
InetAddressAndPort dest = InetAddressAndPort.getByNameOverrideDefaults("127.0.0.3", PORT);
- SyncRequest message = new SyncRequest(DESC, local, src, dest, Collections.singleton(FULL_RANGE), PreviewKind.NONE, false);
+ // TODO: Do we want to test with a transfer ID?
+ SyncRequest message = new SyncRequest(DESC, local, src, dest, Collections.singleton(FULL_RANGE), PreviewKind.NONE, false, null);
testRepairMessageWrite("service.SyncRequest.bin", SyncRequest.serializer, message);
}
@@ -228,9 +229,9 @@ public class SerializationsTest extends AbstractSerializationsTester
Lists.newArrayList(new StreamSummary(TABLE_ID, emptyList(), 5, 100)),
Lists.newArrayList(new StreamSummary(TABLE_ID, emptyList(), 500, 10))
));
- SyncResponse success = new SyncResponse(DESC, src, dest, true, summaries);
+ SyncResponse success = new SyncResponse(DESC, src, dest, true, summaries, RANDOM_UUID, null);
// sync fail
- SyncResponse fail = new SyncResponse(DESC, src, dest, false, emptyList());
+ SyncResponse fail = new SyncResponse(DESC, src, dest, false, emptyList(), null, null);
testRepairMessageWrite("service.SyncComplete.bin", SyncResponse.serializer, success, fail);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org