You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by bd...@apache.org on 2026/04/14 21:46:59 UTC
(cassandra) 10/36: Implement unified log reconciliation
This is an automated email from the ASF dual-hosted git repository.
bdeggleston pushed a commit to branch cep-45-mutation-tracking
in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit 9676c0eaeb265094f0f679733ab3de819ca61646
Author: Aleksey Yeschenko <al...@apache.org>
AuthorDate: Fri May 16 11:35:32 2025 +0100
Implement unified log reconciliation
patch by Aleksey Yeschenko; reviewed by Blake Eggleston for
CASSANDRA-20729
Co-authored-by: Aleksey Yeschenko <al...@apache.org>
Co-authored-by: Blake Eggleston <bl...@ultrablake.com>
---
.../db/AbstractReadCommandVerbHandler.java | 9 +
src/java/org/apache/cassandra/db/Keyspace.java | 24 +-
.../apache/cassandra/db/MutationVerbHandler.java | 1 +
src/java/org/apache/cassandra/dht/Splitter.java | 2 +-
.../apache/cassandra/journal/ActiveSegment.java | 15 +-
.../apache/cassandra/journal/EntrySerializer.java | 10 +
src/java/org/apache/cassandra/journal/Flusher.java | 8 +-
src/java/org/apache/cassandra/journal/Journal.java | 42 ++
.../apache/cassandra/journal/RecordPointer.java | 14 +-
src/java/org/apache/cassandra/journal/Segment.java | 11 +
.../org/apache/cassandra/journal/Segments.java | 13 +-
.../cassandra/locator/EndpointsForRange.java | 5 +
src/java/org/apache/cassandra/net/Message.java | 11 +-
src/java/org/apache/cassandra/net/Verb.java | 37 +-
.../cassandra/replication/ActiveLogReconciler.java | 207 +++++++
...icatedOffsets.java => BroadcastLogOffsets.java} | 20 +-
.../cassandra/replication/CoordinatorLog.java | 130 +++--
.../cassandra/replication/ExpiredStatePurger.java | 112 ++++
.../cassandra/replication/ForwardedWrite.java | 236 +++-----
.../cassandra/replication/IncomingMutations.java | 158 ++++++
.../cassandra/replication/Log2OffsetsMap.java | 75 ++-
.../cassandra/replication/MutationJournal.java | 21 +-
.../cassandra/replication/MutationSummary.java | 17 +
.../replication/MutationTrackingService.java | 228 ++++++--
...conciledMutations.java => Node2OffsetsMap.java} | 35 +-
.../org/apache/cassandra/replication/Offsets.java | 74 ++-
.../cassandra/replication/OutgoingMutations.java | 154 +++++
.../replication/PullMutationsRequest.java | 75 +++
.../cassandra/replication/PushMutationRequest.java | 183 ++++++
.../cassandra/replication/ReconciliationPlan.java | 210 -------
.../org/apache/cassandra/replication/Shard.java | 89 ++-
.../cassandra/replication/TrackedWriteRequest.java | 36 +-
.../replication/UnreconciledMutations.java | 160 +++++-
.../replication/UnreconciledMutationsReplica.java | 193 -------
.../org/apache/cassandra/service/StorageProxy.java | 5 +-
.../service/TrackedWriteResponseHandler.java | 16 +-
.../service/reads/range/RangeCommandIterator.java | 6 +-
.../reads/tracked/ExtendingCompletedRead.java | 12 +-
.../reads/tracked/FilteredFollowupRead.java | 17 +-
.../reads/tracked/PartialTrackedRangeRead.java | 13 +-
.../service/reads/tracked/PartialTrackedRead.java | 23 +-
.../tracked/PartialTrackedSinglePartitionRead.java | 2 +-
...dReconcileNotify.java => ReadReconcileAck.java} | 48 +-
.../reads/tracked/ReadReconcileReceive.java | 137 -----
.../service/reads/tracked/ReadReconcileSend.java | 166 ------
.../service/reads/tracked/ReadReconciliations.java | 396 +++++++++++++
.../reads/tracked/TrackedLocalReadCoordinator.java | 617 ---------------------
.../service/reads/tracked/TrackedLocalReads.java | 272 +++++----
.../service/reads/tracked/TrackedRead.java | 160 +++---
.../reads/tracked/TrackedSummaryResponse.java | 49 +-
.../distributed/test/ReadRepairTestBase.java | 20 +-
.../distributed/test/ReadRepairTester.java | 9 +-
.../tracking/MutationTrackingPendingReadTest.java | 4 +-
.../test/tracking/MutationTrackingTest.java | 58 +-
.../replication/CoordinatorLogOffsetsTest.java | 12 +-
.../apache/cassandra/replication/OffsetsTest.java | 95 +++-
56 files changed, 2758 insertions(+), 1994 deletions(-)
diff --git a/src/java/org/apache/cassandra/db/AbstractReadCommandVerbHandler.java b/src/java/org/apache/cassandra/db/AbstractReadCommandVerbHandler.java
index 9647891f8a..24e9a88974 100644
--- a/src/java/org/apache/cassandra/db/AbstractReadCommandVerbHandler.java
+++ b/src/java/org/apache/cassandra/db/AbstractReadCommandVerbHandler.java
@@ -34,9 +34,11 @@ import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.ClusterMetadataService;
import org.apache.cassandra.tcm.Epoch;
+import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.utils.FBUtilities;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static org.apache.cassandra.utils.MonotonicClock.Global.approxTime;
public abstract class AbstractReadCommandVerbHandler<T> implements IVerbHandler<T>
{
@@ -46,6 +48,13 @@ public abstract class AbstractReadCommandVerbHandler<T> implements IVerbHandler<
public final void doVerb(Message<T> message)
{
+ if (approxTime.now() > message.expiresAtNanos())
+ {
+ Tracing.trace("Discarding read from {} (timed out)", message.from());
+ MessagingService.instance().metrics.recordDroppedMessage(message, message.elapsedSinceCreated(NANOSECONDS), NANOSECONDS);
+ return;
+ }
+
ClusterMetadata metadata = ClusterMetadata.current();
if (message.epoch().isAfter(Epoch.EMPTY))
{
diff --git a/src/java/org/apache/cassandra/db/Keyspace.java b/src/java/org/apache/cassandra/db/Keyspace.java
index 532b534aa5..d6d0e45eac 100644
--- a/src/java/org/apache/cassandra/db/Keyspace.java
+++ b/src/java/org/apache/cassandra/db/Keyspace.java
@@ -615,24 +615,30 @@ public class Keyspace
if (TEST_FAIL_WRITES && getMetadata().name.equals(TEST_FAIL_WRITES_KS))
throw new RuntimeException("Testing write failures");
+ boolean started;
try (WriteContext ctx = trackedWriteHandler.beginWrite(mutation, true))
{
- MutationTrackingService.instance.startWriting(mutation);
+ started = MutationTrackingService.instance.startWriting(mutation);
- for (PartitionUpdate upd : mutation.getPartitionUpdates())
+ if (started)
{
- ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(upd.metadata().id);
- if (cfs == null)
+ for (PartitionUpdate upd : mutation.getPartitionUpdates())
{
- logger.error("Attempting to mutate non-existant table {} ({}.{})", upd.metadata().id, upd.metadata().keyspace, upd.metadata().name);
- continue;
- }
+ ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(upd.metadata().id);
+ if (cfs == null)
+ {
+ logger.error("Attempting to mutate non-existant table {} ({}.{})", upd.metadata().id, upd.metadata().keyspace, upd.metadata().name);
+ continue;
+ }
- cfs.getWriteHandler().write(mutation.id(), upd, ctx, true);
+ cfs.getWriteHandler().write(mutation.id(), upd, ctx, true);
+ }
}
}
- MutationTrackingService.instance.finishWriting(mutation);
+ if (started)
+ MutationTrackingService.instance.finishWriting(mutation);
+
if (future != null)
future.trySuccess(null);
diff --git a/src/java/org/apache/cassandra/db/MutationVerbHandler.java b/src/java/org/apache/cassandra/db/MutationVerbHandler.java
index 5b19ebbf98..84932b2aa3 100644
--- a/src/java/org/apache/cassandra/db/MutationVerbHandler.java
+++ b/src/java/org/apache/cassandra/db/MutationVerbHandler.java
@@ -55,6 +55,7 @@ public class MutationVerbHandler extends AbstractMutationVerbHandler<Mutation>
if (ackTo != null)
{
logger.trace("Enqueuing response for direct acknowledgement of forwarded tracked mutation to coordinator {}", ackTo.coordinator);
+ response = Message.builder(response).withId(ackTo.id).build();
MessagingService.instance().send(response, ackTo.coordinator);
}
}
diff --git a/src/java/org/apache/cassandra/dht/Splitter.java b/src/java/org/apache/cassandra/dht/Splitter.java
index 165a2ecf9b..6df33d6ad3 100644
--- a/src/java/org/apache/cassandra/dht/Splitter.java
+++ b/src/java/org/apache/cassandra/dht/Splitter.java
@@ -239,7 +239,7 @@ public abstract class Splitter extends AccordSplitter
* @param parts the number of subranges
* @return {@code parts} even subranges of {@code range}
*/
- private Set<Range<Token>> split(Range<Token> range, int parts)
+ public Set<Range<Token>> split(Range<Token> range, int parts)
{
// the range might not have enough tokens to split
BigInteger numTokens = tokensInRange(range);
diff --git a/src/java/org/apache/cassandra/journal/ActiveSegment.java b/src/java/org/apache/cassandra/journal/ActiveSegment.java
index 373b766267..d7c26c9104 100644
--- a/src/java/org/apache/cassandra/journal/ActiveSegment.java
+++ b/src/java/org/apache/cassandra/journal/ActiveSegment.java
@@ -35,7 +35,6 @@ import org.slf4j.LoggerFactory;
import accord.utils.Invariants;
-import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.utils.Simulate;
import org.apache.cassandra.utils.SyncUtil;
import org.apache.cassandra.utils.concurrent.OpOrder;
@@ -375,9 +374,9 @@ public final class ActiveSegment<K, V> extends Segment<K, V>
*/
@SuppressWarnings({ "resource", "RedundantSuppression" }) // op group will be closed by Allocation#write()
- Allocation allocate(int entrySize)
+ Allocation allocate(int recordSize)
{
- int totalSize = totalEntrySize(entrySize);
+ int totalSize = EntrySerializer.totalEntrySize(keySupport, recordSize, descriptor.userVersion);
OpOrder.Group opGroup = appendOrder.start();
try
{
@@ -396,14 +395,6 @@ public final class ActiveSegment<K, V> extends Segment<K, V>
}
}
- private int totalEntrySize(int recordSize)
- {
- return EntrySerializer.headerSize(keySupport, descriptor.userVersion)
- + recordSize
- + TypeSizes.INT_SIZE // CRC
- ;
- }
-
// allocate bytes in the segment, or return -1 if not enough space
private int allocateBytes(int size)
{
@@ -528,7 +519,7 @@ public final class ActiveSegment<K, V> extends Segment<K, V>
RecordPointer recordPointer()
{
- return new RecordPointer(descriptor.timestamp, start);
+ return new RecordPointer(descriptor.timestamp, start, length);
}
}
diff --git a/src/java/org/apache/cassandra/journal/EntrySerializer.java b/src/java/org/apache/cassandra/journal/EntrySerializer.java
index 70d1e934d4..2c35b0619f 100644
--- a/src/java/org/apache/cassandra/journal/EntrySerializer.java
+++ b/src/java/org/apache/cassandra/journal/EntrySerializer.java
@@ -231,6 +231,16 @@ public final class EntrySerializer
+ TypeSizes.INT_SIZE; // CRC
}
+ static <K> int overheadSize(KeySupport<K> keySupport, int userVersion)
+ {
+ return headerSize(keySupport, userVersion) + TypeSizes.INT_SIZE; // CRC
+ }
+
+ static <K> int totalEntrySize(KeySupport<K> keySupport, int recordSize, int userVersion)
+ {
+ return recordSize + overheadSize(keySupport, userVersion);
+ }
+
public static final class EntryHolder<K>
{
public K key;
diff --git a/src/java/org/apache/cassandra/journal/Flusher.java b/src/java/org/apache/cassandra/journal/Flusher.java
index 70ec6d5593..bd1998eae6 100644
--- a/src/java/org/apache/cassandra/journal/Flusher.java
+++ b/src/java/org/apache/cassandra/journal/Flusher.java
@@ -73,7 +73,7 @@ final class Flusher<K, V>
volatile long flushStartedAt;
// the time of the earliest flush that has completed an fsync; all Allocations written before this time are durable
volatile long fsyncFinishedFor = flushStartedAt;
- volatile RecordPointer fsyncFinishedForPosition = new RecordPointer(0, 0);
+ volatile RecordPointer fsyncFinishedForPosition = new RecordPointer(0, 0, 0);
// a signal that writers can wait on to be notified of a completed flush in PERIODIC FlushMode
private final WaitQueue fsyncComplete = newWaitQueue(); // TODO (expected): this is only used for testing, can we remove this?
@@ -214,7 +214,7 @@ final class Flusher<K, V>
fsyncedTo = fsyncTo.writtenToAtLeast();
fsyncTo.fsync();
}
- fsyncFinishedForPosition = new RecordPointer(fsyncTo.descriptor.timestamp, fsyncedTo, startedAt);
+ fsyncFinishedForPosition = new RecordPointer(fsyncTo.descriptor.timestamp, fsyncedTo, 0, startedAt);
fsyncFinishedFor = startedAt;
fsyncComplete.signalAll();
long finishedAt = clock.now();
@@ -383,7 +383,7 @@ final class Flusher<K, V>
private void afterFSync(long startedAt, long segment, int position)
{
- fsyncFinishedForPosition = new RecordPointer(segment, position, startedAt);
+ fsyncFinishedForPosition = new RecordPointer(segment, position, 0, startedAt);
fsyncFinishedFor = startedAt;
callbacks.onFsync();
fsyncComplete.signalAll();
@@ -481,7 +481,7 @@ final class Flusher<K, V>
public RecordPointer flushAsync(ActiveSegment<K, V>.Allocation alloc)
{
written.incrementAndGet();
- return new RecordPointer(alloc.descriptor().timestamp, alloc.start(), clock.now());
+ return new RecordPointer(alloc.descriptor().timestamp, alloc.start(), 0, clock.now());
}
@Override
diff --git a/src/java/org/apache/cassandra/journal/Journal.java b/src/java/org/apache/cassandra/journal/Journal.java
index 29972bb20d..8d69f06afc 100644
--- a/src/java/org/apache/cassandra/journal/Journal.java
+++ b/src/java/org/apache/cassandra/journal/Journal.java
@@ -36,6 +36,8 @@ import java.util.function.Function;
import java.util.function.Predicate;
import java.util.zip.CRC32;
+import javax.annotation.Nullable;
+
import com.codahale.metrics.Timer.Context;
import com.google.common.annotations.VisibleForTesting;
@@ -482,6 +484,46 @@ public class Journal<K, V> implements Shutdownable
return false;
}
+ /**
+ * @param id user-provided record id
+ * @return record pointer of the last entry with the provided id, or null if not found
+ */
+ @Nullable
+ public RecordPointer lookUpLast(K id)
+ {
+ try (OpOrder.Group group = readOrder.start())
+ {
+ for (Segment<K, V> segment : segments.get().allSorted(false))
+ {
+ long[] offsets = segment.index().lookUp(id);
+ if (offsets.length != 0)
+ {
+ long offsetAndSize = offsets[offsets.length - 1];
+ int offset = Index.readOffset(offsetAndSize);
+ int size = Index.readSize(offsetAndSize);
+ return new RecordPointer(segment.descriptor.timestamp, offset, size);
+ }
+ }
+ }
+ return null;
+ }
+
+ public int sizeOfRecord(RecordPointer pointer)
+ {
+ Descriptor descriptor = segments.get().descriptor(pointer.segment);
+ Invariants.nonNull(descriptor);
+ return pointer.size - EntrySerializer.overheadSize(keySupport, descriptor.userVersion);
+ }
+
+ public boolean read(RecordPointer pointer, RecordConsumer<K> consumer)
+ {
+ try (OpOrder.Group group = readOrder.start())
+ {
+ Segment<K, V> segment = segments.get().get(pointer.segment);
+ return segment != null && segment.read(pointer, consumer);
+ }
+ }
+
/**
* Synchronously write a record to the journal.
* <p/>
diff --git a/src/java/org/apache/cassandra/journal/RecordPointer.java b/src/java/org/apache/cassandra/journal/RecordPointer.java
index 1439e63c90..22d874712d 100644
--- a/src/java/org/apache/cassandra/journal/RecordPointer.java
+++ b/src/java/org/apache/cassandra/journal/RecordPointer.java
@@ -24,25 +24,27 @@ import com.google.common.primitives.Longs;
// TODO: make this available in the accord table as an ID
public class RecordPointer implements Comparable<RecordPointer>
{
- public final long segment; // unique segment id
- public final int position; // record start position within the segment
+ public final long segment; // unique segment id
+ public final int position; // record start position within the segment
+ public final int size; // full size of the record
public final long writtenAt; // only set for periodic mode
- public RecordPointer(long segment, int position)
+ public RecordPointer(long segment, int position, int size)
{
- this(segment, position, 0);
+ this(segment, position, size, 0);
}
- public RecordPointer(long segment, int position, long writtenAt)
+ public RecordPointer(long segment, int position, int size, long writtenAt)
{
this.segment = segment;
this.position = position;
+ this.size = size;
this.writtenAt = writtenAt;
}
public RecordPointer(RecordPointer pointer)
{
- this(pointer.segment, pointer.position, pointer.writtenAt);
+ this(pointer.segment, pointer.position, pointer.size, pointer.writtenAt);
}
@Override
diff --git a/src/java/org/apache/cassandra/journal/Segment.java b/src/java/org/apache/cassandra/journal/Segment.java
index 7335bb2ca2..361f156f9a 100644
--- a/src/java/org/apache/cassandra/journal/Segment.java
+++ b/src/java/org/apache/cassandra/journal/Segment.java
@@ -109,6 +109,17 @@ public abstract class Segment<K, V> implements SelfRefCounted<Segment<K, V>>, Co
return true;
}
+ boolean read(RecordPointer pointer, RecordConsumer<K> consumer)
+ {
+ EntrySerializer.EntryHolder<K> into = new EntrySerializer.EntryHolder<>();
+ if (read(pointer.position, pointer.size, into))
+ {
+ consumer.accept(descriptor.timestamp, pointer.position, into.key, into.value, descriptor.userVersion);
+ return true;
+ }
+ return false;
+ }
+
void readAll(K id, EntrySerializer.EntryHolder<K> into, RecordConsumer<K> onEntry)
{
long[] all = index().lookUpAll(id);
diff --git a/src/java/org/apache/cassandra/journal/Segments.java b/src/java/org/apache/cassandra/journal/Segments.java
index ea8dafd304..eadd5e17f2 100644
--- a/src/java/org/apache/cassandra/journal/Segments.java
+++ b/src/java/org/apache/cassandra/journal/Segments.java
@@ -22,6 +22,7 @@ import java.util.List;
import java.util.function.Predicate;
import org.agrona.collections.Long2ObjectHashMap;
+import javax.annotation.Nullable;
import accord.utils.Invariants;
import accord.utils.SortedArrays.SortedArrayList;
@@ -104,15 +105,11 @@ public class Segments<K, V>
return this.segments.values();
}
- public int count(Predicate<? super Segment<K, V>> predicate)
+ @Nullable
+ Descriptor descriptor(long timestamp)
{
- int count = 0;
- for (Segment<K, V> segment : segments.values())
- {
- if (predicate.test(segment))
- ++count;
- }
- return count;
+ Segment<K, V> segment = segments.get(timestamp);
+ return segment == null ? null : segment.descriptor;
}
/**
diff --git a/src/java/org/apache/cassandra/locator/EndpointsForRange.java b/src/java/org/apache/cassandra/locator/EndpointsForRange.java
index 6ac12707c2..c75f01f647 100644
--- a/src/java/org/apache/cassandra/locator/EndpointsForRange.java
+++ b/src/java/org/apache/cassandra/locator/EndpointsForRange.java
@@ -61,6 +61,11 @@ public class EndpointsForRange extends Endpoints<EndpointsForRange>
return new EndpointsForToken(token, list, byEndpoint);
}
+ public EndpointsForRange withRange(Range<Token> range)
+ {
+ return new EndpointsForRange(range, list, byEndpoint);
+ }
+
@Override
public EndpointsForRange snapshot()
{
diff --git a/src/java/org/apache/cassandra/net/Message.java b/src/java/org/apache/cassandra/net/Message.java
index 272266e0ef..f8ecfd58a6 100644
--- a/src/java/org/apache/cassandra/net/Message.java
+++ b/src/java/org/apache/cassandra/net/Message.java
@@ -19,11 +19,7 @@ package org.apache.cassandra.net;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.Collections;
-import java.util.EnumMap;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Supplier;
@@ -259,6 +255,11 @@ public class Message<T> implements ResponseContext
return outWithParam(nextId(), verb, 0, payload, flag2.addTo(flag1.addTo(0)), null, null);
}
+ public static <T> Message<T> outWithRequestTime(Verb verb, T payload, Dispatcher.RequestTime requestTime)
+ {
+ return outWithFlags(verb, payload, requestTime, Collections.emptyList());
+ }
+
public static <T> Message<T> outWithFlags(Verb verb, T payload, Dispatcher.RequestTime requestTime, List<MessageFlag> flags)
{
assert !verb.isResponse();
diff --git a/src/java/org/apache/cassandra/net/Verb.java b/src/java/org/apache/cassandra/net/Verb.java
index fa162b4cf4..79362bf5dc 100644
--- a/src/java/org/apache/cassandra/net/Verb.java
+++ b/src/java/org/apache/cassandra/net/Verb.java
@@ -77,8 +77,10 @@ import org.apache.cassandra.repair.messages.SyncRequest;
import org.apache.cassandra.repair.messages.SyncResponse;
import org.apache.cassandra.repair.messages.ValidationRequest;
import org.apache.cassandra.repair.messages.ValidationResponse;
+import org.apache.cassandra.replication.BroadcastLogOffsets;
import org.apache.cassandra.replication.ForwardedWrite;
-import org.apache.cassandra.replication.ShardReplicatedOffsets;
+import org.apache.cassandra.replication.PullMutationsRequest;
+import org.apache.cassandra.replication.PushMutationRequest;
import org.apache.cassandra.schema.SchemaMutationsSerializer;
import org.apache.cassandra.schema.SchemaPullVerbHandler;
import org.apache.cassandra.schema.SchemaPushVerbHandler;
@@ -133,9 +135,7 @@ import org.apache.cassandra.service.paxos.cleanup.PaxosStartPrepareCleanup;
import org.apache.cassandra.service.paxos.cleanup.PaxosUpdateLowBallot;
import org.apache.cassandra.service.paxos.v1.PrepareVerbHandler;
import org.apache.cassandra.service.paxos.v1.ProposeVerbHandler;
-import org.apache.cassandra.service.reads.tracked.ReadReconcileNotify;
-import org.apache.cassandra.service.reads.tracked.ReadReconcileReceive;
-import org.apache.cassandra.service.reads.tracked.ReadReconcileSend;
+import org.apache.cassandra.service.reads.tracked.ReadReconcileAck;
import org.apache.cassandra.service.reads.tracked.TrackedDataResponse;
import org.apache.cassandra.service.reads.tracked.TrackedRead;
import org.apache.cassandra.service.reads.tracked.TrackedSummaryResponse;
@@ -322,24 +322,25 @@ public enum Verb
TCM_FETCH_PEER_LOG_RSP (818, P0, shortTimeout, FETCH_METADATA, MessageSerializers::logStateSerializer, RESPONSE_HANDLER ),
TCM_FETCH_PEER_LOG_REQ (819, P0, rpcTimeout, FETCH_METADATA, () -> FetchPeerLog.serializer, () -> FetchPeerLog.Handler.instance, TCM_FETCH_PEER_LOG_RSP ),
+ INITIATE_DATA_MOVEMENTS_RSP (814, P1, rpcTimeout, MISC, () -> NoPayload.serializer, RESPONSE_HANDLER ),
+ INITIATE_DATA_MOVEMENTS_REQ (815, P1, rpcTimeout, MISC, () -> DataMovement.serializer, () -> DataMovementVerbHandler.instance, INITIATE_DATA_MOVEMENTS_RSP ),
+ DATA_MOVEMENT_EXECUTED_RSP (816, P1, rpcTimeout, MISC, () -> NoPayload.serializer, RESPONSE_HANDLER ),
+ DATA_MOVEMENT_EXECUTED_REQ (817, P1, rpcTimeout, MISC, () -> DataMovement.Status.serializer, () -> DataMovements.instance, DATA_MOVEMENT_EXECUTED_RSP ),
+
// tracked replication
- READ_RECONCILE_SEND (901, P0, rpcTimeout, READ, () -> ReadReconcileSend.serializer, () -> ReadReconcileSend.verbHandler ),
- READ_RECONCILE_RCV (902, P0, rpcTimeout, MUTATION, () -> ReadReconcileReceive.serializer, () -> ReadReconcileReceive.verbHandler ),
- READ_RECONCILE_NOTIFY (903, P0, rpcTimeout, REQUEST_RESPONSE, () -> ReadReconcileNotify.serializer, () -> ReadReconcileNotify.verbHandler ),
- FORWARDING_WRITE (904, P3, writeTimeout, MUTATION, () -> ForwardedWrite.Request.serializer, () -> ForwardedWrite.verbHandler),
- BROADCAST_LOG_OFFSETS (905, P1, rpcTimeout, MISC, () -> ShardReplicatedOffsets.serializer, () -> ShardReplicatedOffsets.verbHandler),
-
- TRACKED_PARTITION_READ_RSP (906, P2, readTimeout, REQUEST_RESPONSE, () -> TrackedDataResponse.serializer, RESPONSE_HANDLER ),
- TRACKED_PARTITION_READ_REQ (907, P3, readTimeout, READ, () -> TrackedRead.DataRequest.serializer, () -> TrackedRead.verbHandler, TRACKED_PARTITION_READ_RSP),
+ PULL_MUTATIONS_REQ (901, P2, readTimeout, READ, () -> PullMutationsRequest.serializer, () -> PullMutationsRequest.verbHandler ),
+ PUSH_MUTATION_REQ (902, P3, writeTimeout, MUTATION, () -> PushMutationRequest.serializer, () -> PushMutationRequest.verbHandler, MUTATION_RSP ),
+ READ_RECONCILE_ACK (903, P2, readTimeout, REQUEST_RESPONSE, () -> ReadReconcileAck.serializer, () -> ReadReconcileAck.verbHandler ),
+ FORWARD_WRITE_REQ (904, P3, writeTimeout, MUTATION, () -> ForwardedWrite.serializer, () -> ForwardedWrite.verbHandler ),
+ BROADCAST_LOG_OFFSETS (905, P1, rpcTimeout, MISC, () -> BroadcastLogOffsets.serializer, () -> BroadcastLogOffsets.verbHandler ),
+
+ TRACKED_PARTITION_READ_RSP (906, P2, readTimeout, REQUEST_RESPONSE, () -> TrackedDataResponse.serializer, RESPONSE_HANDLER ),
+ TRACKED_PARTITION_READ_REQ (907, P3, readTimeout, READ, () -> TrackedRead.DataRequest.serializer, () -> TrackedRead.verbHandler, TRACKED_PARTITION_READ_RSP),
TRACKED_RANGE_READ_RSP (908, P2, rangeTimeout, REQUEST_RESPONSE, () -> TrackedDataResponse.serializer, RESPONSE_HANDLER ),
TRACKED_RANGE_READ_REQ (909, P3, rangeTimeout, READ, () -> TrackedRead.DataRequest.serializer, () -> TrackedRead.verbHandler, TRACKED_RANGE_READ_RSP ),
- TRACKED_SUMMARY_RSP (910, P2, readTimeout, REQUEST_RESPONSE, () -> TrackedSummaryResponse.serializer, () -> TrackedSummaryResponse.verbHandler ),
+ TRACKED_SUMMARY_RSP (910, P2, readTimeout, REQUEST_RESPONSE, () -> TrackedSummaryResponse.serializer, () -> TrackedSummaryResponse.verbHandler ),
TRACKED_SUMMARY_REQ (911, P3, readTimeout, READ, () -> TrackedRead.SummaryRequest.serializer, () -> TrackedRead.verbHandler, TRACKED_SUMMARY_RSP ),
- INITIATE_DATA_MOVEMENTS_RSP (814, P1, rpcTimeout, MISC, () -> NoPayload.serializer, RESPONSE_HANDLER ),
- INITIATE_DATA_MOVEMENTS_REQ (815, P1, rpcTimeout, MISC, () -> DataMovement.serializer, () -> DataMovementVerbHandler.instance, INITIATE_DATA_MOVEMENTS_RSP ),
- DATA_MOVEMENT_EXECUTED_RSP (816, P1, rpcTimeout, MISC, () -> NoPayload.serializer, RESPONSE_HANDLER ),
- DATA_MOVEMENT_EXECUTED_REQ (817, P1, rpcTimeout, MISC, () -> DataMovement.Status.serializer, () -> DataMovements.instance, DATA_MOVEMENT_EXECUTED_RSP ),
// accord
ACCORD_SIMPLE_RSP (119, P2, writeTimeout, IMMEDIATE, () -> accordEmbedded(EnumSerializer.simpleReply), AccordService::responseHandlerOrNoop ),
@@ -418,8 +419,6 @@ public enum Verb
@Deprecated(since = "4.0")
INTERNAL_RSP (23, P1, rpcTimeout, INTERNAL_RESPONSE, () -> null, () -> ResponseVerbHandler.instance ),
- // largest used ID: 116
-
// CUSTOM VERBS
UNUSED_CUSTOM_VERB (CUSTOM,
0, P1, rpcTimeout, INTERNAL_RESPONSE, () -> null, () -> null ),
diff --git a/src/java/org/apache/cassandra/replication/ActiveLogReconciler.java b/src/java/org/apache/cassandra/replication/ActiveLogReconciler.java
new file mode 100644
index 0000000000..54ded3766c
--- /dev/null
+++ b/src/java/org/apache/cassandra/replication/ActiveLogReconciler.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.replication;
+
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+
+import org.agrona.concurrent.ManyToOneConcurrentLinkedQueue;
+import org.apache.cassandra.concurrent.Interruptible;
+import org.apache.cassandra.concurrent.Shutdownable;
+import org.apache.cassandra.exceptions.RequestFailure;
+import org.apache.cassandra.journal.RecordPointer;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessageFlag;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.NoPayload;
+import org.apache.cassandra.net.RequestCallback;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.utils.concurrent.Semaphore;
+
+import static java.util.concurrent.TimeUnit.MINUTES;
+import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
+import static org.apache.cassandra.concurrent.ExecutorFactory.SystemThreadTag.NON_DAEMON;
+import static org.apache.cassandra.concurrent.InfiniteLoopExecutor.Interrupts.SYNCHRONIZED;
+import static org.apache.cassandra.concurrent.InfiniteLoopExecutor.SimulatorSafe.SAFE;
+import static org.apache.cassandra.utils.concurrent.Semaphore.newSemaphore;
+
+// TODO (expected): handle temporarily down nodes
+public final class ActiveLogReconciler implements Shutdownable
+{
+ public enum Priority { HIGH, REGULAR }
+
+ // prioritised delivery of mutations that are needed by reads;
+ private final ManyToOneConcurrentLinkedQueue<Task> highPriorityTasks;
+
+ // regular write retries
+ private final ManyToOneConcurrentLinkedQueue<Task> regularPriorityTasks;
+
+ private final Interruptible executor;
+ private final Semaphore haveWork;
+
+ ActiveLogReconciler()
+ {
+ highPriorityTasks = new ManyToOneConcurrentLinkedQueue<>();
+ regularPriorityTasks = new ManyToOneConcurrentLinkedQueue<>();
+ haveWork = newSemaphore(1);
+ executor = executorFactory().infiniteLoop("Active-Log-Reconciler", new SendRunnable(), SAFE, NON_DAEMON, SYNCHRONIZED);
+ }
+
+ /**
+ * Schedule delivery of a mutation to the specified host
+ * TODO (expected): deduplicate via OutgoingMutations
+ */
+ void schedule(ShortMutationId mutationId, InetAddressAndPort toHost, Priority priority)
+ {
+ queue(priority).offer(new Task(mutationId, toHost));
+ haveWork.release(1);
+ }
+
+ /**
+ * Schedule delivery of mutations to the specified host
+ * TODO (expected): deduplicate via OutgoingMutations
+ */
+ void schedule(Offsets offsets, InetAddressAndPort toHost, Priority priority)
+ {
+ ManyToOneConcurrentLinkedQueue<Task> queue = queue(priority);
+ offsets.forEach(id -> queue.offer(new Task(id, toHost)));
+ haveWork.release(1);
+ }
+
+ private ManyToOneConcurrentLinkedQueue<Task> queue(Priority priority)
+ {
+ switch (priority)
+ {
+ case HIGH: return highPriorityTasks;
+ case REGULAR: return regularPriorityTasks;
+ default: throw new IllegalStateException();
+ }
+ }
+
+ private class SendRunnable implements Interruptible.Task
+ {
+ @Override
+ public void run(Interruptible.State state) throws InterruptedException
+ {
+ if (isPaused || isShutdown) return;
+
+ // TODO (expected): backoff, rate limits, per host and total
+ Task task;
+ while ((task = highPriorityTasks.poll()) != null)
+ task.send();
+ while ((task = regularPriorityTasks.poll()) != null)
+ task.send();
+
+ haveWork.acquire(1);
+ }
+ }
+
+ private static final class Task implements RequestCallback<NoPayload>
+ {
+ private final ShortMutationId mutationId;
+ private final InetAddressAndPort toHost;
+
+ Task(ShortMutationId mutationId, InetAddressAndPort toHost)
+ {
+ this.mutationId = mutationId;
+ this.toHost = toHost;
+ }
+
+ @Override
+ public boolean invokeOnFailure()
+ {
+ return true;
+ }
+
+ @Override
+ public void onResponse(Message<NoPayload> msg)
+ {
+ MutationTrackingService.instance.receivedWriteResponse(mutationId, toHost);
+ }
+
+ @Override
+ public void onFailure(InetAddressAndPort from, RequestFailure failureReason)
+ {
+ MutationTrackingService.instance.retryFailedWrite(mutationId, toHost, failureReason);
+ }
+
+ void send()
+ {
+ RecordPointer pointer = MutationJournal.instance.lookUp(mutationId);
+ Preconditions.checkNotNull(pointer, "Mutation %s not found in the journal", mutationId);
+
+ Message<PushMutationRequest> message =
+ Message.outWithFlag(Verb.PUSH_MUTATION_REQ,
+ new PushMutationRequest.Referenced(mutationId, pointer),
+ MessageFlag.CALL_BACK_ON_FAILURE);
+ MessagingService.instance().sendWithCallback(message, toHost, this);
+ }
+ }
+
+ private volatile boolean isShutdown = false;
+ private volatile boolean isPaused = false;
+
+ @Override
+ public boolean isTerminated()
+ {
+ return executor.isTerminated();
+ }
+
+ @Override
+ public void shutdown()
+ {
+ executor.shutdown();
+ }
+
+ @Override
+ public Object shutdownNow()
+ {
+ return executor.shutdownNow();
+ }
+
+ @Override
+ public boolean awaitTermination(long timeout, TimeUnit units) throws InterruptedException
+ {
+ return executor.awaitTermination(timeout, units);
+ }
+
+ public void shutdownBlocking() throws InterruptedException
+ {
+ isShutdown = true;
+ if (!executor.isTerminated())
+ {
+ executor.shutdown();
+ executor.awaitTermination(1, MINUTES);
+ }
+ }
+
+ @VisibleForTesting
+ void pauseForTesting()
+ {
+ isPaused = true;
+ }
+
+ @VisibleForTesting
+ void resumeForTesting()
+ {
+ isPaused = false;
+ }
+}
diff --git a/src/java/org/apache/cassandra/replication/ShardReplicatedOffsets.java b/src/java/org/apache/cassandra/replication/BroadcastLogOffsets.java
similarity index 82%
rename from src/java/org/apache/cassandra/replication/ShardReplicatedOffsets.java
rename to src/java/org/apache/cassandra/replication/BroadcastLogOffsets.java
index 267b63c9d2..b3fc118613 100644
--- a/src/java/org/apache/cassandra/replication/ShardReplicatedOffsets.java
+++ b/src/java/org/apache/cassandra/replication/BroadcastLogOffsets.java
@@ -34,15 +34,15 @@ import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.net.IVerbHandler;
-public class ShardReplicatedOffsets
+public class BroadcastLogOffsets
{
- private static final Logger logger = LoggerFactory.getLogger(ShardReplicatedOffsets.class);
+ private static final Logger logger = LoggerFactory.getLogger(BroadcastLogOffsets.class);
private final String keyspace;
private final Range<Token> range;
private final List<Offsets.Immutable> replicatedOffsets;
- public ShardReplicatedOffsets(String keyspace, Range<Token> range, List<Offsets.Immutable> offsets)
+ public BroadcastLogOffsets(String keyspace, Range<Token> range, List<Offsets.Immutable> offsets)
{
this.keyspace = keyspace;
this.range = range;
@@ -60,8 +60,8 @@ public class ShardReplicatedOffsets
return "ShardReplicatedOffsets{" + keyspace + ", " + range + ", " + replicatedOffsets + '}';
}
- public static final IVerbHandler<ShardReplicatedOffsets> verbHandler = message -> {
- ShardReplicatedOffsets replicatedOffsets = message.payload;
+ public static final IVerbHandler<BroadcastLogOffsets> verbHandler = message -> {
+ BroadcastLogOffsets replicatedOffsets = message.payload;
logger.trace("Received replicated offsets {} from {}", replicatedOffsets, message.from());
MutationTrackingService.instance.updateReplicatedOffsets(replicatedOffsets.keyspace,
replicatedOffsets.range,
@@ -69,10 +69,10 @@ public class ShardReplicatedOffsets
message.from());
};
- public static final IVersionedSerializer<ShardReplicatedOffsets> serializer = new IVersionedSerializer<>()
+ public static final IVersionedSerializer<BroadcastLogOffsets> serializer = new IVersionedSerializer<>()
{
@Override
- public void serialize(ShardReplicatedOffsets status, DataOutputPlus out, int version) throws IOException
+ public void serialize(BroadcastLogOffsets status, DataOutputPlus out, int version) throws IOException
{
out.writeUTF(status.keyspace);
AbstractBounds.tokenSerializer.serialize(status.range, out, version);
@@ -82,7 +82,7 @@ public class ShardReplicatedOffsets
}
@Override
- public ShardReplicatedOffsets deserialize(DataInputPlus in, int version) throws IOException
+ public BroadcastLogOffsets deserialize(DataInputPlus in, int version) throws IOException
{
String keyspace = in.readUTF();
Range<Token> range = (Range<Token>) AbstractBounds.tokenSerializer.deserialize(in, IPartitioner.global(), version);
@@ -90,11 +90,11 @@ public class ShardReplicatedOffsets
List<Offsets.Immutable> replicatedOffsets = new ArrayList<>(count);
for (int i = 0; i < count; ++i)
replicatedOffsets.add(Offsets.serializer.deserialize(in, version));
- return new ShardReplicatedOffsets(keyspace, range, replicatedOffsets);
+ return new BroadcastLogOffsets(keyspace, range, replicatedOffsets);
}
@Override
- public long serializedSize(ShardReplicatedOffsets replicatedOffsets, int version)
+ public long serializedSize(BroadcastLogOffsets replicatedOffsets, int version)
{
long size = 0;
size += TypeSizes.sizeof(replicatedOffsets.keyspace);
diff --git a/src/java/org/apache/cassandra/replication/CoordinatorLog.java b/src/java/org/apache/cassandra/replication/CoordinatorLog.java
index 1470bf594d..13228256e2 100644
--- a/src/java/org/apache/cassandra/replication/CoordinatorLog.java
+++ b/src/java/org/apache/cassandra/replication/CoordinatorLog.java
@@ -28,6 +28,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Preconditions;
+
+import org.agrona.collections.IntArrayList;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.PartitionPosition;
import org.apache.cassandra.dht.AbstractBounds;
@@ -45,18 +47,21 @@ public abstract class CoordinatorLog
protected final CoordinatorLogId logId;
protected final Participants participants;
+ protected final UnreconciledMutations unreconciledMutations;
protected final Offsets.Mutable[] witnessedOffsets;
protected final Offsets.Mutable reconciledOffsets;
protected final ReadWriteLock lock;
- abstract UnreconciledMutations unreconciledMutations();
+ abstract void receivedWriteResponse(ShortMutationId mutationId, int fromHostId);
CoordinatorLog(int localHostId, CoordinatorLogId logId, Participants participants)
{
this.localHostId = localHostId;
this.logId = logId;
this.participants = participants;
+
+ this.unreconciledMutations = new UnreconciledMutations();
this.lock = new ReentrantReadWriteLock();
Offsets.Mutable[] ids = new Offsets.Mutable[participants.size()];
@@ -73,34 +78,6 @@ public abstract class CoordinatorLog
: new CoordinatorLogReplica(localHostId, id, participants);
}
- void receivedWriteResponse(MutationId mutationId, int onHostId)
- {
- Preconditions.checkArgument(!mutationId.isNone());
- Preconditions.checkArgument(!Objects.equals(onHostId, ClusterMetadata.current().myNodeId().id()));
- logger.trace("witnessed remote mutation {} from {}", mutationId, onHostId);
- lock.writeLock().lock();
- try
- {
- if (!get(onHostId).add(mutationId.offset()))
- return; // already witnessed; very uncommon but possible path
-
- if (!getLocal().contains(mutationId.offset()))
- return; // local host hasn't witnessed yet -> no cleanup needed
-
- if (remoteReplicasWitnessed(mutationId.offset()))
- {
- logger.trace("marking mutation {} as fully reconciled", mutationId);
- // if all replicas have now witnessed the id, remove it from the index
- unreconciledMutations().remove(mutationId.offset());
- reconciledOffsets.add(mutationId.offset());
- }
- }
- finally
- {
- lock.writeLock().unlock();
- }
- }
-
void updateReplicatedOffsets(Offsets offsets, int onHostId)
{
lock.writeLock().lock();
@@ -114,7 +91,7 @@ public abstract class CoordinatorLog
if (othersWitnessed(offset, onHostId))
{
reconciledOffsets.add(offset);
- unreconciledMutations().remove(offset);
+ unreconciledMutations.remove(offset);
}
}
});
@@ -140,15 +117,16 @@ public abstract class CoordinatorLog
}
}
- void startWriting(Mutation mutation)
+ boolean startWriting(Mutation mutation)
{
lock.writeLock().lock();
try
{
if (getLocal().contains(mutation.id().offset()))
- return; // already witnessed; shouldn't get to this path often (duplicate mutation)
+ return false; // already witnessed; shouldn't get to this path often (duplicate mutation)
- unreconciledMutations().startWriting(mutation);
+ unreconciledMutations.startWriting(mutation);
+ return true;
}
finally
{
@@ -164,15 +142,16 @@ public abstract class CoordinatorLog
try
{
int offset = mutation.id().offset();
+ // we've raced with another write, no need to do anything else
if (!getLocal().add(offset))
- throw new IllegalStateException("finishWriting() called on a locally witnessed mutation " + mutation.id());
+ return;
- unreconciledMutations().finishWriting(mutation);
+ unreconciledMutations.finishWriting(mutation);
if (remoteReplicasWitnessed(offset))
{
reconciledOffsets.add(offset);
- unreconciledMutations().remove(offset);
+ unreconciledMutations.remove(offset);
}
}
finally
@@ -192,7 +171,7 @@ public abstract class CoordinatorLog
return true;
}
- private boolean remoteReplicasWitnessed(int offset)
+ protected boolean remoteReplicasWitnessed(int offset)
{
return othersWitnessed(offset, localHostId);
}
@@ -207,7 +186,7 @@ public abstract class CoordinatorLog
try
{
reconciledInto.addAll(reconciledOffsets);
- return unreconciledMutations().collect(token, tableId, includePending, unreconciledInto);
+ return unreconciledMutations.collect(token, tableId, includePending, unreconciledInto);
}
finally
{
@@ -225,7 +204,39 @@ public abstract class CoordinatorLog
try
{
reconciledInto.addAll(reconciledOffsets);
- return unreconciledMutations().collect(range, tableId, includePending, unreconciledInto);
+ return unreconciledMutations.collect(range, tableId, includePending, unreconciledInto);
+ }
+ finally
+ {
+ lock.readLock().unlock();
+ }
+ }
+
+ /**
+ * Collect the offsets in {@code remoteOffsets} that are missing from the local log.
+ */
+ void collectLocallyMissingMutations(Offsets remoteOffsets, Log2OffsetsMap.Mutable into)
+ {
+ lock.readLock().lock();
+ try
+ {
+ into.add(Offsets.Immutable.difference(remoteOffsets, getLocal()));
+ }
+ finally
+ {
+ lock.readLock().unlock();
+ }
+ }
+
+ void collectRemotelyMissingMutations(Offsets localOffsets, IntArrayList remoteNodeIds, Node2OffsetsMap into)
+ {
+ lock.readLock().lock();
+ try
+ {
+ remoteNodeIds.forEachInt(remoteNodeId -> {
+ Offsets missing = Offsets.Immutable.difference(get(remoteNodeId), localOffsets);
+ if (!missing.isEmpty()) into.add(remoteNodeId, missing);
+ });
}
finally
{
@@ -269,21 +280,42 @@ public abstract class CoordinatorLog
'}';
}
- public static class CoordinatorLogPrimary extends CoordinatorLog
+ static class CoordinatorLogPrimary extends CoordinatorLog
{
private final AtomicLong sequenceId = new AtomicLong(-1);
- private final UnreconciledMutationsReplica unreconciledMutations;
CoordinatorLogPrimary(int localHostId, CoordinatorLogId logId, Participants participants)
{
super(localHostId, logId, participants);
- unreconciledMutations = new UnreconciledMutationsReplica();
}
@Override
- UnreconciledMutationsReplica unreconciledMutations()
+ void receivedWriteResponse(ShortMutationId mutationId, int fromHostId)
{
- return unreconciledMutations;
+ Preconditions.checkArgument(!mutationId.isNone());
+ Preconditions.checkArgument(!Objects.equals(fromHostId, ClusterMetadata.current().myNodeId().id()));
+ logger.trace("witnessed remote mutation {} from {}", mutationId, fromHostId);
+ lock.writeLock().lock();
+ try
+ {
+ if (!get(fromHostId).add(mutationId.offset()))
+ return; // already witnessed; very uncommon but possible path
+
+ if (!getLocal().contains(mutationId.offset()))
+ return; // local host hasn't witnessed yet -> no cleanup needed
+
+ if (remoteReplicasWitnessed(mutationId.offset()))
+ {
+ logger.trace("marking mutation {} as fully reconciled", mutationId);
+ // if all replicas have now witnessed the id, remove it from the index
+ unreconciledMutations.remove(mutationId.offset());
+ reconciledOffsets.add(mutationId.offset());
+ }
+ }
+ finally
+ {
+ lock.writeLock().unlock();
+ }
}
MutationId nextId()
@@ -309,20 +341,18 @@ public abstract class CoordinatorLog
}
}
- public static class CoordinatorLogReplica extends CoordinatorLog
+ static class CoordinatorLogReplica extends CoordinatorLog
{
- private final UnreconciledMutationsReplica unreconciledMutations;
-
CoordinatorLogReplica(int localHostId, CoordinatorLogId logId, Participants participants)
{
super(localHostId, logId, participants);
- this.unreconciledMutations = new UnreconciledMutationsReplica();
}
@Override
- UnreconciledMutationsReplica unreconciledMutations()
+ void receivedWriteResponse(ShortMutationId mutationId, int fromHostId)
{
- return unreconciledMutations;
+ // no-op
}
}
+
}
diff --git a/src/java/org/apache/cassandra/replication/ExpiredStatePurger.java b/src/java/org/apache/cassandra/replication/ExpiredStatePurger.java
new file mode 100644
index 0000000000..552f62e105
--- /dev/null
+++ b/src/java/org/apache/cassandra/replication/ExpiredStatePurger.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.replication;
+
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.concurrent.ScheduledExecutorPlus;
+import org.apache.cassandra.concurrent.Shutdownable;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.utils.Clock;
+
+import static java.util.concurrent.TimeUnit.MINUTES;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
+import static org.apache.cassandra.concurrent.ExecutorFactory.SimulatorSemantics.NORMAL;
+
+/**
+ * Since most of our state machines don't use 2 way callbacks, expirations are handled here.
+ * </p>
+ * TODO (expected): switch to using accord.impl.RequestCallbacks instead of walking the maps
+ */
+public class ExpiredStatePurger implements Shutdownable
+{
+ private static final Logger logger = LoggerFactory.getLogger(ExpiredStatePurger.class);
+ public static final ExpiredStatePurger instance = new ExpiredStatePurger();
+
+ private final ScheduledExecutorPlus executor = executorFactory().scheduled("Expired-State-Purger", NORMAL);
+ private final CopyOnWriteArrayList<Expireable> expireables = new CopyOnWriteArrayList<>();
+
+ public interface Expireable
+ {
+ int expire(long nanoTime);
+ }
+
+ public ExpiredStatePurger()
+ {
+ long expirationInterval = defaultExpirationInterval();
+ executor.scheduleWithFixedDelay(this::expire, expirationInterval, expirationInterval, NANOSECONDS);
+ }
+
+ public void register(Expireable expireable)
+ {
+ expireables.add(expireable);
+ }
+
+ private void expire()
+ {
+ long nanoTime = Clock.Global.nanoTime();
+ for (Expireable expireable : expireables)
+ {
+ int n = expireable.expire(nanoTime);
+ if (n > 0) logger.trace("Expired {} {} entries", n, expireable);
+ }
+ }
+
+ private long defaultExpirationInterval()
+ {
+ return DatabaseDescriptor.getMinRpcTimeout(NANOSECONDS) / 2;
+ }
+
+ @Override
+ public void shutdown()
+ {
+ executor.shutdown();
+ }
+
+ @Override
+ public boolean isTerminated()
+ {
+ return executor.isTerminated();
+ }
+
+ @Override
+ public Object shutdownNow()
+ {
+ return executor.shutdownNow();
+ }
+
+ public void shutdownBlocking() throws InterruptedException
+ {
+ if (executor == null || executor.isTerminated())
+ return;
+
+ executor.shutdown();
+ executor.awaitTermination(1, MINUTES);
+ }
+
+ @Override
+ public boolean awaitTermination(long timeout, TimeUnit units) throws InterruptedException
+ {
+ return executor.awaitTermination(timeout, units);
+ }
+}
diff --git a/src/java/org/apache/cassandra/replication/ForwardedWrite.java b/src/java/org/apache/cassandra/replication/ForwardedWrite.java
index e12e020b90..c6014257e8 100644
--- a/src/java/org/apache/cassandra/replication/ForwardedWrite.java
+++ b/src/java/org/apache/cassandra/replication/ForwardedWrite.java
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.cassandra.replication;
import java.io.IOException;
@@ -31,6 +30,7 @@ import com.google.common.collect.Sets;
import org.apache.cassandra.db.*;
import org.apache.cassandra.exceptions.RequestFailure;
import org.apache.cassandra.locator.*;
+import org.apache.cassandra.tracing.Tracing;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -55,6 +55,8 @@ import org.apache.cassandra.transport.Dispatcher;
import org.apache.cassandra.utils.FBUtilities;
import static org.apache.cassandra.net.Verb.MUTATION_REQ;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static org.apache.cassandra.utils.MonotonicClock.Global.approxTime;
/**
* For a forwarded write there are 2 nodes involved in coordination, a coordinator and a leader. The coordinator is the
@@ -65,91 +67,7 @@ public class ForwardedWrite
{
private static final Logger logger = LoggerFactory.getLogger(ForwardedWrite.class);
- public interface Request
- {
- enum Kind
- {
- MUTATION(0);
-
- private final byte id;
-
- Kind(int id)
- {
- this.id = (byte) id;
- }
-
- IVersionedSerializer<Request> serializer()
- {
- switch (this)
- {
- case MUTATION:
- return MutationRequest.serializer;
- default:
- throw new IllegalStateException("Unhandled kind " + this);
- }
- }
-
- static final IVersionedSerializer<Kind> serializer = new IVersionedSerializer<Request.Kind>()
- {
- @Override
- public void serialize(Kind kind, DataOutputPlus out, int version) throws IOException
- {
- out.writeByte(kind.id);
-
- }
-
- @Override
- public Kind deserialize(DataInputPlus in, int version) throws IOException
- {
- byte id = in.readByte();
- switch (id)
- {
- case 0:
- return MUTATION;
- default:
- throw new IllegalStateException("Unknown kind: " + id);
- }
- }
-
- @Override
- public long serializedSize(Kind kind, int version)
- {
- return TypeSizes.BYTE_SIZE;
- }
- };
- }
-
- Kind kind();
- DecoratedKey key();
- void applyLocallyAndForwardToReplicas(CoordinatorAckInfo ackTo);
-
- IVersionedSerializer<Request> serializer = new IVersionedSerializer<>()
- {
- @Override
- public void serialize(Request request, DataOutputPlus out, int version) throws IOException
- {
- Kind.serializer.serialize(request.kind(), out, version);
- request.kind().serializer().serialize(request, out, version);
- }
-
- @Override
- public Request deserialize(DataInputPlus in, int version) throws IOException
- {
- Kind kind = Kind.serializer.deserialize(in, version);
- return kind.serializer().deserialize(in, version);
- }
-
- @Override
- public long serializedSize(Request request, int version)
- {
- long size = Kind.serializer.serializedSize(request.kind(), version);
- size += request.kind().serializer().serializedSize(request, version);
- return size;
- }
- };
- }
-
- public static class MutationRequest implements Request
+ public static class MutationRequest
{
private final Mutation mutation;
private final Set<NodeId> recipients;
@@ -175,19 +93,11 @@ public class ForwardedWrite
this.recipients = recipients;
}
- @Override
- public Kind kind()
- {
- return Kind.MUTATION;
- }
-
- @Override
public DecoratedKey key()
{
return mutation.key();
}
- @Override
public void applyLocallyAndForwardToReplicas(CoordinatorAckInfo ackTo)
{
Preconditions.checkState(ackTo != null);
@@ -197,7 +107,7 @@ public class ForwardedWrite
MutationId id = MutationTrackingService.instance.nextMutationId(keyspaceName, token);
// Do not wait for handler completion, since the coordinator is already waiting and we don't want to block the stage
- LeaderCallback handler = new LeaderCallback(keyspaceName, mutation.key().getToken(), id, ackTo);
+ LeaderCallback handler = new LeaderCallback(id, ackTo);
applyLocallyAndForwardToReplicas(mutation.withMutationId(id), recipients, handler, ackTo);
}
@@ -248,7 +158,6 @@ public class ForwardedWrite
.withRequestTime(handler.getRequestTime())
.withFlag(MessageFlag.CALL_BACK_ON_FAILURE)
.withParam(ParamType.COORDINATOR_ACK_INFO, ackTo)
- .withId(ackTo.id)
.build();
Replica replica = replicas.get(recipient);
@@ -286,41 +195,6 @@ public class ForwardedWrite
TrackedWriteRequest.sendMessagesToRemoteDC(message, EndpointsForToken.copyOf(mutation.key().getToken(), dcReplicas), handler, ackTo);
}
}
-
- public static final IVersionedSerializer<Request> serializer = new IVersionedSerializer<>()
- {
- @Override
- public void serialize(Request r, DataOutputPlus out, int version) throws IOException
- {
- MutationRequest request = (MutationRequest) r;
- Mutation.serializer.serialize(request.mutation, out, version);
- out.writeInt(request.recipients.size());
- for (NodeId recipient : request.recipients)
- NodeId.messagingSerializer.serialize(recipient, out, version);
- }
-
- @Override
- public Request deserialize(DataInputPlus in, int version) throws IOException
- {
- Mutation mutation = Mutation.serializer.deserialize(in, version);
- int numRecipients = in.readInt();
- Set<NodeId> recipients = Sets.newHashSetWithExpectedSize(numRecipients);
- for (int i = 0; i < numRecipients; i++)
- recipients.add(NodeId.messagingSerializer.deserialize(in, version));
- return new MutationRequest(mutation, recipients);
- }
-
- @Override
- public long serializedSize(Request r, int version)
- {
- MutationRequest request = (MutationRequest) r;
- long size = Mutation.serializer.serializedSize(request.mutation, version);
- size += TypeSizes.INT_SIZE;
- for (NodeId recipient : request.recipients)
- size += NodeId.messagingSerializer.serializedSize(recipient, version);
- return size;
- }
- };
}
public static AbstractWriteResponseHandler<Object> forwardMutation(Mutation mutation, ReplicaPlan.ForWrite plan, AbstractReplicationStrategy strategy, Dispatcher.RequestTime requestTime)
@@ -331,25 +205,35 @@ public class ForwardedWrite
Token token = mutation.key().getToken();
Keyspace keyspace = Keyspace.open(mutation.getKeyspaceName());
EndpointsForRange endpoints = cm.placements.get(keyspace.getMetadata().params.replication).writes.forRange(token).get();
- if (logger.isTraceEnabled())
- logger.trace("Finding best leader from replicas {}", endpoints);
+ logger.trace("Finding best leader from replicas {}", endpoints);
// TODO: Should match ReplicaPlans.findCounterLeaderReplica, including DC-local priority, current health, severity, etc.
- Replica leader = proximity.sortedByProximity(FBUtilities.getBroadcastAddressAndPort(), endpoints).get(0);
+ Replica leader = null;
+ for (Replica replica : proximity.sortedByProximity(FBUtilities.getBroadcastAddressAndPort(), endpoints))
+ {
+ if (plan.isAlive(replica))
+ leader = replica;
+ }
+ Preconditions.checkState(leader != null, "Could not find leader for %s", mutation);
// create callback and forward to leader
- if (logger.isTraceEnabled())
- logger.trace("Selected {} as leader for mutation with key {}", leader.endpoint(), mutation.key());
+ logger.trace("Selected {} as leader for mutation with key {}", leader.endpoint(), mutation.key());
AbstractWriteResponseHandler<Object> handler = strategy.getWriteResponseHandler(plan, null, WriteType.SIMPLE, null, requestTime);
// Add callbacks for replicas to respond directly to coordinator
- Message<Request> toLeader = Message.out(Verb.FORWARDING_WRITE, new MutationRequest(mutation, plan));
+ Message<MutationRequest> toLeader = Message.outWithRequestTime(Verb.FORWARD_WRITE_REQ, new MutationRequest(mutation, plan), requestTime);
for (Replica endpoint : endpoints)
{
- if (logger.isTraceEnabled())
+ if (plan.isAlive(endpoint))
+ {
logger.trace("Adding forwarding callback for response from {} id {}", endpoint, toLeader.id());
- MessagingService.instance().callbacks.addWithExpiration(handler, toLeader, endpoint);
+ MessagingService.instance().callbacks.addWithExpiration(handler, toLeader, endpoint);
+ }
+ else
+ {
+ handler.expired();
+ }
}
MessagingService.instance().send(toLeader, leader.endpoint());
@@ -357,26 +241,60 @@ public class ForwardedWrite
return handler;
}
- public static final IVerbHandler<Request> verbHandler = new IVerbHandler<>()
+ public static final IVersionedSerializer<MutationRequest> serializer = new IVersionedSerializer<>()
{
@Override
- public void doVerb(Message<Request> incoming)
+ public void serialize(MutationRequest request, DataOutputPlus out, int version) throws IOException
{
- if (logger.isTraceEnabled())
- logger.trace("Received incoming ForwardedWriteRequest {} id {}", incoming, incoming.id());
- CoordinatorAckInfo ackTo = CoordinatorAckInfo.toCoordinator(incoming.from(), incoming.id());
- Request request = incoming.payload;
+ Mutation.serializer.serialize(request.mutation, out, version);
+ out.writeInt(request.recipients.size());
+ for (NodeId recipient : request.recipients)
+ NodeId.messagingSerializer.serialize(recipient, out, version);
+ }
- // Once we support epoch changes, check epoch from coordinator here, after potential queueing on the Stage
- try
- {
- request.applyLocallyAndForwardToReplicas(ackTo);
- }
- catch (Exception e)
- {
- logger.error("Exception while executing forwarded write with key {} on leader", request.key(), e);
- MessagingService.instance().respondWithFailure(RequestFailureReason.UNKNOWN, incoming);
- }
+ @Override
+ public MutationRequest deserialize(DataInputPlus in, int version) throws IOException
+ {
+ Mutation mutation = Mutation.serializer.deserialize(in, version);
+ int numRecipients = in.readInt();
+ Set<NodeId> recipients = Sets.newHashSetWithExpectedSize(numRecipients);
+ for (int i = 0; i < numRecipients; i++)
+ recipients.add(NodeId.messagingSerializer.deserialize(in, version));
+ return new MutationRequest(mutation, recipients);
+ }
+
+ @Override
+ public long serializedSize(MutationRequest request, int version)
+ {
+ long size = Mutation.serializer.serializedSize(request.mutation, version);
+ size += TypeSizes.INT_SIZE;
+ for (NodeId recipient : request.recipients)
+ size += NodeId.messagingSerializer.serializedSize(recipient, version);
+ return size;
+ }
+ };
+
+ public static final IVerbHandler<MutationRequest> verbHandler = incoming ->
+ {
+ if (approxTime.now() > incoming.expiresAtNanos())
+ {
+ Tracing.trace("Discarding mutation from {} (timed out)", incoming.from());
+ MessagingService.instance().metrics.recordDroppedMessage(incoming, incoming.elapsedSinceCreated(NANOSECONDS), NANOSECONDS);
+ return;
+ }
+ logger.trace("Received incoming ForwardedWriteRequest {} id {}", incoming, incoming.id());
+ CoordinatorAckInfo ackTo = CoordinatorAckInfo.toCoordinator(incoming.from(), incoming.id());
+ MutationRequest request = incoming.payload;
+
+ // Once we support epoch changes, check epoch from coordinator here, after potential queueing on the Stage
+ try
+ {
+ request.applyLocallyAndForwardToReplicas(ackTo);
+ }
+ catch (Exception e)
+ {
+ logger.error("Exception while executing forwarded write with key {} on leader", request.key(), e);
+ MessagingService.instance().respondWithFailure(RequestFailureReason.UNKNOWN, incoming);
}
};
@@ -384,16 +302,12 @@ public class ForwardedWrite
// See org.apache.cassandra.service.TrackedWriteResponseHandler.onResponse, this class should probably merge with that one
public static class LeaderCallback implements RequestCallback<NoPayload>
{
- private final String keyspace;
- private final Token token;
private final MutationId id;
private final CoordinatorAckInfo ackTo;
private final Dispatcher.RequestTime requestTime = Dispatcher.RequestTime.forImmediateExecution();
- public LeaderCallback(String keyspace, Token token, MutationId id, CoordinatorAckInfo ackTo)
+ public LeaderCallback(MutationId id, CoordinatorAckInfo ackTo)
{
- this.keyspace = keyspace;
- this.token = token;
this.id = id;
this.ackTo = ackTo;
}
@@ -403,7 +317,7 @@ public class ForwardedWrite
{
// Local mutations are witnessed from Keyspace.applyInternalTracked
if (msg != null)
- MutationTrackingService.instance.receivedWriteResponse(keyspace, token, id, msg.from());
+ MutationTrackingService.instance.receivedWriteResponse(id, msg.from());
// Local write needs to be ack'd to coordinator
if (msg == null && ackTo != null)
diff --git a/src/java/org/apache/cassandra/replication/IncomingMutations.java b/src/java/org/apache/cassandra/replication/IncomingMutations.java
new file mode 100644
index 0000000000..d67e152c92
--- /dev/null
+++ b/src/java/org/apache/cassandra/replication/IncomingMutations.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.replication;
+
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.utils.Clock;
+import org.jctools.maps.NonBlockingHashMap;
+
+/**
+ * Listeners for incoming mutations requested by e.g. {@link org.apache.cassandra.service.reads.tracked.TrackedLocalReads}.
+ * </p>
+ * The mutations being requested may not necessarily have a corresponding {@link CoordinatorLog}
+ * instance just yet - e.g. it's a mutation from a fresh coordinator log that we are requesting,
+ * and we were only made aware of it via missing offsets in read reconciliation.
+ * </p>
+ * For the above reason it's not folded into an instance {@link CoordinatorLog} but instead lives
+ * inside {@link MutationTrackingService}.
+ */
+public class IncomingMutations implements ExpiredStatePurger.Expireable
+{
+ private static final Logger logger = LoggerFactory.getLogger(IncomingMutations.class);
+
+ private final NonBlockingHashMap<ShortMutationId, Listeners> listenersMap = new NonBlockingHashMap<>();
+
+ /**
+ * Register to be notified to an incoming mutation.
+ * @return true if this is the first active listener added for this id
+ */
+ public boolean subscribe(ShortMutationId mutationId, Callback callback)
+ {
+ Listeners prev, next;
+ do
+ {
+ prev = listenersMap.get(mutationId);
+ next = Listeners.addListener(prev, callback);
+ }
+ while (listenersMap.putIfMatchAllowNull(mutationId, next, prev) != prev);
+ return prev == null;
+ }
+
+ public void invokeListeners(ShortMutationId mutationId)
+ {
+ Listeners listeners = listenersMap.remove(mutationId);
+ if (listeners != null) listeners.invokeListeners(mutationId);
+ }
+
+ @Override
+ public int expire(long nanoTime)
+ {
+ int n = 0;
+ for (Map.Entry<ShortMutationId, Listeners> entry : listenersMap.entrySet())
+ {
+ ShortMutationId id = entry.getKey();
+ Listeners listeners = entry.getValue();
+ if (listeners.isExpired(nanoTime) && listenersMap.remove(id, listeners))
+ {
+ listeners.expireListeners(id);
+ n++;
+ }
+ }
+ return n;
+ }
+
+ private static final class Listeners
+ {
+ private final long createdAt;
+ private final Callback[] callbacks;
+
+ Listeners(Callback callback)
+ {
+ this(new Callback[] { callback }, Clock.Global.nanoTime());
+ }
+
+ Listeners(Callback[] callbacks, long createdAt)
+ {
+ this.callbacks = callbacks;
+ this.createdAt = createdAt;
+ }
+
+ Listeners addListener(Callback callback)
+ {
+ Callback[] newCallbacks = new Callback[callbacks.length + 1];
+ System.arraycopy(callbacks, 0, newCallbacks, 0, callbacks.length);
+ newCallbacks[callbacks.length] = callback;
+ return new Listeners(newCallbacks, createdAt);
+ }
+
+ static Listeners addListener(Listeners listeners, Callback callback)
+ {
+ return listeners == null ? new Listeners(callback) : listeners.addListener(callback);
+ }
+
+ void invokeListeners(ShortMutationId mutationId)
+ {
+ for (Callback callback : callbacks)
+ {
+ try
+ {
+ callback.onSuccess(mutationId);
+ }
+ catch (Throwable e)
+ {
+ logger.error("Caught an error while processing onSuccess() callback for {}: {}", e, mutationId);
+ }
+ }
+ }
+
+ void expireListeners(ShortMutationId mutationId)
+ {
+ for (Callback callback : callbacks)
+ {
+ try
+ {
+ callback.onTimeout(mutationId);
+ }
+ catch (Throwable e)
+ {
+ logger.error("Caught an error while processing onTimeout() callback for {}: {}", e, mutationId);
+ }
+ }
+ }
+
+ boolean isExpired(long nanoTime)
+ {
+ return createdAt + DatabaseDescriptor.getWriteRpcTimeout(TimeUnit.NANOSECONDS) < nanoTime;
+ }
+ }
+
+ public interface Callback
+ {
+ void onSuccess(ShortMutationId mutationId);
+
+ default void onTimeout(ShortMutationId mutationId)
+ {
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/replication/Log2OffsetsMap.java b/src/java/org/apache/cassandra/replication/Log2OffsetsMap.java
index efcdd195ec..f7b03b15c5 100644
--- a/src/java/org/apache/cassandra/replication/Log2OffsetsMap.java
+++ b/src/java/org/apache/cassandra/replication/Log2OffsetsMap.java
@@ -19,6 +19,7 @@
package org.apache.cassandra.replication;
import java.io.IOException;
+import java.util.Collection;
import java.util.Iterator;
import com.google.common.base.Preconditions;
@@ -29,31 +30,34 @@ import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.replication.MutationSummary.CoordinatorSummary;
public abstract class Log2OffsetsMap<T extends Offsets> implements Iterable<ShortMutationId>
{
- abstract Long2ObjectHashMap<T> offsetMap();
+ abstract Long2ObjectHashMap<T> asMap();
@Override
public Iterator<ShortMutationId> iterator()
{
- return Iterables.concat(offsetMap().values()).iterator();
+ return Iterables.concat(asMap().values()).iterator();
+ }
+
+ public Collection<T> offsets()
+ {
+ return asMap().values();
}
public int idCount()
{
int count = 0;
- for (T offsets : offsetMap().values())
+ for (T offsets : offsets())
count += offsets.offsetCount();
return count;
}
public boolean isEmpty()
{
- for (T offsets : offsetMap().values())
- if (!offsets.isEmpty())
- return false;
- return true;
+ return Iterables.all(offsets(), Offsets::isEmpty);
}
private static abstract class AbstractMutable<T extends Offsets.AbstractMutable<T>> extends Log2OffsetsMap<T>
@@ -61,13 +65,15 @@ public abstract class Log2OffsetsMap<T extends Offsets> implements Iterable<Shor
protected final Long2ObjectHashMap<T> offsetMap = new Long2ObjectHashMap<>();
@Override
- Long2ObjectHashMap<T> offsetMap()
+ Long2ObjectHashMap<T> asMap()
{
return offsetMap;
}
protected abstract T createOrNull(Offsets.RangeIterator iterator);
+
protected abstract T create(CoordinatorLogId logId);
+
protected abstract T copy(Offsets offsets);
protected T create(long logId)
@@ -98,10 +104,22 @@ public abstract class Log2OffsetsMap<T extends Offsets> implements Iterable<Shor
public void addAll(Log2OffsetsMap<?> that)
{
- for (Offsets offsets : that.offsetMap().values())
+ for (Offsets offsets : that.asMap().values())
add(offsets);
}
+ public void addAll(MutationSummary summary)
+ {
+ for (int i = 0, size = summary.size(); i < size; i++)
+ addAll(summary.get(i));
+ }
+
+ public void addAll(CoordinatorSummary summary)
+ {
+ add(summary.reconciled);
+ add(summary.unreconciled);
+ }
+
public void remove(Offsets offsets)
{
T existing = offsetMap.get(offsets.logId().asLong());
@@ -114,6 +132,7 @@ public abstract class Log2OffsetsMap<T extends Offsets> implements Iterable<Shor
return;
}
+ // TODO (expected): ? shouldn't we updating Offsets in-place here? where did Offsets.remove() go?
T next = createOrNull(Offsets.difference(existing.rangeIterator(), offsets.rangeIterator()));
if (next == null)
offsetMap.remove(offsets.logId().asLong());
@@ -121,11 +140,41 @@ public abstract class Log2OffsetsMap<T extends Offsets> implements Iterable<Shor
offsetMap.put(offsets.logId().asLong(), next);
}
+ public void remove(ShortMutationId id)
+ {
+ T existing = offsetMap.get(id.logId());
+ if (existing == null)
+ return;
+
+ existing.remove(id.offset());
+
+ if (existing.isEmpty())
+ offsetMap.remove(id.logId());
+ }
+
public void removeAll(Log2OffsetsMap<?> that)
{
- for (Offsets offsets : that.offsetMap().values())
+ for (Offsets offsets : that.asMap().values())
remove(offsets);
}
+
+ public void removeAll(Iterable<ShortMutationId> ids)
+ {
+ for (ShortMutationId id : ids)
+ remove(id);
+ }
+
+ public void removeAll(MutationSummary summary)
+ {
+ for (int i = 0, size = summary.size(); i < size; i++)
+ removeAll(summary.get(i));
+ }
+
+ public void removeAll(CoordinatorSummary summary)
+ {
+ remove(summary.reconciled);
+ remove(summary.unreconciled);
+ }
}
public static class Mutable extends AbstractMutable<Offsets.Mutable>
@@ -159,7 +208,7 @@ public abstract class Log2OffsetsMap<T extends Offsets> implements Iterable<Shor
}
@Override
- Long2ObjectHashMap<Offsets.Immutable> offsetMap()
+ Long2ObjectHashMap<Offsets.Immutable> asMap()
{
return offsetMap;
}
@@ -192,13 +241,13 @@ public abstract class Log2OffsetsMap<T extends Offsets> implements Iterable<Shor
}
}
- public static final IVersionedSerializer<Log2OffsetsMap.Immutable> serializer = new IVersionedSerializer<Log2OffsetsMap.Immutable>()
+ public static final IVersionedSerializer<Log2OffsetsMap.Immutable> serializer = new IVersionedSerializer<>()
{
@Override
public void serialize(Log2OffsetsMap.Immutable mo, DataOutputPlus out, int version) throws IOException
{
out.writeInt(mo.offsetMap.size());
- for (Offsets.Immutable offsets : mo.offsetMap().values())
+ for (Offsets.Immutable offsets : mo.asMap().values())
Offsets.serializer.serialize(offsets, out, version);
}
diff --git a/src/java/org/apache/cassandra/replication/MutationJournal.java b/src/java/org/apache/cassandra/replication/MutationJournal.java
index 31730e98c6..bb0a93cd6a 100644
--- a/src/java/org/apache/cassandra/replication/MutationJournal.java
+++ b/src/java/org/apache/cassandra/replication/MutationJournal.java
@@ -75,11 +75,30 @@ public class MutationJournal
return journal.readLast(id);
}
- public boolean read(ShortMutationId id, RecordConsumer<ShortMutationId> consumer)
+ boolean read(ShortMutationId id, RecordConsumer<ShortMutationId> consumer)
{
return journal.readLast(id, consumer);
}
+ /**
+ * @return record pointer of the last mutation with the provided id, or null if not found
+ */
+ @Nullable
+ RecordPointer lookUp(ShortMutationId id)
+ {
+ return journal.lookUpLast(id);
+ }
+
+ int sizeOfRecord(RecordPointer pointer)
+ {
+ return journal.sizeOfRecord(pointer);
+ }
+
+ boolean read(RecordPointer pointer, RecordConsumer<ShortMutationId> consumer)
+ {
+ return journal.read(pointer, consumer);
+ }
+
public void readAll(Iterable<ShortMutationId> ids, Collection<Mutation> into)
{
for (ShortMutationId id : ids)
diff --git a/src/java/org/apache/cassandra/replication/MutationSummary.java b/src/java/org/apache/cassandra/replication/MutationSummary.java
index c786b295fe..f3bc6051d2 100644
--- a/src/java/org/apache/cassandra/replication/MutationSummary.java
+++ b/src/java/org/apache/cassandra/replication/MutationSummary.java
@@ -29,6 +29,7 @@ import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.utils.AbstractIterator;
public class MutationSummary
{
@@ -324,6 +325,22 @@ public class MutationSummary
}
}
+ public Iterator<Offsets> onlyUnreconciled()
+ {
+ return new AbstractIterator<>()
+ {
+ int i = 0;
+
+ @Override
+ protected Offsets computeNext()
+ {
+ if (i >= summaries.size()) return endOfData();
+ Offsets offsets = summaries.get(i++).unreconciled;
+ return offsets.isEmpty() ? computeNext() : offsets;
+ }
+ };
+ }
+
public static final IVersionedSerializer<MutationSummary> serializer = new IVersionedSerializer<>()
{
@Override
diff --git a/src/java/org/apache/cassandra/replication/MutationTrackingService.java b/src/java/org/apache/cassandra/replication/MutationTrackingService.java
index c22ac87007..7ec6758707 100644
--- a/src/java/org/apache/cassandra/replication/MutationTrackingService.java
+++ b/src/java/org/apache/cassandra/replication/MutationTrackingService.java
@@ -17,19 +17,28 @@
*/
package org.apache.cassandra.replication;
+import java.util.Collections;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.BiConsumer;
import java.util.function.Consumer;
import java.util.function.IntSupplier;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.agrona.collections.IntArrayList;
+import org.agrona.collections.IntHashSet;
import org.apache.cassandra.concurrent.ScheduledExecutorPlus;
import org.apache.cassandra.concurrent.Shutdownable;
import org.apache.cassandra.db.DecoratedKey;
@@ -37,7 +46,9 @@ import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.PartitionPosition;
import org.apache.cassandra.dht.AbstractBounds;
import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Splitter;
import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.exceptions.RequestFailure;
import org.apache.cassandra.gms.FailureDetector;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.Message;
@@ -48,11 +59,14 @@ import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.reads.tracked.TrackedLocalReads;
import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.ownership.ReplicaGroups;
+import org.apache.cassandra.tcm.ownership.VersionedEndpoints;
import org.apache.cassandra.utils.FBUtilities;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static java.util.concurrent.TimeUnit.MINUTES;
import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
import static org.apache.cassandra.concurrent.ExecutorFactory.SimulatorSemantics.NORMAL;
@@ -60,12 +74,25 @@ import static org.apache.cassandra.concurrent.ExecutorFactory.SimulatorSemantics
// TODO (expected): handle topology changes
public class MutationTrackingService
{
+ /**
+ * Split ranges into this many shards.
+ *
+ * TODO (expected): ability to rebalance / change this constant
+ */
+ private static final int SHARD_MULTIPLIER = 8;
+
private static final Logger logger = LoggerFactory.getLogger(MutationTrackingService.class);
public static final MutationTrackingService instance = new MutationTrackingService();
private final TrackedLocalReads localReads = new TrackedLocalReads();
- private final ReplicatedOffsetsBroadcaster broadcaster = new ReplicatedOffsetsBroadcaster();
- private final ConcurrentHashMap<String, KeyspaceShards> shards = new ConcurrentHashMap<>();
+ private final ConcurrentHashMap<String, KeyspaceShards> keyspaceShards = new ConcurrentHashMap<>();
+ private final ConcurrentHashMap<CoordinatorLogId, Shard> log2ShardMap = new ConcurrentHashMap<>();
+
+ private final ReplicatedOffsetsBroadcaster offsetsBroadcaster = new ReplicatedOffsetsBroadcaster();
+ private final ActiveLogReconciler activeReconciler = new ActiveLogReconciler();
+
+ private final IncomingMutations incomingMutations = new IncomingMutations();
+ private final OutgoingMutations outgoingMutations = new OutgoingMutations();
private volatile boolean started = false;
@@ -81,9 +108,11 @@ public class MutationTrackingService
for (KeyspaceMetadata keyspace : metadata.schema.getKeyspaces())
if (keyspace.useMutationTracking())
- shards.put(keyspace.name, KeyspaceShards.make(keyspace, metadata, this::nextHostLogId));
+ keyspaceShards.put(keyspace.name, KeyspaceShards.make(keyspace, metadata, this::nextHostLogId, this::onNewLog));
- broadcaster.start();
+ offsetsBroadcaster.start();
+
+ ExpiredStatePurger.instance.register(incomingMutations);
started = true;
}
@@ -95,9 +124,10 @@ public class MutationTrackingService
public void shutdownBlocking() throws InterruptedException
{
- localReads.shutdownBlocking();
- broadcaster.shutdown();
- broadcaster.awaitTermination(1, TimeUnit.MINUTES);
+ offsetsBroadcaster.shutdown();
+ offsetsBroadcaster.awaitTermination(1, TimeUnit.MINUTES);
+ activeReconciler.shutdownBlocking();
+ ExpiredStatePurger.instance.shutdownBlocking();
}
public TrackedLocalReads localReads()
@@ -107,42 +137,69 @@ public class MutationTrackingService
public MutationId nextMutationId(String keyspace, Token token)
{
- MutationId id = getOrCreate(keyspace).nextMutationId(token);
+ MutationId id = getOrCreateShards(keyspace).nextMutationId(token);
logger.trace("Created new mutation id {}", id);
return id;
}
- public void receivedWriteResponse(String keyspace, Token token, MutationId mutationId, InetAddressAndPort onHost)
+ public void sentWriteRequest(Mutation mutation, IntHashSet toHostIds)
+ {
+ Preconditions.checkArgument(!mutation.id().isNone());
+// outgoingMutations.sentWriteRequest(mutation, toHostIds);
+ }
+
+ public void receivedWriteResponse(ShortMutationId mutationId, InetAddressAndPort fromHost)
{
Preconditions.checkArgument(!mutationId.isNone());
- getOrCreate(keyspace).receivedWriteResponse(token, mutationId, onHost);
+ Shard shard = getShardNullable(mutationId);
+ // A response to the coordinator (for a forwarded write) won't have the coordinator log matching it
+ if (shard != null)
+ shard.receivedWriteResponse(mutationId, fromHost);
+// outgoingMutations.receivedWriteResponse(mutationId, ClusterMetadata.current().directory.peerId(fromHost).id());
+ }
+
+ public void retryFailedWrite(ShortMutationId mutationId, InetAddressAndPort onHost, RequestFailure reason)
+ {
+ Preconditions.checkArgument(!mutationId.isNone());
+// outgoingMutations.writeFailed(mutationId, reason, onHost);
+ activeReconciler.schedule(mutationId, onHost, ActiveLogReconciler.Priority.REGULAR);
}
public void updateReplicatedOffsets(String keyspace, Range<Token> range, List<? extends Offsets> offsets, InetAddressAndPort onHost)
{
- getOrCreate(keyspace).updateReplicatedOffsets(range, offsets, onHost);
+ getOrCreateShards(keyspace).updateReplicatedOffsets(range, offsets, onHost);
}
- public void startWriting(Mutation mutation)
+ public boolean startWriting(Mutation mutation)
{
Preconditions.checkArgument(!mutation.id().isNone());
- getOrCreate(mutation.getKeyspaceName()).startWriting(mutation);
+ return getOrCreateShards(mutation.getKeyspaceName()).startWriting(mutation);
}
public void finishWriting(Mutation mutation)
{
Preconditions.checkArgument(!mutation.id().isNone());
- getOrCreate(mutation.getKeyspaceName()).finishWriting(mutation);
+ getOrCreateShards(mutation.getKeyspaceName()).finishWriting(mutation);
+ incomingMutations.invokeListeners(mutation.id());
+ }
+
+ /**
+ * Register to be notified to an incoming mutation.
+ * @return true if this is the first active listener added for this id
+ */
+ public boolean registerMutationCallback(ShortMutationId mutationId, IncomingMutations.Callback callback)
+ {
+ return incomingMutations.subscribe(mutationId, callback);
}
public MutationSummary createSummaryForKey(DecoratedKey key, TableId tableId, boolean includePending)
{
- return getOrCreate(tableId).createSummaryForKey(key, tableId, includePending);
+ return getOrCreateShards(tableId).createSummaryForKey(key, tableId, includePending);
}
public MutationSummary createSummaryForRange(AbstractBounds<PartitionPosition> range, TableId tableId, boolean includePending)
{
- return getOrCreate(tableId).createSummaryForRange(range, tableId, includePending);
+ return getOrCreateShards(tableId).createSummaryForRange(range, tableId, includePending);
}
public MutationSummary createSummaryForRange(Range<Token> range, TableId tableId, boolean includePending)
@@ -152,25 +209,62 @@ public class MutationTrackingService
void forEachKeyspace(Consumer<KeyspaceShards> consumer)
{
- for (KeyspaceShards keyspaceShards : shards.values())
+ for (KeyspaceShards keyspaceShards : keyspaceShards.values())
consumer.accept(keyspaceShards);
}
- private KeyspaceShards getOrCreate(TableId tableId)
+ public void collectLocallyMissingMutations(MutationSummary remoteSummary, Log2OffsetsMap.Mutable into)
+ {
+ Iterator<Offsets> iterator = remoteSummary.onlyUnreconciled();
+ while (iterator.hasNext())
+ {
+ Offsets offsets = iterator.next();
+ Shard shard = getShardNullable(offsets.logId);
+ if (shard == null)
+ into.add(offsets); // if the log/shard are unknown, then all the offsets are also unkown/missing
+ else
+ shard.collectLocallyMissingMutations(offsets, into);
+ }
+ }
+
+ public void collectRemotelyMissingMutations(Offsets localOffsets, IntArrayList remoteNodeIds, Node2OffsetsMap into)
+ {
+ Shard shard = getShard(localOffsets.logId());
+ shard.collectRemotelyMissingMutations(localOffsets, remoteNodeIds, into);
+ }
+
+ public void requestMissingMutations(Offsets offsets, InetAddressAndPort forHost)
+ {
+ activeReconciler.schedule(offsets, forHost, ActiveLogReconciler.Priority.HIGH);
+ }
+
+ @Nullable
+ private Shard getShardNullable(CoordinatorLogId logId)
+ {
+ return log2ShardMap.get(logId);
+ }
+
+ @Nonnull
+ private Shard getShard(CoordinatorLogId logId)
+ {
+ return Preconditions.checkNotNull(log2ShardMap.get(logId));
+ }
+
+ private KeyspaceShards getOrCreateShards(TableId tableId)
{
//noinspection DataFlowIssue
- return getOrCreate(Schema.instance.getTableMetadata(tableId).keyspace);
+ return getOrCreateShards(Schema.instance.getTableMetadata(tableId).keyspace);
}
- private KeyspaceShards getOrCreate(String keyspace)
+ private KeyspaceShards getOrCreateShards(String keyspace)
{
- KeyspaceShards ks = shards.get(keyspace);
+ KeyspaceShards ks = keyspaceShards.get(keyspace);
if (ks != null)
return ks;
ClusterMetadata csm = ClusterMetadata.current();
KeyspaceMetadata ksm = csm.schema.getKeyspaceMetadata(keyspace);
- return shards.computeIfAbsent(keyspace, ignore -> KeyspaceShards.make(ksm, csm, this::nextHostLogId));
+ return keyspaceShards.computeIfAbsent(keyspace, ignore -> KeyspaceShards.make(ksm, csm, this::nextHostLogId, this::onNewLog));
}
// TODO (expected): durability
@@ -185,8 +279,8 @@ public class MutationTrackingService
// Could pass through SSTable bounds to exclude shards for non-overlapping ranges, but this will mostly be
// called on flush for L0 SSTables with wide bounds.
- KeyspaceShards keyspaceShards = shards.get(keyspace);
- if (keyspaceShards == null)
+ KeyspaceShards shards = keyspaceShards.get(keyspace);
+ if (shards == null)
{
logger.debug("Could not find shards for keyspace {}", keyspace);
return false;
@@ -195,7 +289,7 @@ public class MutationTrackingService
for (Long logId : logOffsets)
{
CoordinatorLogId coordinatorLogId = new CoordinatorLogId(logId);
- CoordinatorLog log = keyspaceShards.logs.get(coordinatorLogId);
+ CoordinatorLog log = shards.logs.get(coordinatorLogId);
if (log == null)
{
logger.warn("Could not determine lifecycle for unknown logId {}, not marking as durably reconciled", coordinatorLogId);
@@ -208,41 +302,66 @@ public class MutationTrackingService
return true;
}
+ // TODO (expected): when topology and state truncation is implemented, implement cleanup of this map as well
+ private void onNewLog(Shard shard, CoordinatorLog log)
+ {
+ log2ShardMap.put(log.logId, shard);
+ }
+
private static class KeyspaceShards implements Shard.Subscriber
{
private final String keyspace;
private final Map<Range<Token>, Shard> shards;
+ private final ReplicaGroups groups;
+ private final BiConsumer<Shard, CoordinatorLog> onNewLog;
private transient final Map<Range<PartitionPosition>, Shard> ppShards;
private transient final Map<CoordinatorLogId, CoordinatorLog> logs;
- static KeyspaceShards make(KeyspaceMetadata keyspace, ClusterMetadata cluster, IntSupplier logIdProvider)
+ static KeyspaceShards make(KeyspaceMetadata keyspace, ClusterMetadata cluster, IntSupplier logIdProvider, BiConsumer<Shard, CoordinatorLog> onNewLog)
{
Preconditions.checkArgument(keyspace.params.replicationType.isTracked());
+
Map<Range<Token>, Shard> shards = new HashMap<>();
- cluster.placements.get(keyspace.params.replication).writes.forEach((tokenRange, forRange) -> {
+ Map<Range<Token>, VersionedEndpoints.ForRange> groups = new HashMap<>();
+
+ cluster.placements.get(keyspace.params.replication).writes.forEach((fullTokenRange, forRange) -> {
if (!forRange.endpoints().contains(FBUtilities.getBroadcastAddressAndPort()))
return;
- IntArrayList participants = new IntArrayList(forRange.size(), IntArrayList.DEFAULT_NULL_VALUE);
+
+ IntArrayList participantList = new IntArrayList(forRange.size(), IntArrayList.DEFAULT_NULL_VALUE);
for (InetAddressAndPort endpoint : forRange.endpoints())
- participants.add(cluster.directory.peerId(endpoint).id());
- Shard shard = new Shard(keyspace.name, tokenRange, cluster.myNodeId().id(), new Participants(participants), forRange.lastModified(), logIdProvider);
- shards.put(tokenRange, shard);
+ participantList.add(cluster.directory.peerId(endpoint).id());
+ Participants participants = new Participants(participantList);
+
+ Optional<Splitter> splitter = fullTokenRange.left.getPartitioner().splitter();
+ Set<Range<Token>> ranges = splitter.isPresent() && SHARD_MULTIPLIER > 1
+ ? splitter.get().split(fullTokenRange, SHARD_MULTIPLIER)
+ : Collections.singleton(fullTokenRange);
+
+ for (Range<Token> tokenRange : ranges)
+ {
+ shards.put(tokenRange, new Shard(keyspace.name, tokenRange, cluster.myNodeId().id(), participants, forRange.lastModified(), logIdProvider, onNewLog));
+ groups.put(tokenRange, forRange.map(original -> original.withRange(tokenRange)));
+ }
});
- return new KeyspaceShards(keyspace.name, shards);
+ return new KeyspaceShards(keyspace.name, shards, new ReplicaGroups(groups), onNewLog);
}
- KeyspaceShards(String keyspace, Map<Range<Token>, Shard> shards)
+ KeyspaceShards(String keyspace, Map<Range<Token>, Shard> shards, ReplicaGroups groups, BiConsumer<Shard, CoordinatorLog> onNewLog)
{
this.keyspace = keyspace;
this.shards = shards;
+ this.groups = groups;
+ this.onNewLog = onNewLog;
- this.ppShards = new HashMap<>();
this.logs = new HashMap<>();
+ HashMap<Range<PartitionPosition>, Shard> ppShards = new HashMap<>();
shards.forEach((range, shard) -> {
ppShards.put(Range.makeRowRange(range), shard);
shard.addSubscriber(this);
});
+ this.ppShards = ppShards;
}
MutationId nextMutationId(Token token)
@@ -250,19 +369,14 @@ public class MutationTrackingService
return lookUp(token).nextId();
}
- void receivedWriteResponse(Token token, MutationId mutationId, InetAddressAndPort onHost)
- {
- lookUp(token).receivedWriteResponse(mutationId, onHost);
- }
-
void updateReplicatedOffsets(Range<Token> range, List<? extends Offsets> offsets, InetAddressAndPort onHost)
{
shards.get(range).updateReplicatedOffsets(offsets, onHost);
}
- void startWriting(Mutation mutation)
+ boolean startWriting(Mutation mutation)
{
- lookUp(mutation).startWriting(mutation);
+ return lookUp(mutation).startWriting(mutation);
}
void finishWriting(Mutation mutation)
@@ -313,10 +427,7 @@ public class MutationTrackingService
Shard lookUp(Token token)
{
- ClusterMetadata csm = ClusterMetadata.current();
- KeyspaceMetadata ksm = csm.schema.getKeyspaceMetadata(keyspace);
- Range<Token> range = ClusterMetadata.current().placements.get(ksm.params.replication).writes.forRange(token).range();
- return shards.get(range);
+ return shards.get(groups.forRange(token).range());
}
@Override
@@ -372,6 +483,15 @@ public class MutationTrackingService
return executor.awaitTermination(timeout, units);
}
+ public void shutdownBlocking() throws InterruptedException
+ {
+ if (!executor.isTerminated())
+ {
+ executor.shutdown();
+ executor.awaitTermination(1, MINUTES);
+ }
+ }
+
@Override
public void run()
{
@@ -385,11 +505,11 @@ public class MutationTrackingService
private void run(Shard shard)
{
- ShardReplicatedOffsets replicatedOffsets = shard.collectReplicatedOffsets();
+ BroadcastLogOffsets replicatedOffsets = shard.collectReplicatedOffsets();
if (replicatedOffsets.isEmpty())
return;
- Message<ShardReplicatedOffsets> message = Message.out(Verb.BROADCAST_LOG_OFFSETS, replicatedOffsets);
+ Message<BroadcastLogOffsets> message = Message.out(Verb.BROADCAST_LOG_OFFSETS, replicatedOffsets);
for (InetAddressAndPort target : shard.remoteReplicas())
if (FailureDetector.instance.isAlive(target))
@@ -400,6 +520,18 @@ public class MutationTrackingService
@VisibleForTesting
public void broadcastOffsetsForTesting()
{
- broadcaster.run();
+ offsetsBroadcaster.run();
+ }
+
+ @VisibleForTesting
+ public void pauseActiveReconciler()
+ {
+ activeReconciler.pauseForTesting();
+ }
+
+ @VisibleForTesting
+ public void resumeActiveReconciler()
+ {
+ activeReconciler.resumeForTesting();
}
}
diff --git a/src/java/org/apache/cassandra/replication/UnreconciledMutations.java b/src/java/org/apache/cassandra/replication/Node2OffsetsMap.java
similarity index 52%
copy from src/java/org/apache/cassandra/replication/UnreconciledMutations.java
copy to src/java/org/apache/cassandra/replication/Node2OffsetsMap.java
index a84bdc5162..12fa18c98a 100644
--- a/src/java/org/apache/cassandra/replication/UnreconciledMutations.java
+++ b/src/java/org/apache/cassandra/replication/Node2OffsetsMap.java
@@ -17,25 +17,32 @@
*/
package org.apache.cassandra.replication;
-import org.apache.cassandra.db.Mutation;
-import org.apache.cassandra.db.PartitionPosition;
-import org.apache.cassandra.dht.AbstractBounds;
-import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.schema.TableId;
+import org.agrona.collections.Int2ObjectHashMap;
+import org.agrona.collections.IntObjConsumer;
/**
- * Tracks unreconciled local mutations - the subset of all unreconciled mutations
- * that have been witnessed, or are currently being written to, on the local node.
+ * A mapping of nodes to Offsets; currently we only need an internally mutable implementation.
*/
-interface UnreconciledMutations
+public class Node2OffsetsMap
{
- void startWriting(Mutation mutation);
+ private final Int2ObjectHashMap<Offsets> map = new Int2ObjectHashMap<>();
- void finishWriting(Mutation mutation);
+ public void add(int node, Offsets offsets)
+ {
+ Offsets.Mutable current = (Offsets.Mutable) map.get(node);
+ if (current != null)
+ current.addAll(offsets);
+ else
+ map.put(node, Offsets.Mutable.copy(offsets));
+ }
- void remove(int offset);
+ public void forEach(IntObjConsumer<Offsets> consumer)
+ {
+ map.forEachInt(consumer);
+ }
- boolean collect(Token token, TableId tableId, boolean includePending, Offsets.OffsetReciever into);
-
- boolean collect(AbstractBounds<PartitionPosition> range, TableId tableId, boolean includePending, Offsets.OffsetReciever into);
+ public void clear()
+ {
+ map.clear();
+ }
}
diff --git a/src/java/org/apache/cassandra/replication/Offsets.java b/src/java/org/apache/cassandra/replication/Offsets.java
index 28e2d978f7..bd7204fc8a 100644
--- a/src/java/org/apache/cassandra/replication/Offsets.java
+++ b/src/java/org/apache/cassandra/replication/Offsets.java
@@ -27,6 +27,8 @@ import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.utils.AbstractIterator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Arrays;
@@ -35,6 +37,7 @@ import java.util.Iterator;
public abstract class Offsets implements Iterable<ShortMutationId>
{
+ private static final Logger logger = LoggerFactory.getLogger(Offsets.class);
private static final int INITIAL_CAPACITY = 16;
protected final CoordinatorLogId logId;
@@ -241,8 +244,26 @@ public abstract class Offsets implements Iterable<ShortMutationId>
public void addAll(Offsets other, RangeConsumer onAdded)
{
- for (int i = 0; i < other.size; i += 2)
- add(other.bounds[i], other.bounds[i + 1], onAdded);
+ if (size == 0)
+ {
+ int minLength = Math.max(other.size, INITIAL_CAPACITY);
+ if (bounds.length < minLength)
+ bounds = new int[minLength];
+
+ System.arraycopy(other.bounds, 0, bounds, 0, other.size);
+ this.size = other.size;
+
+ if (onAdded != RangeConsumer.NONE)
+ {
+ for (int i = 0; i < size; i += 2)
+ onAdded.consume(logId, bounds[i], bounds[i + 1]);
+ }
+ }
+ else
+ {
+ for (int i = 0; i < other.size; i += 2)
+ add(other.bounds[i], other.bounds[i + 1], onAdded);
+ }
}
public void addAll(Offsets other)
@@ -319,6 +340,44 @@ public abstract class Offsets implements Iterable<ShortMutationId>
return true;
}
+ /**
+ * Remove an offset in-place
+ */
+ public void remove(int offset)
+ {
+ if (size == 0)
+ return;
+
+ int pos = Arrays.binarySearch(bounds, 0, size, offset);
+ if (pos >= 0) // offset matches one of the bounds exactly
+ {
+ if (pos - 1 >= 0 && bounds[pos - 1] == offset)
+ pos--; // normalize pos to always point to floor if this is a single-offset range
+
+ if (bounds[pos] == bounds[pos + 1]) // remove entire single-offset range
+ {
+ System.arraycopy(bounds, pos + 2, bounds, pos, size - pos - 2);
+ bounds[--size] = 0;
+ bounds[--size] = 0;
+ }
+ else if (pos % 2 == 0)
+ {
+ bounds[pos] = offset + 1; // offset is at a range start, shrink it
+ }
+ else
+ {
+ bounds[pos] = offset - 1; // offset is at a range end, shrink it
+ }
+ }
+ else if ((-pos - 1) % 2 != 0) // offset falls within an existing range, need to split it into two
+ {
+ pos = -pos - 1;
+ insert(pos + 1, offset + 1, bounds[pos]);
+ bounds[pos] = offset - 1;
+ }
+ // the offset is before all existing ranges, in-between two, or after all exsiting ranges
+ }
+
private enum AddAction
{
INSERT, MOVE, INCLUDE;
@@ -395,6 +454,17 @@ public abstract class Offsets implements Iterable<ShortMutationId>
AddAction eMerge;
{
+ // if the insert spans multiple ranges and the inserted end value
+ // is less than the start of eRange - 1 (ie: cant' be merged), decrement
+ // the range by one so we can extend the previous range
+ if (eRange > sRange) {
+ int rStart = bounds[rangeStart(eRange)];
+
+ if (end < rStart -1)
+ {
+ eRange--;
+ }
+ }
int rStart = bounds[rangeStart(eRange)];
int rEnd = bounds[rangeEnd(eRange)];
diff --git a/src/java/org/apache/cassandra/replication/OutgoingMutations.java b/src/java/org/apache/cassandra/replication/OutgoingMutations.java
new file mode 100644
index 0000000000..901722800c
--- /dev/null
+++ b/src/java/org/apache/cassandra/replication/OutgoingMutations.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.replication;
+
+import org.agrona.collections.IntHashSet;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.exceptions.RequestFailureReason;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.jctools.maps.NonBlockingHashMapLong;
+
+/**
+ * Keep track of outgoing mutations so that we don't double-send any mutations
+ * when read reconciliation writes race with regular writes and/or each other.
+ * </p>
+ * Regular writes have a callback and need to be retried;
+ * Reconciliation writes have no callback, and only need to be retried if the writing node owns the log.
+ * </p>
+ * Note: this could live inside {@link CoordinatorLog}, since we cannot ever send a mutation
+ * that we don't have an instance of {@link CoordinatorLog} for, but for symmetry with
+ * {@link IncomingMutations} we put this inside {@link MutationTrackingService}.
+ */
+class OutgoingMutations
+{
+ private final NonBlockingHashMapLong<LogOutgoingMutations> log2MutationsMap = new NonBlockingHashMapLong<>();
+
+ void sentWriteRequest(Mutation mutation, IntHashSet toHostIds)
+ {
+ getOrCreate(mutation.id()).sentWriteRequest(mutation, toHostIds);
+ }
+
+ void receivedWriteResponse(ShortMutationId mutationId, int fromHostId)
+ {
+ getOrCreate(mutationId).receivedWriteResponse(mutationId, fromHostId);
+ }
+
+ void writeFailed(ShortMutationId mutationId, RequestFailureReason reason, InetAddressAndPort onHost)
+ {
+ getOrCreate(mutationId).writeFailed(mutationId, reason, onHost);
+ }
+
+ private LogOutgoingMutations getOrCreate(ShortMutationId mutationId)
+ {
+ LogOutgoingMutations mutations = log2MutationsMap.get(mutationId.logId());
+ if (mutations == null)
+ mutations = log2MutationsMap.computeIfAbsent(mutationId.logId(), ignore -> new LogOutgoingMutations());
+ return mutations;
+ }
+
+ private static class LogOutgoingMutations
+ {
+ // {host, offset} -> per-participant mutation state map
+ private final NonBlockingHashMapLong<OutgoingState> states = new NonBlockingHashMapLong<>();
+
+ void sentWriteRequest(Mutation mutation, IntHashSet toHostIds)
+ {
+ IntHashSet.IntIterator iterator = toHostIds.iterator();
+ while (iterator.hasNext())
+ {
+ OutgoingState state = getOrCreateState(iterator.nextValue(), mutation.id().offset());
+ state.setInFlight();
+ }
+ }
+
+ void receivedWriteResponse(ShortMutationId mutationId, int fromHostId)
+ {
+ OutgoingState state = remove(fromHostId, mutationId.offset());
+ if (state != null)
+ state.receivedWriteResponse();
+ }
+
+ void writeFailed(ShortMutationId mutationId, RequestFailureReason reason, InetAddressAndPort onHost)
+ {
+ // TODO check if need to implement this
+ }
+
+ // TODO FIXME: proper implementation
+ OutgoingState getOrCreateState(int hostId, int offset)
+ {
+ long key = key(hostId, offset);
+ OutgoingState state = states.get(key);
+ if (state != null)
+ return state;
+ return states.putIfAbsent(key, new OutgoingState());
+ }
+
+ OutgoingState remove(int hostId, int offset)
+ {
+ long key = key(hostId, offset);
+ return states.remove(key);
+ }
+
+ private long key(int hostId, int offset)
+ {
+ return ((long) hostId << 32) | offset;
+ }
+
+ private enum OutgoingStatus
+ {
+ IN_FLIGHT, ENQUEUED, DELIVERED
+ }
+
+ private static class OutgoingState
+ {
+ // TODO: callbacks for completion or failure
+ private volatile OutgoingStatus status;
+
+ void setInFlight()
+ {
+ status = OutgoingStatus.IN_FLIGHT;
+ }
+
+ void setEnqueued()
+ {
+ status = OutgoingStatus.ENQUEUED;
+ }
+
+ void receivedWriteResponse()
+ {
+ status = OutgoingStatus.DELIVERED;
+ // TODO: invoke all callbacks
+ }
+ }
+
+ private static class InFlight extends OutgoingState
+ {
+ }
+
+ private static class Queued extends OutgoingState
+ {
+ }
+
+ public interface Callback
+ {
+ void onSuccess();
+
+ void onFailure();
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/replication/PullMutationsRequest.java b/src/java/org/apache/cassandra/replication/PullMutationsRequest.java
new file mode 100644
index 0000000000..ec5632ab5b
--- /dev/null
+++ b/src/java/org/apache/cassandra/replication/PullMutationsRequest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.replication;
+
+import java.io.IOException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.IVerbHandler;
+import org.apache.cassandra.net.Message;
+
+public final class PullMutationsRequest
+{
+ private static final Logger logger = LoggerFactory.getLogger(PullMutationsRequest.class);
+
+ private final Offsets.Immutable offsets;
+
+ public PullMutationsRequest(Offsets.Immutable offsets)
+ {
+ this.offsets = offsets;
+ }
+
+ public static IVersionedSerializer<PullMutationsRequest> serializer = new IVersionedSerializer<>()
+ {
+ @Override
+ public void serialize(PullMutationsRequest pull, DataOutputPlus out, int version) throws IOException
+ {
+ Offsets.serializer.serialize(pull.offsets, out, version);
+ }
+
+ @Override
+ public PullMutationsRequest deserialize(DataInputPlus in, int version) throws IOException
+ {
+ return new PullMutationsRequest(Offsets.serializer.deserialize(in, version));
+ }
+
+ @Override
+ public long serializedSize(PullMutationsRequest pull, int version)
+ {
+ return Offsets.serializer.serializedSize(pull.offsets, version);
+ }
+ };
+
+ public static IVerbHandler<PullMutationsRequest> verbHandler = new IVerbHandler<>()
+ {
+ @Override
+ public void doVerb(Message<PullMutationsRequest> message)
+ {
+ InetAddressAndPort forHost = message.from();
+ Offsets offsets = message.payload.offsets;
+ logger.trace("Received pull mutations request from {} for {}", forHost, offsets);
+ MutationTrackingService.instance.requestMissingMutations(offsets, forHost);
+ }
+ };
+}
diff --git a/src/java/org/apache/cassandra/replication/PushMutationRequest.java b/src/java/org/apache/cassandra/replication/PushMutationRequest.java
new file mode 100644
index 0000000000..8e673816b5
--- /dev/null
+++ b/src/java/org/apache/cassandra/replication/PushMutationRequest.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.replication;
+
+import java.io.IOException;
+
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.exceptions.WriteTimeoutException;
+import org.apache.cassandra.io.IVersionedAsymmetricSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.journal.RecordPointer;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.IVerbHandler;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.NoPayload;
+import org.apache.cassandra.tracing.Tracing;
+
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static org.apache.cassandra.db.commitlog.CommitLogSegment.ENTRY_OVERHEAD_SIZE;
+import static org.apache.cassandra.utils.MonotonicClock.Global.approxTime;
+
+public interface PushMutationRequest
+{
+ Logger logger = LoggerFactory.getLogger(PushMutationRequest.class);
+
+ long serializedSize(int version);
+ void serialize(DataOutputPlus out, int version) throws IOException;
+
+ class Referenced implements PushMutationRequest
+ {
+ private final ShortMutationId id;
+ private final RecordPointer pointer;
+
+ Referenced(ShortMutationId id, RecordPointer pointer)
+ {
+ this.id = id;
+ this.pointer = pointer;
+ }
+
+ @Override
+ public long serializedSize(int version)
+ {
+ // TODO (expected): handle mismatched (messaging) versions
+ int size = MutationJournal.instance.sizeOfRecord(pointer);
+ Preconditions.checkState(size > 0, "Couldn't read mutation %s size from the mutation journal", id);
+ return size;
+ }
+
+ @Override
+ public void serialize(DataOutputPlus out, int version) throws IOException
+ {
+ boolean read = MutationJournal.instance.read(pointer, (segment, position, key, buffer, userVersion) ->
+ {
+ try
+ {
+ out.write(buffer); // TODO (expected): handle mismatched (messaging) versions
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException(e);
+ }
+ });
+
+ if (!read) throw new IllegalStateException("Couldn't find mutation " + id + " in the mutation journal");
+ }
+ }
+
+ class Materialized implements PushMutationRequest
+ {
+ public final Mutation mutation;
+
+ Materialized(Mutation mutation)
+ {
+ this.mutation = mutation;
+ }
+
+ @Override
+ public long serializedSize(int version)
+ {
+ return Mutation.serializer.serializedSize(mutation, version);
+ }
+
+ @Override
+ public void serialize(DataOutputPlus out, int version) throws IOException
+ {
+ Mutation.serializer.serialize(mutation, out, version);
+
+ }
+
+ static Materialized deserialize(DataInputPlus in, int version) throws IOException
+ {
+ return new Materialized(Mutation.serializer.deserialize(in, version));
+ }
+ }
+
+ IVersionedAsymmetricSerializer<PushMutationRequest, Materialized> serializer = new IVersionedAsymmetricSerializer<>()
+ {
+ @Override
+ public long serializedSize(PushMutationRequest mutation, int version)
+ {
+ return mutation.serializedSize(version);
+ }
+
+ @Override
+ public void serialize(PushMutationRequest mutation, DataOutputPlus out, int version) throws IOException
+ {
+ mutation.serialize(out, version);
+ }
+
+ @Override
+ public Materialized deserialize(DataInputPlus in, int version) throws IOException
+ {
+ return Materialized.deserialize(in, version);
+ }
+ };
+
+ IVerbHandler<Materialized> verbHandler = new IVerbHandler<>()
+ {
+ @Override
+ public void doVerb(Message<Materialized> message)
+ {
+ if (approxTime.now() > message.expiresAtNanos())
+ {
+ Tracing.trace("Discarding mutation from {} (timed out)", message.from());
+ MessagingService.instance().metrics.recordDroppedMessage(message, message.elapsedSinceCreated(NANOSECONDS), NANOSECONDS);
+ return;
+ }
+
+ // TODO (expected): here and elsewhere, use Journal's size validation, not CommitLog's
+ message.payload.mutation.validateSize(MessagingService.current_version, ENTRY_OVERHEAD_SIZE);
+
+ try
+ {
+ applyMutation(message);
+ }
+ catch (WriteTimeoutException wto)
+ {
+ failed();
+ }
+ }
+
+ private void applyMutation(Message<Materialized> message)
+ {
+ Message<NoPayload> response = message.emptyResponse();
+ InetAddressAndPort respondTo = message.respondTo();
+ message.payload.mutation.applyFuture().addCallback(o -> respond(response, respondTo), wto -> failed());
+ }
+
+ private void respond(Message<NoPayload> response, InetAddressAndPort respondTo)
+ {
+ Tracing.trace("Enqueuing response to {}", respondTo);
+ logger.trace("Enqueuing response to {}", respondTo);
+ MessagingService.instance().send(response, respondTo);
+ }
+
+ private void failed()
+ {
+ Tracing.trace("Payload application resulted in WriteTimeout, not replying");
+ }
+ };
+}
diff --git a/src/java/org/apache/cassandra/replication/ReconciliationPlan.java b/src/java/org/apache/cassandra/replication/ReconciliationPlan.java
deleted file mode 100644
index ac085bf839..0000000000
--- a/src/java/org/apache/cassandra/replication/ReconciliationPlan.java
+++ /dev/null
@@ -1,210 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.replication;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableMap;
-
-import org.apache.cassandra.db.TypeSizes;
-import org.apache.cassandra.io.IVersionedSerializer;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.replication.MutationSummary.CoordinatorSummary;
-
-public class ReconciliationPlan
-{
- private final ImmutableMap<InetAddressAndPort, Log2OffsetsMap.Immutable> txPlan;
-
- public ReconciliationPlan(ImmutableMap<InetAddressAndPort, Log2OffsetsMap.Immutable> txPlan)
- {
- this.txPlan = txPlan;
- }
-
- public Set<InetAddressAndPort> nodes()
- {
- return txPlan.keySet();
- }
-
- public Log2OffsetsMap.Immutable peerReconciliation(InetAddressAndPort to)
- {
- return txPlan.get(to);
- }
-
- public Log2OffsetsMap.Immutable offsetsFor(InetAddressAndPort node)
- {
- return txPlan.get(node);
- }
-
- public boolean isEmpty()
- {
- return txPlan.isEmpty();
- }
-
- private static class PlanBuilder
- {
- final InetAddressAndPort node;
-
- final MutationSummary summary;
- final Map<InetAddressAndPort, Log2OffsetsMap.Immutable.Builder> peerReconciliations = new HashMap<>();
-
- public PlanBuilder(InetAddressAndPort node, MutationSummary summary)
- {
- this.node = node;
- this.summary = summary;
- }
-
- public void send(InetAddressAndPort to, Offsets sequenceIds)
- {
- peerReconciliations.computeIfAbsent(to, ep -> new Log2OffsetsMap.Immutable.Builder()).add(sequenceIds);
- }
-
- ReconciliationPlan build()
- {
- ImmutableMap.Builder<InetAddressAndPort, Log2OffsetsMap.Immutable> builder = ImmutableMap.builder();
- peerReconciliations.forEach((to, ids) -> builder.put(to, ids.build()));
- return new ReconciliationPlan(builder.build());
- }
- }
-
- // TODO (desired): rework Offsets set logic usage to use Offsets#RangeIterator instead of rematerializing sets
- private static class CoordinatorLogReconciliation
- {
- final CoordinatorLogId logId;
- Offsets.Immutable reconciled;
- Offsets.Immutable unreconciled;
-
- Map<InetAddressAndPort, Offsets> unreconciledNodes = new HashMap<>();
-
- CoordinatorLogReconciliation(CoordinatorLogId logId)
- {
- this.logId = logId;
- }
-
- void addPeerSummary(InetAddressAndPort peer, CoordinatorSummary summary)
- {
- Preconditions.checkArgument(summary.logId().equals(logId));
- reconciled = Offsets.Immutable.union(reconciled, summary.reconciled);
- unreconciled = Offsets.Immutable.union(unreconciled, summary.unreconciled);
- unreconciledNodes.put(peer, summary.unreconciled);
- }
-
- void createPlan(Map<InetAddressAndPort, PlanBuilder> plan)
- {
- // remove reconciled ids
- Offsets.Immutable allIds = Offsets.Immutable.difference(unreconciled, reconciled);
- for (InetAddressAndPort receiver : plan.keySet())
- {
- Offsets.Immutable missing = Offsets.Immutable.difference(allIds, unreconciledNodes.get(receiver));
- if (missing.isEmpty())
- continue;
-
- // TODO: look into more intelligent ways to distribute mutation requests
- for (Map.Entry<InetAddressAndPort, Offsets> sender : unreconciledNodes.entrySet())
- {
- if (sender.getKey().equals(receiver))
- continue;
-
- Offsets senderIds = sender.getValue();
- PlanBuilder senderPlan = plan.get(sender.getKey());
-
- Offsets.Immutable requestedIds = Offsets.Immutable.intersection(missing, senderIds);
- senderPlan.send(receiver, requestedIds);
-
- missing = Offsets.Immutable.difference(missing, requestedIds);
- if (missing.rangeCount() == 0)
- break;
- }
- }
- }
- }
-
- public static Map<InetAddressAndPort, ReconciliationPlan> calculateReconciliation(Map<InetAddressAndPort, MutationSummary> summaries)
- {
- Map<InetAddressAndPort, PlanBuilder> planBuilders = new HashMap<>();
- Map<CoordinatorLogId, CoordinatorLogReconciliation> coordinatorReconciliations = new HashMap<>();
-
- // organize data by peer and log id
- summaries.forEach((node, summary) -> {
-
- planBuilders.put(node, new PlanBuilder(node, summary));
-
- for (int i=0; i<summary.size(); i++)
- {
- CoordinatorSummary coordinatorSummary = summary.get(i);
- CoordinatorLogReconciliation reconciliation = coordinatorReconciliations.computeIfAbsent(coordinatorSummary.logId(), CoordinatorLogReconciliation::new);
- reconciliation.addPeerSummary(node, coordinatorSummary);
- }
- });
-
- coordinatorReconciliations.values().forEach(planBuilder -> planBuilder.createPlan(planBuilders));
-
- Map<InetAddressAndPort, ReconciliationPlan> plans = new HashMap<>();
- planBuilders.forEach((node, planBuilder) -> {
- ReconciliationPlan plan = planBuilder.build();
- if (!plan.isEmpty())
- plans.put(node, plan);
- });
- return plans;
- }
-
- public static final IVersionedSerializer<ReconciliationPlan> serializer = new IVersionedSerializer<>()
- {
- @Override
- public void serialize(ReconciliationPlan plan, DataOutputPlus out, int version) throws IOException
- {
- out.writeInt(plan.txPlan.size());
- for (Map.Entry<InetAddressAndPort, Log2OffsetsMap.Immutable> entry : plan.txPlan.entrySet())
- {
- InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serialize(entry.getKey(), out, version);
- Log2OffsetsMap.Immutable.serializer.serialize(entry.getValue(), out, version);
- }
- }
-
- @Override
- public ReconciliationPlan deserialize(DataInputPlus in, int version) throws IOException
- {
- int size = in.readInt();
- ImmutableMap.Builder<InetAddressAndPort, Log2OffsetsMap.Immutable> builder = ImmutableMap.builderWithExpectedSize(size);
- for (int i = 0; i < size; i++)
- {
- InetAddressAndPort endpoint = InetAddressAndPort.Serializer.inetAddressAndPortSerializer.deserialize(in, version);
- Log2OffsetsMap.Immutable offsets = Log2OffsetsMap.Immutable.serializer.deserialize(in, version);
- builder.put(endpoint, offsets);
- }
- return new ReconciliationPlan(builder.build());
- }
-
- @Override
- public long serializedSize(ReconciliationPlan plan, int version)
- {
- long size = TypeSizes.sizeof(plan.txPlan.size());
- for (Map.Entry<InetAddressAndPort, Log2OffsetsMap.Immutable> entry : plan.txPlan.entrySet())
- {
- size += InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serializedSize(entry.getKey(), version);
- size += Log2OffsetsMap.Immutable.serializer.serializedSize(entry.getValue(), version);
- }
- return size;
- }
- };
-}
diff --git a/src/java/org/apache/cassandra/replication/Shard.java b/src/java/org/apache/cassandra/replication/Shard.java
index 059ed84546..6588bede09 100644
--- a/src/java/org/apache/cassandra/replication/Shard.java
+++ b/src/java/org/apache/cassandra/replication/Shard.java
@@ -19,17 +19,20 @@ package org.apache.cassandra.replication;
import java.util.ArrayList;
import java.util.List;
+import java.util.function.BiConsumer;
import java.util.function.IntSupplier;
+import javax.annotation.Nonnull;
+
import com.google.common.base.Preconditions;
+import org.agrona.collections.IntArrayList;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.PartitionPosition;
import org.apache.cassandra.dht.AbstractBounds;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.replication.CoordinatorLog.CoordinatorLogPrimary;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.Epoch;
import org.apache.cassandra.tcm.membership.NodeId;
@@ -42,11 +45,26 @@ public class Shard
private final int localHostId;
private final Participants participants;
private final Epoch sinceEpoch;
+ private final BiConsumer<Shard, CoordinatorLog> onNewLog;
private final NonBlockingHashMapLong<CoordinatorLog> logs;
// TODO (expected): add support for log rotation
- private final CoordinatorLogPrimary currentLocalLog;
+ private final CoordinatorLog.CoordinatorLogPrimary currentLocalLog;
+
+ private final List<Subscriber> subscribers = new ArrayList<>();
+
+ public interface Subscriber
+ {
+ default void onLogCreation(CoordinatorLog log) {}
+ default void onSubscribe(CoordinatorLog currentLog) {}
+ }
- Shard(String keyspace, Range<Token> tokenRange, int localHostId, Participants participants, Epoch sinceEpoch, IntSupplier logIdProvider)
+ Shard(String keyspace,
+ Range<Token> tokenRange,
+ int localHostId,
+ Participants participants,
+ Epoch sinceEpoch,
+ IntSupplier logIdProvider,
+ BiConsumer<Shard, CoordinatorLog> onNewLog)
{
Preconditions.checkArgument(participants.contains(localHostId));
@@ -56,6 +74,7 @@ public class Shard
this.participants = participants;
this.sinceEpoch = sinceEpoch;
this.logs = new NonBlockingHashMapLong<>();
+ this.onNewLog = onNewLog;
this.currentLocalLog = startNewLog(localHostId, logIdProvider.getAsInt(), participants);
CoordinatorLogId logId = currentLocalLog.logId;
Preconditions.checkArgument(!logId.isNone());
@@ -67,10 +86,10 @@ public class Shard
return currentLocalLog.nextId();
}
- void receivedWriteResponse(MutationId mutationId, InetAddressAndPort onHost)
+ void receivedWriteResponse(ShortMutationId mutationId, InetAddressAndPort fromHost)
{
- int onHostId = ClusterMetadata.current().directory.peerId(onHost).id();
- getOrCreate(mutationId).receivedWriteResponse(mutationId, onHostId);
+ int fromHostId = ClusterMetadata.current().directory.peerId(fromHost).id();
+ getOrCreate(mutationId).receivedWriteResponse(mutationId, fromHostId);
}
void updateReplicatedOffsets(List<? extends Offsets> offsets, InetAddressAndPort onHost)
@@ -80,14 +99,14 @@ public class Shard
getOrCreate(logOffsets.logId()).updateReplicatedOffsets(logOffsets, onHostId);
}
- void startWriting(Mutation mutation)
+ boolean startWriting(Mutation mutation)
{
- getOrCreate(mutation.id()).startWriting(mutation);
+ return getOrCreate(mutation).startWriting(mutation);
}
void finishWriting(Mutation mutation)
{
- getOrCreate(mutation.id()).finishWriting(mutation);
+ getOrCreate(mutation).finishWriting(mutation);
}
void addSummaryForKey(Token token, boolean includePending, MutationSummary.Builder builder)
@@ -106,6 +125,18 @@ public class Shard
});
}
+ void collectLocallyMissingMutations(Offsets remoteOffsets, Log2OffsetsMap.Mutable into)
+ {
+ CoordinatorLog log = get(remoteOffsets.logId());
+ log.collectLocallyMissingMutations(remoteOffsets, into);
+ }
+
+ void collectRemotelyMissingMutations(Offsets localOffsets, IntArrayList remoteNodeIds, Node2OffsetsMap into)
+ {
+ CoordinatorLog log = get(localOffsets.logId());
+ log.collectRemotelyMissingMutations(localOffsets, remoteNodeIds, into);
+ }
+
List<InetAddressAndPort> remoteReplicas()
{
List<InetAddressAndPort> replicas = new ArrayList<>(participants.size() - 1);
@@ -121,7 +152,7 @@ public class Shard
/**
* Collects replicated offsets for the logs owned by this coordinator on this shard.
*/
- ShardReplicatedOffsets collectReplicatedOffsets()
+ BroadcastLogOffsets collectReplicatedOffsets()
{
List<Offsets.Immutable> offsets = new ArrayList<>();
for (CoordinatorLog log : logs.values())
@@ -131,17 +162,25 @@ public class Shard
offsets.add(logOffsets);
}
- return new ShardReplicatedOffsets(keyspace, tokenRange, offsets);
+ return new BroadcastLogOffsets(keyspace, tokenRange, offsets);
}
/**
* Creates a new coordinator log for this host. Primarily on Shard init (node startup or topology change).
* Also on keyspace creation.
*/
- private static CoordinatorLog.CoordinatorLogPrimary startNewLog(int localHostId, int hostLogId, Participants participants)
+ private CoordinatorLog.CoordinatorLogPrimary startNewLog(int localHostId, int hostLogId, Participants participants)
{
CoordinatorLogId logId = new CoordinatorLogId(localHostId, hostLogId);
- return new CoordinatorLog.CoordinatorLogPrimary(localHostId, logId, participants);
+ CoordinatorLog.CoordinatorLogPrimary log =
+ new CoordinatorLog.CoordinatorLogPrimary(localHostId, logId, participants);
+ onNewLog.accept(this, log);
+ return log;
+ }
+
+ private CoordinatorLog getOrCreate(Mutation mutation)
+ {
+ return getOrCreate(mutation.id());
}
private CoordinatorLog getOrCreate(MutationId mutationId)
@@ -155,28 +194,36 @@ public class Shard
return getOrCreate(logId.asLong());
}
+ @Nonnull
+ private CoordinatorLog get(CoordinatorLogId logId)
+ {
+ return Preconditions.checkNotNull(logs.get(logId.asLong()));
+ }
+
private CoordinatorLog getOrCreate(long logId)
{
CoordinatorLog log = logs.get(logId);
if (log != null)
return log;
CoordinatorLog newLog = logs.computeIfAbsent(logId, ignore -> CoordinatorLog.create(localHostId, new CoordinatorLogId(logId), participants));
+ onNewLog.accept(this, newLog);
for (Subscriber subscriber : subscribers)
subscriber.onLogCreation(newLog);
return newLog;
}
- private final List<Subscriber> subscribers = new ArrayList<>();
-
- public interface Subscriber
- {
- default void onLogCreation(CoordinatorLog log) {}
- default void onSubscribe(CoordinatorLog currentLog) {}
- }
-
public void addSubscriber(Subscriber subscriber)
{
subscriber.onSubscribe(currentLocalLog);
subscribers.add(subscriber);
}
+
+ private CoordinatorLog create(long logId)
+ {
+ CoordinatorLog log = CoordinatorLog.create(localHostId, new CoordinatorLogId(logId), participants);
+ onNewLog.accept(this, log);
+ for (Subscriber subscriber : subscribers)
+ subscriber.onLogCreation(log);
+ return log;
+ }
}
diff --git a/src/java/org/apache/cassandra/replication/TrackedWriteRequest.java b/src/java/org/apache/cassandra/replication/TrackedWriteRequest.java
index 2673e3151d..e9ee3ce7cf 100644
--- a/src/java/org/apache/cassandra/replication/TrackedWriteRequest.java
+++ b/src/java/org/apache/cassandra/replication/TrackedWriteRequest.java
@@ -28,6 +28,7 @@ import com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.agrona.collections.IntHashSet;
import org.apache.cassandra.concurrent.DebuggableTask;
import org.apache.cassandra.concurrent.Stage;
import org.apache.cassandra.config.DatabaseDescriptor;
@@ -55,6 +56,7 @@ import org.apache.cassandra.net.RequestCallback;
import org.apache.cassandra.net.Verb;
import org.apache.cassandra.service.AbstractWriteResponseHandler;
import org.apache.cassandra.service.TrackedWriteResponseHandler;
+import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.transport.Dispatcher;
import org.apache.cassandra.utils.FBUtilities;
@@ -101,10 +103,8 @@ public class TrackedWriteRequest
writeMetrics.localRequests.mark();
MutationId id = MutationTrackingService.instance.nextMutationId(keyspaceName, token);
mutation = mutation.withMutationId(id);
- TrackedWriteResponseHandler handler = TrackedWriteResponseHandler.wrap(rs.getWriteResponseHandler(plan, null, WriteType.SIMPLE, null, requestTime),
- keyspaceName,
- mutation.key().getToken(),
- id);
+ TrackedWriteResponseHandler handler =
+ TrackedWriteResponseHandler.wrap(rs.getWriteResponseHandler(plan, null, WriteType.SIMPLE, null, requestTime), id);
applyLocallyAndSendToReplicas(mutation, plan, handler);
return handler;
}
@@ -149,10 +149,10 @@ public class TrackedWriteRequest
if (message == null)
{
- Message.Builder<Mutation> builder = Message.builder(MUTATION_REQ, mutation)
+ message = Message.builder(MUTATION_REQ, mutation)
.withRequestTime(handler.getRequestTime())
- .withFlag(MessageFlag.CALL_BACK_ON_FAILURE);
- message = builder.build();
+ .withFlag(MessageFlag.CALL_BACK_ON_FAILURE)
+ .build();
}
String dc = DatabaseDescriptor.getLocator().location(destination.endpoint()).datacenter;
@@ -178,16 +178,32 @@ public class TrackedWriteRequest
Preconditions.checkState(applyLocally); // the coordinator is always a replica
applyMutationLocally(mutation, handler);
+ IntHashSet remoteReplicas = null;
+ if (localDCReplicas != null || remoteDCReplicas != null)
+ remoteReplicas = new IntHashSet();
+
if (localDCReplicas != null)
+ {
for (Replica replica : localDCReplicas)
+ {
MessagingService.instance().sendWriteWithCallback(message, replica, handler);
+ remoteReplicas.add(ClusterMetadata.current().directory.peerId(replica.endpoint()).id());
+ }
+ }
if (remoteDCReplicas != null)
{
// for each datacenter, send the message to one node to relay the write to other replicas
for (List<Replica> dcReplicas : remoteDCReplicas.values())
+ {
sendMessagesToRemoteDC(message, EndpointsForToken.copyOf(mutation.key().getToken(), dcReplicas), handler, null);
+ for (Replica replica : dcReplicas)
+ remoteReplicas.add(ClusterMetadata.current().directory.peerId(replica.endpoint()).id());
+ }
}
+
+ if (remoteReplicas != null)
+ MutationTrackingService.instance.sentWriteRequest(mutation, remoteReplicas);
}
static void applyMutationLocally(Mutation mutation, RequestCallback<NoPayload> handler)
@@ -268,9 +284,9 @@ public class TrackedWriteRequest
* Send the message to the first replica of targets, and have it forward the message to others in its DC
*/
static void sendMessagesToRemoteDC(Message<? extends IMutation> message,
- EndpointsForToken targets,
- RequestCallback<NoPayload> handler,
- ForwardedWrite.CoordinatorAckInfo ackTo)
+ EndpointsForToken targets,
+ RequestCallback<NoPayload> handler,
+ ForwardedWrite.CoordinatorAckInfo ackTo)
{
final Replica target;
diff --git a/src/java/org/apache/cassandra/replication/UnreconciledMutations.java b/src/java/org/apache/cassandra/replication/UnreconciledMutations.java
index a84bdc5162..740882b739 100644
--- a/src/java/org/apache/cassandra/replication/UnreconciledMutations.java
+++ b/src/java/org/apache/cassandra/replication/UnreconciledMutations.java
@@ -17,6 +17,16 @@
*/
package org.apache.cassandra.replication;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Sets;
+
+import org.agrona.collections.Int2ObjectHashMap;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.PartitionPosition;
import org.apache.cassandra.dht.AbstractBounds;
@@ -27,15 +37,153 @@ import org.apache.cassandra.schema.TableId;
* Tracks unreconciled local mutations - the subset of all unreconciled mutations
* that have been witnessed, or are currently being written to, on the local node.
*/
-interface UnreconciledMutations
+public class UnreconciledMutations
{
- void startWriting(Mutation mutation);
+ private final Int2ObjectHashMap<Entry> statesMap = new Int2ObjectHashMap<>();
+ private final SortedSet<Entry> statesSet = new TreeSet<>(Entry.comparator);
+
+ enum Visibility
+ {
+ PENDING, // written to the journal, but not yet to LSM
+ VISIBLE, // written to both the journal and LSM
+ }
+
+ private static final class Entry
+ {
+ private static final Comparator<Entry> comparator = (left, right) ->
+ {
+ int cmp = left.token.compareTo(right.token);
+ return (cmp != 0) ? cmp : Integer.compare(left.offset, right.offset);
+ };
+
+ final Token token;
+ final int offset;
+ final Object tableOrTables;
+ private Visibility visibility;
+
+ Entry(Token token, int offset, Object tableOrTables, Visibility visibility)
+ {
+ this.token = token;
+ this.offset = offset;
+ this.tableOrTables = tableOrTables;
+ this.visibility = visibility;
+ }
+
+ static Entry create(Mutation mutation)
+ {
+ Collection<TableId> ids = mutation.getTableIds();
+ Preconditions.checkArgument(!ids.isEmpty());
+ return new Entry(mutation.key().getToken(), mutation.id().offset(), tableOrTables(mutation), Visibility.PENDING);
+ }
+
+ private static Object tableOrTables(Mutation mutation)
+ {
+ Collection<TableId> ids = mutation.getTableIds();
+ Preconditions.checkArgument(!ids.isEmpty());
+ return ids.size() == 1 ? ids.iterator().next() : Sets.newHashSet(mutation.getTableIds());
+ }
+
+ boolean contains(TableId tableId)
+ {
+ return tableOrTables instanceof Set
+ ? ((Set<?>) tableOrTables).contains(tableId)
+ : tableId.equals(tableOrTables);
+ }
+
+ boolean isVisible()
+ {
+ return visibility == Visibility.VISIBLE;
+ }
+
+ static Entry start(Token token, boolean isInclusive)
+ {
+ return new Entry(token, isInclusive ? 0 : Integer.MAX_VALUE, null, null);
+ }
+
+ static Entry end(Token token, boolean isInclusive)
+ {
+ return new Entry(token, isInclusive ? Integer.MAX_VALUE : 0, null, null);
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (!(o instanceof Entry))
+ return false;
+ Entry that = (Entry) o;
+ return this.offset == that.offset && this.token.equals(that.token);
+ }
+ }
+
+ public void startWriting(Mutation mutation)
+ {
+ Entry entry = Entry.create(mutation);
+ statesMap.put(entry.offset, entry);
+ statesSet.add(entry);
+ }
+
+ public void finishWriting(Mutation mutation)
+ {
+ Preconditions.checkArgument(!mutation.id().isNone());
+ Entry entry = statesMap.get(mutation.id().offset());
+ if (entry == null)
+ return;
+ entry.visibility = Visibility.VISIBLE;
+ }
+
+ public void remove(int offset)
+ {
+ Entry state = statesMap.remove(offset);
+ Preconditions.checkNotNull(state);
+ statesSet.remove(state);
+ }
- void finishWriting(Mutation mutation);
+ public boolean collect(AbstractBounds<PartitionPosition> range, TableId tableId, boolean includePending, Offsets.OffsetReciever into)
+ {
+ Entry start = Entry.start(range.left.getToken(), range.left.kind() != PartitionPosition.Kind.MAX_BOUND);
+ Entry end = Entry.end(range.right.getToken(), range.right.kind() != PartitionPosition.Kind.MIN_BOUND);
+ return collect(start, end, tableId, includePending, into);
+ }
- void remove(int offset);
+ public boolean collect(Token token, TableId tableId, boolean includePending, Offsets.OffsetReciever into)
+ {
+ SortedSet<Entry> subset = statesSet.subSet(Entry.start(token, true), Entry.end(token, true));
+ return collect(subset, tableId, includePending, into);
+ }
- boolean collect(Token token, TableId tableId, boolean includePending, Offsets.OffsetReciever into);
+ private boolean collect(SortedSet<Entry> subset, TableId tableId, boolean includePending, Offsets.OffsetReciever into)
+ {
+ boolean found = false;
+ for (Entry entry : subset)
+ {
+ if (entry.contains(tableId) && (includePending || entry.isVisible()))
+ {
+ into.add(entry.offset);
+ found = true;
+ }
+ }
+ return found;
+ }
- boolean collect(AbstractBounds<PartitionPosition> range, TableId tableId, boolean includePending, Offsets.OffsetReciever into);
+ private boolean collect(Entry start, Entry end, TableId tableId, boolean includePending, Offsets.OffsetReciever into)
+ {
+ int cmp = start.token.compareTo(end.token);
+ if (cmp == 0)
+ {
+ // full range
+ return collect(statesSet, tableId, includePending, into);
+ }
+ else if (cmp > 0)
+ {
+ // wrap around range
+ boolean lFound = collect(statesSet.headSet(end), tableId, includePending, into);
+ boolean rFound = collect(statesSet.tailSet(start), tableId, includePending, into);
+ return lFound || rFound;
+ }
+ else
+ {
+ // contiguous range
+ return collect(statesSet.subSet(start, end), tableId, includePending, into);
+ }
+ }
}
diff --git a/src/java/org/apache/cassandra/replication/UnreconciledMutationsReplica.java b/src/java/org/apache/cassandra/replication/UnreconciledMutationsReplica.java
deleted file mode 100644
index 821c8db0fc..0000000000
--- a/src/java/org/apache/cassandra/replication/UnreconciledMutationsReplica.java
+++ /dev/null
@@ -1,193 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.replication;
-
-import java.util.Collection;
-import java.util.Comparator;
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.TreeSet;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Sets;
-
-import org.agrona.collections.Int2ObjectHashMap;
-import org.apache.cassandra.db.Mutation;
-import org.apache.cassandra.db.PartitionPosition;
-import org.apache.cassandra.dht.AbstractBounds;
-import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.schema.TableId;
-
-/**
- * Tracks unreconciled local mutations - the subset of all unreconciled mutations
- * that have been witnessed, or are currently being written to, on the local node.
- */
-class UnreconciledMutationsReplica implements UnreconciledMutations
-{
- private final Int2ObjectHashMap<Entry> statesMap = new Int2ObjectHashMap<>();
- private final SortedSet<Entry> statesSet = new TreeSet<>(Entry.comparator);
-
- enum Visibility
- {
- PENDING, // written to the journal, but not yet to LSM
- VISIBLE, // written to both the journal and LSM
- }
-
- private static final class Entry
- {
- private static final Comparator<Entry> comparator = (left, right) ->
- {
- int cmp = left.token.compareTo(right.token);
- return (cmp != 0) ? cmp : Integer.compare(left.offset, right.offset);
- };
-
- final Token token;
- final int offset;
- final Object tableOrTables;
- private Visibility visibility;
-
- Entry(Token token, int offset, Object tableOrTables, Visibility visibility)
- {
- this.token = token;
- this.offset = offset;
- this.tableOrTables = tableOrTables;
- this.visibility = visibility;
- }
-
- static Entry create(Mutation mutation)
- {
- Collection<TableId> ids = mutation.getTableIds();
- Preconditions.checkArgument(!ids.isEmpty());
- return new Entry(mutation.key().getToken(), mutation.id().offset(), tableOrTables(mutation), Visibility.PENDING);
- }
-
- private static Object tableOrTables(Mutation mutation)
- {
- Collection<TableId> ids = mutation.getTableIds();
- Preconditions.checkArgument(!ids.isEmpty());
- return ids.size() == 1 ? ids.iterator().next() : Sets.newHashSet(mutation.getTableIds());
- }
-
- boolean contains(TableId tableId)
- {
- return tableOrTables instanceof Set
- ? ((Set<?>) tableOrTables).contains(tableId)
- : tableId.equals(tableOrTables);
- }
-
- boolean isVisible()
- {
- return visibility == Visibility.VISIBLE;
- }
-
- static Entry start(Token token, boolean isInclusive)
- {
- return new Entry(token, isInclusive ? 0 : Integer.MAX_VALUE, null, null);
- }
-
- static Entry end(Token token, boolean isInclusive)
- {
- return new Entry(token, isInclusive ? Integer.MAX_VALUE : 0, null, null);
- }
-
- @Override
- public boolean equals(Object o)
- {
- if (!(o instanceof Entry))
- return false;
- Entry that = (Entry) o;
- return this.offset == that.offset && this.token.equals(that.token);
- }
- }
-
- @Override
- public void startWriting(Mutation mutation)
- {
- Entry entry = Entry.create(mutation);
- statesMap.put(entry.offset, entry);
- statesSet.add(entry);
- }
-
- @Override
- public void finishWriting(Mutation mutation)
- {
- Preconditions.checkArgument(!mutation.id().isNone());
- Entry entry = statesMap.get(mutation.id().offset());
- Preconditions.checkNotNull(entry);
- entry.visibility = Visibility.VISIBLE;
- }
-
- @Override
- public void remove(int offset)
- {
- Entry state = statesMap.remove(offset);
- Preconditions.checkNotNull(state);
- statesSet.remove(state);
- }
-
- @Override
- public boolean collect(Token token, TableId tableId, boolean includePending, Offsets.OffsetReciever into)
- {
- SortedSet<Entry> subset = statesSet.subSet(Entry.start(token, true), Entry.end(token, true));
- return collect(subset, tableId, includePending, into);
- }
-
- @Override
- public boolean collect(AbstractBounds<PartitionPosition> range, TableId tableId, boolean includePending, Offsets.OffsetReciever into)
- {
- Entry start = Entry.start(range.left.getToken(), range.left.kind() != PartitionPosition.Kind.MAX_BOUND);
- Entry end = Entry.end(range.right.getToken(), range.right.kind() != PartitionPosition.Kind.MIN_BOUND);
- return collect(start, end, tableId, includePending, into);
- }
-
- private boolean collect(SortedSet<Entry> subset, TableId tableId, boolean includePending, Offsets.OffsetReciever into)
- {
- boolean found = false;
- for (Entry entry : subset)
- {
- if (entry.contains(tableId) && (includePending || entry.isVisible()))
- {
- into.add(entry.offset);
- found = true;
- }
- }
- return found;
- }
-
- private boolean collect(Entry start, Entry end, TableId tableId, boolean includePending, Offsets.OffsetReciever into)
- {
- int cmp = start.token.compareTo(end.token);
- if (cmp == 0)
- {
- // full range
- return collect(statesSet, tableId, includePending, into);
- }
- else if (cmp > 0)
- {
- // wrap around range
- boolean lFound = collect(statesSet.headSet(end), tableId, includePending, into);
- boolean rFound = collect(statesSet.tailSet(start), tableId, includePending, into);
- return lFound || rFound;
- }
- else
- {
- // contiguous range
- return collect(statesSet.subSet(start, end), tableId, includePending, into);
- }
- }
-}
diff --git a/src/java/org/apache/cassandra/service/StorageProxy.java b/src/java/org/apache/cassandra/service/StorageProxy.java
index 3d41b2f955..b1559d17ba 100644
--- a/src/java/org/apache/cassandra/service/StorageProxy.java
+++ b/src/java/org/apache/cassandra/service/StorageProxy.java
@@ -2722,7 +2722,10 @@ public class StorageProxy implements StorageProxyMBean
ClusterMetadata metadata = ClusterMetadata.current();
for (int i=0; i<cmdCount; i++)
- reads[i] = TrackedRead.Partition.create(metadata, commands.get(i), consistencyLevel);
+ {
+ SinglePartitionReadCommand command = commands.get(i);
+ reads[i] = TrackedRead.Partition.create(metadata, command, consistencyLevel, requestTime);
+ }
for (TrackedRead.Partition read : reads)
read.start(requestTime);
diff --git a/src/java/org/apache/cassandra/service/TrackedWriteResponseHandler.java b/src/java/org/apache/cassandra/service/TrackedWriteResponseHandler.java
index f066182ad1..213cc90976 100644
--- a/src/java/org/apache/cassandra/service/TrackedWriteResponseHandler.java
+++ b/src/java/org/apache/cassandra/service/TrackedWriteResponseHandler.java
@@ -17,7 +17,6 @@
*/
package org.apache.cassandra.service;
-import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.RequestFailure;
import org.apache.cassandra.exceptions.WriteFailureException;
import org.apache.cassandra.exceptions.WriteTimeoutException;
@@ -31,24 +30,18 @@ public class TrackedWriteResponseHandler extends AbstractWriteResponseHandler<No
{
private final AbstractWriteResponseHandler<NoPayload> wrapped;
- private final String keyspace;
- private final Token token;
private final MutationId mutationId;
- private TrackedWriteResponseHandler(
- AbstractWriteResponseHandler<NoPayload> wrapped, String keyspace, Token token, MutationId mutationId)
+ private TrackedWriteResponseHandler(AbstractWriteResponseHandler<NoPayload> wrapped, MutationId mutationId)
{
super(wrapped.replicaPlan, wrapped.callback, wrapped.writeType, null, wrapped.getRequestTime());
this.wrapped = wrapped;
- this.keyspace = keyspace;
- this.token = token;
this.mutationId = mutationId;
}
- public static TrackedWriteResponseHandler wrap(
- AbstractWriteResponseHandler<NoPayload> handler, String keyspace, Token token, MutationId mutationId)
+ public static TrackedWriteResponseHandler wrap(AbstractWriteResponseHandler<NoPayload> handler, MutationId mutationId)
{
- return new TrackedWriteResponseHandler(handler, keyspace, token, mutationId);
+ return new TrackedWriteResponseHandler(handler, mutationId);
}
@Override
@@ -56,13 +49,14 @@ public class TrackedWriteResponseHandler extends AbstractWriteResponseHandler<No
{
// Local mutations are witnessed from Keyspace.applyInternalTracked
if (msg != null)
- MutationTrackingService.instance.receivedWriteResponse(keyspace, token, mutationId, msg.from());
+ MutationTrackingService.instance.receivedWriteResponse(mutationId, msg.from());
wrapped.onResponse(msg);
}
@Override
public void onFailure(InetAddressAndPort from, RequestFailure failure)
{
+ MutationTrackingService.instance.retryFailedWrite(mutationId, from, failure);
wrapped.onFailure(from, failure);
}
diff --git a/src/java/org/apache/cassandra/service/reads/range/RangeCommandIterator.java b/src/java/org/apache/cassandra/service/reads/range/RangeCommandIterator.java
index bf9885e224..4bb96b0128 100644
--- a/src/java/org/apache/cassandra/service/reads/range/RangeCommandIterator.java
+++ b/src/java/org/apache/cassandra/service/reads/range/RangeCommandIterator.java
@@ -22,7 +22,6 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.NoSuchElementException;
-import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import com.google.common.annotations.VisibleForTesting;
@@ -71,6 +70,7 @@ import org.apache.cassandra.utils.CloseableIterator;
import static com.google.common.base.Preconditions.checkState;
import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.readMetrics;
import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.readMetricsForLevel;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
@VisibleForTesting
@@ -342,7 +342,7 @@ public class RangeCommandIterator extends AbstractIterator<RowIterator> implemen
ReplicaPlan.ForRangeRead replicaPlan = replicaPlans.next();
PartitionRangeReadCommand rangeCommand = command.forSubRange(replicaPlan.range(), i == 0);
- TrackedRead.Range read = TrackedRead.Range.create(rangeCommand, replicaPlan);
+ TrackedRead.Range read = TrackedRead.Range.create(rangeCommand, replicaPlan, requestTime);
read.start(requestTime);
concurrentQueries.add(read.iterator());
@@ -493,7 +493,7 @@ public class RangeCommandIterator extends AbstractIterator<RowIterator> implemen
long latency = nanoTime() - requestTime.startedAtNanos();
rangeMetrics.addNano(latency);
rangeMetrics.roundTrips.update(batchesRequested);
- Keyspace.openAndGetStore(command.metadata()).metric.coordinatorScanLatency.update(latency, TimeUnit.NANOSECONDS);
+ Keyspace.openAndGetStore(command.metadata()).metric.coordinatorScanLatency.update(latency, NANOSECONDS);
}
}
diff --git a/src/java/org/apache/cassandra/service/reads/tracked/ExtendingCompletedRead.java b/src/java/org/apache/cassandra/service/reads/tracked/ExtendingCompletedRead.java
index f5fa4dc25a..88f4b496e8 100644
--- a/src/java/org/apache/cassandra/service/reads/tracked/ExtendingCompletedRead.java
+++ b/src/java/org/apache/cassandra/service/reads/tracked/ExtendingCompletedRead.java
@@ -18,6 +18,7 @@
package org.apache.cassandra.service.reads.tracked;
+import org.apache.cassandra.transport.Dispatcher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -132,7 +133,7 @@ class ExtendingCompletedRead implements PartialTrackedRead.CompletedRead
}
@Override
- public Future<TrackedDataResponse> followupRead(TrackedDataResponse initialResponse, ConsistencyLevel consistencyLevel, long expiresAtNanos)
+ public Future<TrackedDataResponse> followupRead(TrackedDataResponse initialResponse, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
{
if (!followUpRequired())
return null;
@@ -150,13 +151,14 @@ class ExtendingCompletedRead implements PartialTrackedRead.CompletedRead
Tracing.trace("Requesting {} extra rows from {} for short read protection", toQuery, FBUtilities.getBroadcastAddressAndPort());
logger.info("Requesting {} extra rows from {} for short read protection", toQuery, FBUtilities.getBroadcastAddressAndPort());
- return makeFollowupRead(initialResponse, toQuery, consistencyLevel, expiresAtNanos);
+ return makeFollowupRead(initialResponse, toQuery, consistencyLevel, requestTime);
}
- protected Future<TrackedDataResponse> makeFollowupRead(TrackedDataResponse initialResponse, int toQuery, ConsistencyLevel consistencyLevel, long expiresAtNanos)
+ protected Future<TrackedDataResponse> makeFollowupRead(TrackedDataResponse initialResponse, int toQuery, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
{
- TrackedRead.Range followUpRead = PartialTrackedRangeRead.makeFollowUpRead(command, followUpBounds, toQuery, consistencyLevel, expiresAtNanos);
- followUpRead.start(expiresAtNanos);
+ TrackedRead.Range followUpRead = PartialTrackedRangeRead.makeFollowUpRead(command, followUpBounds, toQuery, consistencyLevel, requestTime);
+ logger.trace("Short read detected, starting followup read {}", followUpRead);
+ followUpRead.start(requestTime);
AsyncPromise<TrackedDataResponse> combinedRead = new AsyncPromise<>();
followUpRead.future().addCallback((result, failure) -> {
if (failure != null)
diff --git a/src/java/org/apache/cassandra/service/reads/tracked/FilteredFollowupRead.java b/src/java/org/apache/cassandra/service/reads/tracked/FilteredFollowupRead.java
index 121ce1a581..fe87dbad0f 100644
--- a/src/java/org/apache/cassandra/service/reads/tracked/FilteredFollowupRead.java
+++ b/src/java/org/apache/cassandra/service/reads/tracked/FilteredFollowupRead.java
@@ -37,6 +37,7 @@ import org.apache.cassandra.db.partitions.PartitionIterator;
import org.apache.cassandra.dht.AbstractBounds;
import org.apache.cassandra.service.reads.tracked.PartialTrackedRangeRead.FollowUpReadInfo;
import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.transport.Dispatcher;
import org.apache.cassandra.utils.concurrent.AsyncPromise;
import org.apache.cassandra.utils.concurrent.Future;
import org.apache.cassandra.utils.concurrent.FutureCombiner;
@@ -50,7 +51,7 @@ class FilteredFollowupRead extends AsyncPromise<TrackedDataResponse>
private final TrackedDataResponse initialResponse;
private final int toQuery;
private final ConsistencyLevel consistencyLevel;
- private final long expiresAtNanos;
+ private final Dispatcher.RequestTime requestTime;
private final SortedMap<DecoratedKey, FollowUpReadInfo> followUpReadInfo;
private final PartitionRangeReadCommand command;
private final AbstractBounds<PartitionPosition> followUpBounds;
@@ -59,7 +60,7 @@ class FilteredFollowupRead extends AsyncPromise<TrackedDataResponse>
public FilteredFollowupRead(TrackedDataResponse initialResponse,
int toQuery,
ConsistencyLevel consistencyLevel,
- long expiresAtNanos,
+ Dispatcher.RequestTime requestTime,
SortedMap<DecoratedKey, FollowUpReadInfo> followUpReadInfo,
PartitionRangeReadCommand command,
AbstractBounds<PartitionPosition> followUpBounds,
@@ -68,7 +69,7 @@ class FilteredFollowupRead extends AsyncPromise<TrackedDataResponse>
this.initialResponse = initialResponse;
this.toQuery = toQuery;
this.consistencyLevel = consistencyLevel;
- this.expiresAtNanos = expiresAtNanos;
+ this.requestTime = requestTime;
this.followUpReadInfo = followUpReadInfo;
this.command = command;
this.followUpBounds = followUpBounds;
@@ -96,8 +97,8 @@ class FilteredFollowupRead extends AsyncPromise<TrackedDataResponse>
FollowUpReadInfo info = followUpReadInfo.get(key);
remaining -= info.potentialMatches;
SinglePartitionReadCommand cmd = SinglePartitionReadCommand.fromRangeRead(key, command, command.limits().forShortReadRetry(toQuery));
- TrackedRead.Partition read = TrackedRead.Partition.create(metadata, cmd, consistencyLevel);
- read.start(expiresAtNanos);
+ TrackedRead.Partition read = TrackedRead.Partition.create(metadata, cmd, consistencyLevel, requestTime);
+ read.start(requestTime);
futures.add(read.future());
}
@@ -107,8 +108,8 @@ class FilteredFollowupRead extends AsyncPromise<TrackedDataResponse>
if (remaining > 0)
{
partialRead = new AtomicReference<>();
- TrackedRead.Range rangeRead = makeFollowUpRead(command, followUpBounds, remaining, consistencyLevel, expiresAtNanos);
- rangeRead.startLocal(expiresAtNanos, partialRead::set);
+ TrackedRead.Range rangeRead = makeFollowUpRead(command, followUpBounds, remaining, consistencyLevel, requestTime);
+ rangeRead.startLocal(requestTime, partialRead::set);
futures.add(rangeRead.future());
}
else
@@ -153,7 +154,7 @@ class FilteredFollowupRead extends AsyncPromise<TrackedDataResponse>
PartialTrackedRangeRead followUpRangeRead = (PartialTrackedRangeRead) partialRead.get();
nextBounds = followUpRangeRead.followUpBounds();
}
- FilteredFollowupRead followUp = new FilteredFollowupRead(response, toQuery(command, mergedResultCounter), consistencyLevel, expiresAtNanos, nextKeys, command, nextBounds, null);
+ FilteredFollowupRead followUp = new FilteredFollowupRead(response, toQuery(command, mergedResultCounter), consistencyLevel, requestTime, nextKeys, command, nextBounds, null);
followUp.start();
followUp.addCallback((result, failure) -> {
if (failure != null)
diff --git a/src/java/org/apache/cassandra/service/reads/tracked/PartialTrackedRangeRead.java b/src/java/org/apache/cassandra/service/reads/tracked/PartialTrackedRangeRead.java
index b49570ceb7..2f3b58dfdc 100644
--- a/src/java/org/apache/cassandra/service/reads/tracked/PartialTrackedRangeRead.java
+++ b/src/java/org/apache/cassandra/service/reads/tracked/PartialTrackedRangeRead.java
@@ -25,6 +25,7 @@ import java.util.SortedMap;
import java.util.TreeMap;
import com.google.common.base.Preconditions;
+import org.apache.cassandra.transport.Dispatcher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -305,7 +306,7 @@ public abstract class PartialTrackedRangeRead extends AbstractPartialTrackedRead
{
if (wasAugmented)
return extendRead(iterator);
- return CompletedRead.simple(iterator, command);
+ return CompletedRead.simple(iterator, command, command.nowInSec());
}
AbstractBounds<PartitionPosition> followUpBounds()
@@ -354,7 +355,7 @@ public abstract class PartialTrackedRangeRead extends AbstractPartialTrackedRead
return completed.followUpBounds();
}
- protected static TrackedRead.Range makeFollowUpRead(PartitionRangeReadCommand command, AbstractBounds<PartitionPosition> followUpBounds, int toQuery, ConsistencyLevel consistencyLevel, long expiresAtNanos)
+ protected static TrackedRead.Range makeFollowUpRead(PartitionRangeReadCommand command, AbstractBounds<PartitionPosition> followUpBounds, int toQuery, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
{
DataLimits newLimits = command.limits().forShortReadRetry(toQuery);
@@ -369,7 +370,7 @@ public abstract class PartialTrackedRangeRead extends AbstractPartialTrackedRead
followUpCmd.dataRange().keyRange(),
1);
- TrackedRead.Range read = TrackedRead.Range.create(followUpCmd, replicaPlan);
+ TrackedRead.Range read = TrackedRead.Range.create(followUpCmd, replicaPlan, requestTime);
logger.trace("Short read detected, starting followup read {}", read);
return read;
}
@@ -556,15 +557,15 @@ public abstract class PartialTrackedRangeRead extends AbstractPartialTrackedRead
}
@Override
- protected Future<TrackedDataResponse> makeFollowupRead(TrackedDataResponse initialResponse, int toQuery, ConsistencyLevel consistencyLevel, long expiresAtNanos)
+ protected Future<TrackedDataResponse> makeFollowupRead(TrackedDataResponse initialResponse, int toQuery, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
{
if (followUpReadInfo.isEmpty())
- return super.makeFollowupRead(initialResponse, toQuery, consistencyLevel, expiresAtNanos);
+ return super.makeFollowupRead(initialResponse, toQuery, consistencyLevel, requestTime);
FilteredFollowupRead followupRead = new FilteredFollowupRead(initialResponse,
toQuery,
consistencyLevel,
- expiresAtNanos,
+ requestTime,
followUpReadInfo,
command,
followUpBounds,
diff --git a/src/java/org/apache/cassandra/service/reads/tracked/PartialTrackedRead.java b/src/java/org/apache/cassandra/service/reads/tracked/PartialTrackedRead.java
index 99e509ee7f..cd3b137b6f 100644
--- a/src/java/org/apache/cassandra/service/reads/tracked/PartialTrackedRead.java
+++ b/src/java/org/apache/cassandra/service/reads/tracked/PartialTrackedRead.java
@@ -18,7 +18,7 @@
package org.apache.cassandra.service.reads.tracked;
-import java.util.Collection;
+import com.google.common.base.Preconditions;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.ConsistencyLevel;
@@ -31,13 +31,17 @@ import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
import org.apache.cassandra.index.Index;
import org.apache.cassandra.utils.concurrent.Future;
+import org.apache.cassandra.replication.Log2OffsetsMap;
+import org.apache.cassandra.replication.MutationJournal;
+import org.apache.cassandra.replication.ShortMutationId;
+import org.apache.cassandra.transport.Dispatcher;
public interface PartialTrackedRead
{
interface CompletedRead extends AutoCloseable
{
TrackedDataResponse response(); // must be called from the read stage
- Future<TrackedDataResponse> followupRead(TrackedDataResponse initialResponse, ConsistencyLevel consistencyLevel, long expiresAtNanos);
+ Future<TrackedDataResponse> followupRead(TrackedDataResponse initialResponse, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime);
@Override
void close();
@@ -53,7 +57,7 @@ public interface PartialTrackedRead
command.columnFilter());
}
- static CompletedRead simple(UnfilteredPartitionIterator partition, ReadCommand command)
+ static CompletedRead simple(UnfilteredPartitionIterator partition, ReadCommand command, long nowInSec)
{
return new CompletedRead()
{
@@ -64,7 +68,7 @@ public interface PartialTrackedRead
}
@Override
- public Future<TrackedDataResponse> followupRead(TrackedDataResponse initialRead, ConsistencyLevel consistencyLevel, long expiresAtNanos)
+ public Future<TrackedDataResponse> followupRead(TrackedDataResponse initialRead, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
{
return null;
}
@@ -82,9 +86,16 @@ public interface PartialTrackedRead
void augment(Mutation mutation);
- default void augment(Collection<Mutation> mutations)
+ default void augment(Log2OffsetsMap<?> augmentingOffsets)
{
- mutations.forEach(this::augment);
+ augmentingOffsets.forEach(this::augment);
+ }
+
+ default void augment(ShortMutationId mutationId)
+ {
+ Mutation mutation = MutationJournal.instance.read(mutationId);
+ Preconditions.checkNotNull(mutation);
+ augment(mutation);
}
ReadExecutionController executionController();
diff --git a/src/java/org/apache/cassandra/service/reads/tracked/PartialTrackedSinglePartitionRead.java b/src/java/org/apache/cassandra/service/reads/tracked/PartialTrackedSinglePartitionRead.java
index e15dba8a5c..368a4f0bfd 100644
--- a/src/java/org/apache/cassandra/service/reads/tracked/PartialTrackedSinglePartitionRead.java
+++ b/src/java/org/apache/cassandra/service/reads/tracked/PartialTrackedSinglePartitionRead.java
@@ -126,7 +126,7 @@ public class PartialTrackedSinglePartitionRead extends AbstractPartialTrackedRea
@Override
protected CompletedRead createResult(UnfilteredPartitionIterator iterator)
{
- return CompletedRead.simple(iterator, command);
+ return CompletedRead.simple(iterator, command, command.nowInSec());
}
}
diff --git a/src/java/org/apache/cassandra/service/reads/tracked/ReadReconcileNotify.java b/src/java/org/apache/cassandra/service/reads/tracked/ReadReconcileAck.java
similarity index 50%
rename from src/java/org/apache/cassandra/service/reads/tracked/ReadReconcileNotify.java
rename to src/java/org/apache/cassandra/service/reads/tracked/ReadReconcileAck.java
index 6a72a7a01a..bfa716a9e0 100644
--- a/src/java/org/apache/cassandra/service/reads/tracked/ReadReconcileNotify.java
+++ b/src/java/org/apache/cassandra/service/reads/tracked/ReadReconcileAck.java
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.cassandra.service.reads.tracked;
import java.io.IOException;
@@ -23,73 +22,58 @@ import java.io.IOException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.net.IVerbHandler;
-import org.apache.cassandra.net.Message;
-import org.apache.cassandra.replication.MutationTrackingService;
/**
- * Notifies the read coordinator that this node has received mutations
- * from the sending node
+ * Notifies the reconcile coordinator (data node) that this node has received
+ * all missing mutations from involved peers that were needed for the read.
*/
-public class ReadReconcileNotify
+public class ReadReconcileAck
{
- private static final Logger logger = LoggerFactory.getLogger(ReadReconcileNotify.class);
+ private static final Logger logger = LoggerFactory.getLogger(ReadReconcileAck.class);
public final TrackedRead.Id readId;
- public final int syncId;
- public ReadReconcileNotify(TrackedRead.Id readId, int syncId)
+ public ReadReconcileAck(TrackedRead.Id readId)
{
this.readId = readId;
- this.syncId = syncId;
}
@Override
public String toString()
{
- return "ReadReconcileNotify{" +
- "reconciliationId=" + readId +
- ", syncId=" + syncId +
- '}';
+ return "ReadReconcileAck{readId=" + readId + '}';
}
- public static final IVerbHandler<ReadReconcileNotify> verbHandler = new IVerbHandler<ReadReconcileNotify>()
+ public static final IVerbHandler<ReadReconcileAck> verbHandler = message ->
{
- @Override
- public void doVerb(Message<ReadReconcileNotify> message) throws IOException
- {
- ReadReconcileNotify notify = message.payload;
- logger.trace("Received read reconcile notify from {}: {}", message.from(), notify);
- MutationTrackingService.instance.localReads().acknowledgeSync(notify.readId, notify.syncId);
- }
+ ReadReconcileAck notify = message.payload;
+ logger.trace("Received reconcile ack from {}, for {}", message.from(), notify.readId);
+ ReadReconciliations.instance.acceptSyncAck(message.from(), notify.readId);
};
- public static final IVersionedSerializer<ReadReconcileNotify> serializer = new IVersionedSerializer<>()
+ public static final IVersionedSerializer<ReadReconcileAck> serializer = new IVersionedSerializer<>()
{
@Override
- public void serialize(ReadReconcileNotify notify, DataOutputPlus out, int version) throws IOException
+ public void serialize(ReadReconcileAck notify, DataOutputPlus out, int version) throws IOException
{
TrackedRead.Id.serializer.serialize(notify.readId, out, version);
- out.writeInt(notify.syncId);
}
@Override
- public ReadReconcileNotify deserialize(DataInputPlus in, int version) throws IOException
+ public ReadReconcileAck deserialize(DataInputPlus in, int version) throws IOException
{
TrackedRead.Id readId = TrackedRead.Id.serializer.deserialize(in, version);
- int syncId = in.readInt();
- return new ReadReconcileNotify(readId, syncId);
+ return new ReadReconcileAck(readId);
}
@Override
- public long serializedSize(ReadReconcileNotify notify, int version)
+ public long serializedSize(ReadReconcileAck notify, int version)
{
- return TrackedRead.Id.serializer.serializedSize(notify.readId, version)
- + TypeSizes.sizeof(notify.syncId);
+ return TrackedRead.Id.serializer.serializedSize(notify.readId, version);
}
};
}
diff --git a/src/java/org/apache/cassandra/service/reads/tracked/ReadReconcileReceive.java b/src/java/org/apache/cassandra/service/reads/tracked/ReadReconcileReceive.java
deleted file mode 100644
index 6a5fd7db52..0000000000
--- a/src/java/org/apache/cassandra/service/reads/tracked/ReadReconcileReceive.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.service.reads.tracked;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.db.Mutation;
-import org.apache.cassandra.db.TypeSizes;
-import org.apache.cassandra.io.IVersionedSerializer;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.net.IVerbHandler;
-import org.apache.cassandra.net.Message;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.net.Verb;
-import org.apache.cassandra.replication.MutationTrackingService;
-import org.apache.cassandra.utils.CollectionSerializers;
-
-import static org.apache.cassandra.locator.InetAddressAndPort.Serializer.inetAddressAndPortSerializer;
-
-public class ReadReconcileReceive
-{
- private static final Logger logger = LoggerFactory.getLogger(ReadReconcileReceive.class);
-
- public final TrackedRead.Id readId;
- public final int syncId;
- public final InetAddressAndPort coordinator;
- public final List<Mutation> mutations;
-
- public ReadReconcileReceive(TrackedRead.Id readId, int syncId, InetAddressAndPort coordinator, List<Mutation> mutations)
- {
- this.readId = readId;
- this.syncId = syncId;
- this.coordinator = coordinator;
- this.mutations = mutations;
- }
-
- private static String mutationString(List<Mutation> mutations)
- {
- StringBuilder builder = new StringBuilder('[');
- boolean isFirst = true;
- for (Mutation mutation : mutations)
- {
- if (!isFirst)
- {
- builder.append(", ");
- }
- isFirst = false;
- builder.append(mutation.id());
- }
- builder.append(']');
- return builder.toString();
- }
-
- @Override
- public String toString()
- {
- return "ReadReconcileReceive{" +
- "reconciliationId=" + readId +
- ", syncId=" + syncId +
- ", coordinator=" + coordinator +
- ", mutations=" + mutationString(mutations) +
- '}';
- }
-
- public static final IVerbHandler<ReadReconcileReceive> verbHandler = new IVerbHandler<ReadReconcileReceive>()
- {
- @Override
- public void doVerb(Message<ReadReconcileReceive> message) throws IOException
- {
- // TODO: check epoch and tokens?
- ReadReconcileReceive receive = message.payload;
- logger.trace("Received read reconciliation from {}: {}", message.from(), receive);
- receive.mutations.forEach(Mutation::apply);
-
- if (!MutationTrackingService.instance.localReads().receiveMutations(receive.readId, receive.syncId, receive.mutations))
- {
- // if this isn't a locally coordinated read, notify the coordinator
- ReadReconcileNotify notify = new ReadReconcileNotify(receive.readId, receive.syncId);
- MessagingService.instance().send(Message.out(Verb.READ_RECONCILE_NOTIFY, notify), receive.coordinator);
- }
- }
- };
-
- public static final IVersionedSerializer<ReadReconcileReceive> serializer = new IVersionedSerializer<ReadReconcileReceive>()
- {
- @Override
- public void serialize(ReadReconcileReceive rcv, DataOutputPlus out, int version) throws IOException
- {
- TrackedRead.Id.serializer.serialize(rcv.readId, out, version);
- out.writeInt(rcv.syncId);
- inetAddressAndPortSerializer.serialize(rcv.coordinator, out, version);
- CollectionSerializers.serializeCollection(rcv.mutations, out, version, Mutation.serializer);
-
- }
-
- @Override
- public ReadReconcileReceive deserialize(DataInputPlus in, int version) throws IOException
- {
- TrackedRead.Id readId = TrackedRead.Id.serializer.deserialize(in, version);
- int syncId = in.readInt();
- InetAddressAndPort coordinator = inetAddressAndPortSerializer.deserialize(in, version);
- List<Mutation> mutations = CollectionSerializers.deserializeList(in, version, Mutation.serializer);
- return new ReadReconcileReceive(readId, syncId, coordinator, mutations);
- }
-
- @Override
- public long serializedSize(ReadReconcileReceive t, int version)
- {
- return TrackedRead.Id.serializer.serializedSize(t.readId, version)
- + TypeSizes.sizeof(t.syncId)
- + inetAddressAndPortSerializer.serializedSize(t.coordinator, version)
- + CollectionSerializers.serializedCollectionSize(t.mutations, version, Mutation.serializer);
- }
- };
-}
diff --git a/src/java/org/apache/cassandra/service/reads/tracked/ReadReconcileSend.java b/src/java/org/apache/cassandra/service/reads/tracked/ReadReconcileSend.java
deleted file mode 100644
index d89223f6cc..0000000000
--- a/src/java/org/apache/cassandra/service/reads/tracked/ReadReconcileSend.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.service.reads.tracked;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-
-import org.apache.cassandra.db.Mutation;
-import org.apache.cassandra.db.TypeSizes;
-import org.apache.cassandra.io.IVersionedSerializer;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.net.IVerbHandler;
-import org.apache.cassandra.net.Message;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.net.Verb;
-import org.apache.cassandra.replication.Log2OffsetsMap;
-import org.apache.cassandra.replication.MutationJournal;
-import org.apache.cassandra.utils.CollectionSerializers;
-
-/**
- * Instructs a node to send mutations to other node
- */
-public class ReadReconcileSend
-{
- private static final Logger logger = LoggerFactory.getLogger(ReadReconcileSend.class);
-
- public static class PeerSync
- {
- final int syncId;
- final InetAddressAndPort to;
- final Log2OffsetsMap.Immutable plan;
-
- public PeerSync(int syncId, InetAddressAndPort to, Log2OffsetsMap.Immutable plan)
- {
- this.syncId = syncId;
- this.to = to;
- this.plan = plan;
- }
-
- @Override
- public String toString()
- {
- return "PeerSync{" +
- "syncId=" + syncId +
- ", to=" + to +
- ", plan=" + plan +
- '}';
- }
-
- public static final IVersionedSerializer<PeerSync> serializer = new IVersionedSerializer<PeerSync>()
- {
- @Override
- public void serialize(PeerSync sync, DataOutputPlus out, int version) throws IOException
- {
- out.writeInt(sync.syncId);
- InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serialize(sync.to, out, version);
- Log2OffsetsMap.Immutable.serializer.serialize(sync.plan, out, version);
- }
-
- @Override
- public PeerSync deserialize(DataInputPlus in, int version) throws IOException
- {
- return new PeerSync(in.readInt(),
- InetAddressAndPort.Serializer.inetAddressAndPortSerializer.deserialize(in, version),
- Log2OffsetsMap.Immutable.serializer.deserialize(in, version));
- }
-
- @Override
- public long serializedSize(PeerSync sync, int version)
- {
- return TypeSizes.sizeof(sync.syncId)
- + InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serializedSize(sync.to, version)
- + Log2OffsetsMap.Immutable.serializer.serializedSize(sync.plan, version);
- }
- };
- }
-
- public final TrackedRead.Id reconcileId;
- public final ImmutableList<PeerSync> syncTasks;
-
- public ReadReconcileSend(TrackedRead.Id reconcileId, List<PeerSync> syncTasks)
- {
- this.reconcileId = reconcileId;
- this.syncTasks = ImmutableList.copyOf(syncTasks);
- }
-
- @Override
- public String toString()
- {
- return "ReadReconcileSend{" +
- "reconcileId=" + reconcileId +
- ", syncTasks=" + syncTasks +
- '}';
- }
-
- public static final IVerbHandler<ReadReconcileSend> verbHandler = new IVerbHandler<>()
- {
- @Override
- public void doVerb(Message<ReadReconcileSend> message)
- {
- logger.trace("Received ReadReconcileSend from {}: {}", message.from(), message.payload);
- // TODO: check epoch and tokens?
- ReadReconcileSend payload = message.payload;
- for (PeerSync sync : message.payload.syncTasks)
- {
- // TODO (expected): do not deser just to serialize again, if same messaging versions (common case)
- // TODO (expected): don't materialize mutation ids, look up from offset collections
- int mutationCount = sync.plan.idCount();
- List<Mutation> mutations = new ArrayList<>(mutationCount);
- MutationJournal.instance.readAll(sync.plan, mutations);
- Preconditions.checkArgument(mutationCount == mutations.size());
-
- ReadReconcileReceive receive = new ReadReconcileReceive(payload.reconcileId, sync.syncId, message.from(), mutations);
- logger.trace("Sending {} to replica {}", receive, sync.to);
- MessagingService.instance().send(Message.out(Verb.READ_RECONCILE_RCV, receive), sync.to);
- }
- }
- };
-
- public static final IVersionedSerializer<ReadReconcileSend> serializer = new IVersionedSerializer<>()
- {
- @Override
- public void serialize(ReadReconcileSend send, DataOutputPlus out, int version) throws IOException
- {
- TrackedRead.Id.serializer.serialize(send.reconcileId, out, version);
- CollectionSerializers.serializeList(send.syncTasks, out, version, PeerSync.serializer);
- }
-
- @Override
- public ReadReconcileSend deserialize(DataInputPlus in, int version) throws IOException
- {
- TrackedRead.Id readId = TrackedRead.Id.serializer.deserialize(in, version);
- List<PeerSync> syncTasks = CollectionSerializers.deserializeList(in, version, PeerSync.serializer);
- return new ReadReconcileSend(readId, syncTasks);
- }
-
- @Override
- public long serializedSize(ReadReconcileSend send, int version)
- {
- return TrackedRead.Id.serializer.serializedSize(send.reconcileId, version) + CollectionSerializers.serializedCollectionSize(send.syncTasks, version, PeerSync.serializer);
- }
- };
-}
diff --git a/src/java/org/apache/cassandra/service/reads/tracked/ReadReconciliations.java b/src/java/org/apache/cassandra/service/reads/tracked/ReadReconciliations.java
new file mode 100644
index 0000000000..17851a24e3
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/reads/tracked/ReadReconciliations.java
@@ -0,0 +1,396 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.service.reads.tracked;
+
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLongFieldUpdater;
+
+import org.agrona.collections.IntArrayList;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.gms.FailureDetector;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.replication.CoordinatorLogId;
+import org.apache.cassandra.replication.ExpiredStatePurger;
+import org.apache.cassandra.replication.IncomingMutations;
+import org.apache.cassandra.replication.Log2OffsetsMap;
+import org.apache.cassandra.replication.MutationSummary;
+import org.apache.cassandra.replication.MutationTrackingService;
+import org.apache.cassandra.replication.Node2OffsetsMap;
+import org.apache.cassandra.replication.Offsets;
+import org.apache.cassandra.replication.PullMutationsRequest;
+import org.apache.cassandra.replication.ShortMutationId;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.membership.NodeId;
+import org.apache.cassandra.utils.Clock;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.concurrent.Accumulator;
+import org.jctools.maps.NonBlockingHashMap;
+import org.jctools.maps.NonBlockingHashSet;
+
+/**
+ * On the initial coordinator (replica or not):
+ * 0. Issue summary requests to all summary nodes, in parallel
+ * <p>
+ * On the data node / coordinator:
+ * 1. Compute the secondary summary, send to all other nodes for reconcile
+ * 2. For primary logs in the summary, initiate priority delivery of mutations
+ * that are present in our summary but absent from any summary node
+ * 3. Collect summary responses from all summary nodes, request missing mutations
+ * whenever we get a summary from any of the nodes
+ * 4. Collect the requested mutations from other nodes
+ * 5. Collect sync-done acks from every summary node
+ * 6. When all the summaries, all missing mutations, and all sync-done acks have
+ * been received, invoke the reconcile-complete callback
+ * 7. Clean up state after ourselves (states map)
+ * <p>
+ * On each summary node:
+ * 1. Compute the secondary summary, send to all other nodes for reconcile
+ * 2. For primary logs in the summary, initiate priority delivery of mutations
+ * that are present in our summary but absent from any summary or data node
+ * 3. Collect summary responses from all summary nodes, request missing mutations
+ * whenever we get a summary from any of the nodes
+ * 4. Collect the requested mutations from other nodes
+ * 5. When all the summaries and all missing mutations have been received,
+ * respond to the data node with a sync-done message
+ * 6. Clean up state after ourselves (states) map
+ */
+public class ReadReconciliations implements ExpiredStatePurger.Expireable
+{
+ private static final int LOCAL_NODE = ClusterMetadata.current().myNodeId().id();
+
+ public static final ReadReconciliations instance = new ReadReconciliations();
+
+ // We are reusing the read id just for the ease of it, but it doesn't really have to match
+ // read's id. Could be another instance of it, or a UUID or any other cluster-unique identifier.
+ private final NonBlockingHashMap<TrackedRead.Id, Coordinator> coordinators = new NonBlockingHashMap<>();
+
+ /**
+ * Main entry point for summary nodes - if a coordinator's request arrives before any peer responses.
+ * Executed in READ stage. Calculate the summary, process it locally and send it to peers so that they
+ * can fill their own gaps.
+ */
+ public void handleSummaryRequest(TrackedRead.SummaryRequest request)
+ {
+ Coordinator reconcile = getOrCreateCoordinator(request.readId, request.dataNode, request.summaryNodes);
+ if (reconcile.acceptLocalSummary(request.command.createMutationSummary(false)))
+ coordinators.remove(request.readId);
+ }
+
+ /**
+ * The way for the data node to provide its final (seconary) summary.
+ */
+ public void acceptLocalSummary(TrackedRead.Id id, MutationSummary summary, int[] summaryNodes)
+ {
+ Coordinator reconcile = getOrCreateCoordinator(id, LOCAL_NODE, summaryNodes);
+ if (reconcile.acceptLocalSummary(summary))
+ coordinators.remove(id);
+ }
+
+ /**
+ * Can arrive from a peer node before a summary request
+ */
+ public void acceptRemoteSummary(InetAddressAndPort from, TrackedSummaryResponse summary)
+ {
+ Coordinator reconcile = getOrCreateCoordinator(summary.readId, summary.dataNode, summary.summaryNodes);
+ if (reconcile.acceptRemoteSummary(summary.summary, node(from)))
+ coordinators.remove(summary.readId);
+ }
+
+ public void acceptSyncAck(InetAddressAndPort from, TrackedRead.Id id)
+ {
+ Coordinator reconcile = coordinators.get(id);
+ if (reconcile != null && reconcile.acceptSyncAck(node(from))) // could be already timed out / expired
+ coordinators.remove(id);
+ }
+
+ public void acceptMutation(TrackedRead.Id id, ShortMutationId mutationId)
+ {
+ Coordinator reconcile = coordinators.get(id);
+ if (reconcile != null && reconcile.acceptMutation(mutationId)) // could be already timed out / expired
+ coordinators.remove(id);
+ }
+
+ private Coordinator getOrCreateCoordinator(TrackedRead.Id id, int dataNode, int[] summaryNodes)
+ {
+ Coordinator reconcile = coordinators.get(id);
+ if (reconcile != null) return reconcile;
+ return coordinators.computeIfAbsent(id, readId -> new Coordinator(id, dataNode, summaryNodes));
+ }
+
+ @Override
+ public int expire(long nanoTime)
+ {
+ int n = 0;
+ for (Map.Entry<TrackedRead.Id, Coordinator> entry : coordinators.entrySet())
+ {
+ TrackedRead.Id id = entry.getKey();
+ Coordinator coordinator = entry.getValue();
+
+ if (coordinator.isPurgeable(nanoTime) && coordinators.remove(id, coordinator))
+ n++;
+ }
+ return n;
+ }
+
+ private static final class Coordinator
+ {
+ private static final AtomicLongFieldUpdater<Coordinator> remainingUpdater =
+ AtomicLongFieldUpdater.newUpdater(Coordinator.class, "remaining");
+ private volatile long remaining; // three values packed into one atomic long
+
+ private final TrackedRead.Id id;
+ private final int dataNode;
+ private final int[] summaryNodes;
+ private final long expiresAtNanos;
+
+ private final Accumulator<Pair<Integer, MutationSummary>> summaries;
+
+ private final Set<ShortMutationId> requested = new NonBlockingHashSet<>();
+
+ Coordinator(TrackedRead.Id id, int dataNode, int[] summaryNodes)
+ {
+ this.id = id;
+ this.dataNode = dataNode;
+ this.summaryNodes = summaryNodes;
+ this.expiresAtNanos =
+ Clock.Global.nanoTime() + Math.max(DatabaseDescriptor.getReadRpcTimeout(TimeUnit.NANOSECONDS),
+ DatabaseDescriptor.getRangeRpcTimeout(TimeUnit.NANOSECONDS));
+ int remainingSummaries = 1 + summaryNodes.length;
+ int remainingMutations = 0;
+ int remainingSyncAcks = dataNode == LOCAL_NODE ? summaryNodes.length : 0;
+ remaining = remaining(remainingMutations, remainingSummaries, remainingSyncAcks);
+
+ summaries = new Accumulator<>(remainingSummaries);
+ }
+
+ /**
+ * For all the logs in the summary that are owned by us, preemptively prioritise delivery of
+ * any mutations that are absent from other participating nodes according to our primary coordinator
+ * log knowledge. This is an optimisation, and it can be omitted without affecting correctness.
+ */
+ boolean acceptLocalSummary(MutationSummary summary)
+ {
+ IntArrayList remoteNodes = new IntArrayList(summaryNodes.length, Integer.MIN_VALUE);
+ if (dataNode != LOCAL_NODE)
+ remoteNodes.addInt(dataNode);
+ for (int node : summaryNodes)
+ if (node != LOCAL_NODE)
+ remoteNodes.addInt(node);
+
+ if (!remoteNodes.isEmpty())
+ {
+ // send the summary to all peers, so that they can initiate reconciling their gaps ASAP
+ TrackedSummaryResponse response = new TrackedSummaryResponse(id, summary, dataNode, summaryNodes);
+ Message<TrackedSummaryResponse> message = Message.out(Verb.TRACKED_SUMMARY_RSP, response);
+ for (int node : remoteNodes)
+ MessagingService.instance().send(message, host(node));
+ }
+
+ Iterator<Offsets> iter = summary.onlyUnreconciled();
+ Node2OffsetsMap missingOffsets = new Node2OffsetsMap();
+ while (iter.hasNext())
+ {
+ Offsets ourOffsets = iter.next();
+ CoordinatorLogId logId = ourOffsets.logId();
+ if (logId.hostId() != LOCAL_NODE)
+ continue;
+
+ MutationTrackingService.instance.collectRemotelyMissingMutations(ourOffsets, remoteNodes, missingOffsets);
+ // we don't listen to or block on delivery of these mutations, intentionally
+ missingOffsets.forEach(ReadReconciliations::push);
+ missingOffsets.clear();
+ }
+
+ summaries.add(Pair.create(LOCAL_NODE, summary));
+ return updateRemainingAndMaybeComplete(0, -1, 0);
+ }
+
+ /**
+ * Calculate the diff with the remote summary to see which mutations from the summary we lack,
+ * and pull them - from the primary coordinator, if it's alive, or the remote replica, if it's not.
+ */
+ boolean acceptRemoteSummary(MutationSummary summary, int remoteNode)
+ {
+ Log2OffsetsMap.Mutable missingMutations = new Log2OffsetsMap.Mutable();
+ MutationTrackingService.instance.collectLocallyMissingMutations(summary, missingMutations);
+
+ // don't request what's already been requested for other remote summaries
+ // TODO (consider): rely entirely on IncomingMutations deduplication instead
+ missingMutations.removeAll(requested);
+ missingMutations.forEach(requested::add);
+ int missingCount = missingMutations.idCount();
+
+ // TODO (expected, low priority): handle timeouts here to stop earlier
+ IncomingMutations.Callback callback =
+ (mutationId) -> ReadReconciliations.instance.acceptMutation(id, mutationId);
+ for (Offsets offsets : missingMutations.offsets())
+ pull(remoteNode, offsets, callback);
+
+ summaries.add(Pair.create(remoteNode, summary));
+ return updateRemainingAndMaybeComplete(missingCount, -1, 0);
+ }
+
+ boolean acceptSyncAck(int ignoredSyncId)
+ {
+ return updateRemainingAndMaybeComplete(0, 0, -1);
+ }
+
+ boolean acceptMutation(ShortMutationId ignoredMutationId)
+ {
+ return updateRemainingAndMaybeComplete(-1, 0, 0);
+ }
+
+ /**
+ * Remote summaries minus data node summary offsets
+ */
+ private Log2OffsetsMap.Mutable augmentingOffsets()
+ {
+ Log2OffsetsMap.Mutable offsets = new Log2OffsetsMap.Mutable();
+
+ // add up all remote summaries
+ for (int i = 0, size = summaries.size(); i < size; i++)
+ {
+ Pair<Integer, MutationSummary> summary = summaries.get(i);
+ if (summary.left != LOCAL_NODE)
+ offsets.addAll(summary.right);
+ }
+
+ // subtract the local summary
+ for (int i = 0, size = summaries.size(); i < size; i++)
+ {
+ Pair<Integer, MutationSummary> summary = summaries.get(i);
+ if (summary.left == LOCAL_NODE)
+ {
+ offsets.removeAll(summary.right);
+ break;
+ }
+ }
+
+ return offsets;
+ }
+
+ private boolean isDataNode()
+ {
+ return LOCAL_NODE == dataNode;
+ }
+
+ /*
+ * Logic to deal with remaining mutations/summaries/syncAcks counters atomically.
+ */
+
+ private static int remainingMutations(long remaining)
+ {
+ return (int) ((remaining >>> 32) & 0xFFFFFFFFL);
+ }
+
+ private static int remainingSummaries(long remaining)
+ {
+ return (int) ((remaining >>> 16) & 0xFFFFL);
+ }
+
+ private static int remainingSyncAcks(long remaining)
+ {
+ return (int) (remaining & 0xFFFFL);
+ }
+
+ // 32 bits for mutations, 16 bits for summaries, 16 bits for syncAcks
+ private static long remaining(int mutations, int summaries, int syncAcks)
+ {
+ return ((long) mutations << 32) | (((long) summaries << 16) | syncAcks);
+ }
+
+ private long updateRemaining(int mutationsDelta, int summariesDelta, int syncAcksDelta)
+ {
+ long prev, next;
+ do
+ {
+ prev = remaining;
+ int mutations = remainingMutations(prev) + mutationsDelta;
+ int summaries = remainingSummaries(prev) + summariesDelta;
+ int syncAcks = remainingSyncAcks(prev) + syncAcksDelta;
+ next = remaining(mutations, summaries, syncAcks);
+ } while (!remainingUpdater.compareAndSet(this, prev, next));
+ return next;
+ }
+
+ private boolean updateRemainingAndMaybeComplete(int mutationsDelta, int summariesDelta, int syncAcksDelta)
+ {
+ return updateRemaining(mutationsDelta, summariesDelta, syncAcksDelta) == 0 && complete();
+ }
+
+ private boolean complete()
+ {
+ if (isDataNode())
+ MutationTrackingService.instance.localReads().acknowledgeReconcile(id, augmentingOffsets());
+ else
+ MessagingService.instance().send(Message.out(Verb.READ_RECONCILE_ACK, new ReadReconcileAck(id)), host(dataNode));
+
+ return true;
+ }
+
+ boolean isPurgeable(long nanoTime)
+ {
+ return nanoTime - expiresAtNanos > 0;
+ }
+ }
+
+ /**
+ * @param node node id of the remote replica from which we got the summary
+ * @param offsets offsets that we need to pull - from the coordinator, if alive, or from the
+ * node we got the summary from (by definition it also has the mutation)
+ */
+ private static void pull(int node, Offsets offsets, IncomingMutations.Callback callback)
+ {
+ InetAddressAndPort logCoordinator = host(offsets.logId().hostId());
+ InetAddressAndPort pullFrom = FailureDetector.instance.isAlive(logCoordinator)
+ ? logCoordinator
+ : host(node);
+
+ Offsets.Mutable toPull = new Offsets.Mutable(offsets.logId());
+ for (ShortMutationId id : offsets)
+ if (MutationTrackingService.instance.registerMutationCallback(id, callback))
+ toPull.add(id.offset());
+
+ if (!toPull.isEmpty())
+ {
+ PullMutationsRequest pull = new PullMutationsRequest(Offsets.Immutable.copy(toPull));
+ MessagingService.instance().send(Message.out(Verb.PULL_MUTATIONS_REQ, pull), pullFrom);
+ }
+ }
+
+ private static void push(int node, Offsets offsets)
+ {
+ // TODO implement pre-emptive push with read priority
+ }
+
+ private static InetAddressAndPort host(int node)
+ {
+ return ClusterMetadata.current().directory.endpoint(new NodeId(node));
+ }
+
+ private static int node(InetAddressAndPort host)
+ {
+ return ClusterMetadata.current().directory.peerId(host).id();
+ }
+}
diff --git a/src/java/org/apache/cassandra/service/reads/tracked/TrackedLocalReadCoordinator.java b/src/java/org/apache/cassandra/service/reads/tracked/TrackedLocalReadCoordinator.java
deleted file mode 100644
index c62531240a..0000000000
--- a/src/java/org/apache/cassandra/service/reads/tracked/TrackedLocalReadCoordinator.java
+++ /dev/null
@@ -1,617 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.service.reads.tracked;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.ConsistencyLevel;
-import org.apache.cassandra.db.Mutation;
-import org.apache.cassandra.db.ReadCommand;
-import org.apache.cassandra.db.ReadExecutionController;
-import org.apache.cassandra.db.filter.ColumnFilter;
-import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.locator.ReplicaPlan;
-import org.apache.cassandra.metrics.ReadRepairMetrics;
-import org.apache.cassandra.net.Message;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.net.Verb;
-import org.apache.cassandra.replication.Log2OffsetsMap;
-import org.apache.cassandra.replication.MutationJournal;
-import org.apache.cassandra.replication.MutationSummary;
-import org.apache.cassandra.replication.ReconciliationPlan;
-import org.apache.cassandra.replication.ShortMutationId;
-import org.apache.cassandra.utils.Clock;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.concurrent.AbstractFuture;
-import org.apache.cassandra.utils.concurrent.Accumulator;
-import org.apache.cassandra.utils.concurrent.AsyncPromise;
-import org.apache.cassandra.utils.concurrent.Future;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.*;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
-import java.util.function.BiConsumer;
-import java.util.function.Consumer;
-
-public class TrackedLocalReadCoordinator
-{
- private static final Logger logger = LoggerFactory.getLogger(TrackedLocalReadCoordinator.class);
-
- private final TrackedRead.Id readId;
- private final AsyncPromise<TrackedDataResponse> promise;
-
- private volatile State state;
-
- public TrackedLocalReadCoordinator(TrackedRead.Id readId)
- {
- this.readId = readId;
- this.promise = new AsyncPromise<>();
- this.state = INITIALIZED;
- }
-
- public AbstractFuture<TrackedDataResponse> addCallback(BiConsumer<TrackedDataResponse, Throwable> callback)
- {
- return promise.addCallback(callback);
- }
-
- enum Status { INITIALIZED, AWAITING_READ, READING, RECONCILING, ABORTED, COMPLETED }
-
- private static abstract class State
- {
- abstract Status status();
-
- State startLocalRead(TrackedRead.Id readId, AsyncPromise<TrackedDataResponse> promise, ReadCommand command, ReplicaPlan.AbstractForRead<?, ?> replicaPlan, int[] summaryNodes, long expiresAtNanos)
- {
- // TODO: validate permitted state instead of just ignoring events?
- return this;
- }
-
- State receiveInProgressRead(PartialTrackedRead read, MutationSummary summary)
- {
- throw new IllegalStateException("Received in-progress read with status " + status());
- }
-
- State receiveSummary(InetAddressAndPort from, MutationSummary summary)
- {
- logger.trace("Ignoring summary from {} with status {}", from, status());
- return this;
- }
-
- State acknowledgeSync(int syncId)
- {
- logger.trace("Ignoring sync ack {} with status {}", syncId, status());
- return this;
- }
-
- State receiveMutations(List<Mutation> mutations)
- {
- logger.trace("Ignoring {} mutations with status {}", mutations.size(), status());
- return this;
- }
-
- State abort()
- {
- return ABORTED;
- }
-
- boolean isPurgeable(long nanoTime)
- {
- return false;
- }
-
- boolean isReading()
- {
- return is(Status.READING);
- }
-
- boolean isCompleted()
- {
- return is(Status.COMPLETED);
- }
-
- boolean is(Status status)
- {
- return status() == status;
- }
-
- @Override
- public String toString()
- {
- return status().toString();
- }
- }
-
- static final State INITIALIZED = new State()
- {
- @Override
- Status status() { return Status.INITIALIZED; }
-
- @Override
- State startLocalRead(TrackedRead.Id readId, AsyncPromise<TrackedDataResponse> promise, ReadCommand command, ReplicaPlan.AbstractForRead<?, ?> replicaPlan, int[] summaryNodes, long expiresAtNanos)
- {
- return new Reading(readId, promise, command, replicaPlan, summaryNodes, expiresAtNanos);
- }
-
- @Override
- State receiveSummary(InetAddressAndPort from, MutationSummary summary)
- {
- return new AwaitingRead(from, summary);
- }
- };
-
- static final State ABORTED = new State()
- {
- @Override
- Status status() { return Status.ABORTED; }
-
- @Override
- State receiveInProgressRead(PartialTrackedRead read, MutationSummary summary)
- {
- return this; // the read can't complete without data, but it could have been aborted in the meantime
- }
-
- @Override
- boolean isPurgeable(long nanoTime) { return true; }
- };
-
- static final State COMPLETED = new State()
- {
- @Override
- Status status() { return Status.COMPLETED; }
-
- @Override
- boolean isPurgeable(long nanoTime) { return true; }
- };
-
- private static class ReceivedSummary
- {
- final InetAddressAndPort from;
- final MutationSummary summary;
-
- ReceivedSummary(InetAddressAndPort from, MutationSummary summary)
- {
- this.from = from;
- this.summary = summary;
- }
- }
-
- // if we start receiving summaries before we receive the read command, they're
- // collected here
- private static class AwaitingRead extends State
- {
- private long lastUpdateNanos;
- private final List<ReceivedSummary> summaries;
-
- AwaitingRead(InetAddressAndPort summaryFrom, MutationSummary summary)
- {
- summaries = new ArrayList<>();
- summaries.add(new ReceivedSummary(summaryFrom, summary));
- }
-
- @Override
- Status status()
- {
- return Status.AWAITING_READ;
- }
-
- @Override
- State startLocalRead(TrackedRead.Id readId, AsyncPromise<TrackedDataResponse> promise, ReadCommand command, ReplicaPlan.AbstractForRead<?, ?> replicaPlan, int[] summaryNodes, long expiresAtNanos)
- {
- return new Reading(readId, promise, command, replicaPlan, summaryNodes, summaries, expiresAtNanos);
- }
-
- @Override
- State receiveSummary(InetAddressAndPort from, MutationSummary summary)
- {
- summaries.add(new ReceivedSummary(from, summary));
- lastUpdateNanos = Clock.Global.nanoTime();
- return this;
- }
-
- @Override
- boolean isPurgeable(long nanoTime)
- {
- return nanoTime - lastUpdateNanos > DatabaseDescriptor.getReadRpcTimeout(TimeUnit.NANOSECONDS);
- }
- }
-
- private static class Reading extends State
- {
- private final TrackedRead.Id readId;
- private final AsyncPromise<TrackedDataResponse> promise;
- private final ReadCommand command;
- private volatile PartialTrackedRead read;
- private final long expiresAtNanos;
- private final ReplicaPlan.AbstractForRead<?, ?> replicaPlan;
- private final Accumulator<ReceivedSummary> summaries;
- private final int[] summaryNodes; // for speculating when we haven't received enough summaries
-
- Reading(
- TrackedRead.Id readId,
- AsyncPromise<TrackedDataResponse> promise,
- ReadCommand command,
- ReplicaPlan.AbstractForRead<?, ?> replicaPlan,
- int[] summaryNodes,
- long expiresAtNanos)
- {
- this.readId = readId;
- this.promise = promise;
- this.expiresAtNanos = expiresAtNanos;
- this.command = command;
- this.replicaPlan = replicaPlan;
- this.summaries = new Accumulator<>(replicaPlan.readCandidates().size());
- this.summaryNodes = summaryNodes;
- }
-
- Reading(
- TrackedRead.Id readId,
- AsyncPromise<TrackedDataResponse> promise,
- ReadCommand command,
- ReplicaPlan.AbstractForRead<?, ?> replicaPlan,
- int[] summaryNodes,
- List<ReceivedSummary> summaries,
- long expiresAtNanos)
- {
- this(readId, promise, command, replicaPlan, summaryNodes, expiresAtNanos);
- for (ReceivedSummary summary : summaries) this.summaries.add(summary);
- }
-
- @Override
- Status status()
- {
- return Status.READING;
- }
-
- @Override
- boolean isPurgeable(long nanoTime)
- {
- return nanoTime - expiresAtNanos > 0;
- }
-
- private State maybeComplete()
- {
- if (read == null || summaries.size() < replicaPlan.readQuorum())
- return this;
-
- Map<InetAddressAndPort, MutationSummary> summaryMap = new HashMap<>();
- summaries.snapshot().forEach(rc -> summaryMap.put(rc.from, rc.summary));
-
- Map<InetAddressAndPort, ReconciliationPlan> reconciliations = ReconciliationPlan.calculateReconciliation(summaryMap);
-
- if (reconciliations.isEmpty())
- {
- logger.trace("Read complete for {}", readId);
- complete(promise, read, command.columnFilter(), replicaPlan.consistencyLevel(), expiresAtNanos);
- return COMPLETED;
- }
- else
- {
- logger.trace("Beginning reconciliation for {}", readId);
- Reconciling reconciling = new Reconciling(readId, promise, command, read, replicaPlan.consistencyLevel(), expiresAtNanos, reconciliations);
- reconciling.start(); // TODO: don't do this until after the coordinator state is set to reconciling if converting to lock free
- return reconciling;
- }
- }
-
- @Override
- State receiveInProgressRead(PartialTrackedRead read, MutationSummary summary)
- {
- if (this.read != null)
- return this;
-
- logger.trace("In progress read received for {}", readId);
- this.read = read;
- summaries.add(new ReceivedSummary(FBUtilities.getBroadcastAddressAndPort(), summary));
-
- return maybeComplete();
- }
-
- @Override
- State receiveSummary(InetAddressAndPort from, MutationSummary summary)
- {
- logger.trace("Summary received from {} for {}", from, readId);
- summaries.add(new ReceivedSummary(from, summary));
- return maybeComplete();
- }
-
- @Override
- State abort()
- {
- logger.trace("Aborting read {}", readId);
- if (read != null)
- read.close();
- return ABORTED;
- }
- }
-
- private static class PendingSync
- {
- final int syncId;
- final InetAddressAndPort from;
- final InetAddressAndPort to;
- final Log2OffsetsMap.Immutable plan;
-
- PendingSync(int syncId, InetAddressAndPort from, InetAddressAndPort to, Log2OffsetsMap.Immutable plan)
- {
- this.syncId = syncId;
- this.from = from;
- this.to = to;
- this.plan = plan;
- }
-
- ReadReconcileSend.PeerSync toPeerSync()
- {
- return new ReadReconcileSend.PeerSync(syncId, to, plan);
- }
- }
-
- private static class Reconciling extends State
- {
- private final TrackedRead.Id readId;
- private final AsyncPromise<TrackedDataResponse> promise;
- private final ReadCommand command;
- private final PartialTrackedRead read;
- private final ConsistencyLevel consistencyLevel;
- private final long expiresAtNanos;
-
- final Map<InetAddressAndPort, ReconciliationPlan> plans;
- final Log2OffsetsMap.Mutable outstandingMutations = new Log2OffsetsMap.Mutable();
- final Map<Integer, PendingSync> pendingSync = new ConcurrentHashMap<>();
- final int blockFor;
-
- Reconciling(
- TrackedRead.Id readId,
- AsyncPromise<TrackedDataResponse> promise,
- ReadCommand command,
- PartialTrackedRead read,
- ConsistencyLevel consistencyLevel,
- long expiresAtNanos,
- Map<InetAddressAndPort, ReconciliationPlan> plans)
- {
- this.readId = readId;
- this.promise = promise;
- this.command = command;
- this.read = Preconditions.checkNotNull(read);
- this.consistencyLevel = consistencyLevel;
- this.expiresAtNanos = expiresAtNanos;
- this.plans = plans;
-
- int syncs = 0;
- int nextSyncId = 0;
- for (Map.Entry<InetAddressAndPort, ReconciliationPlan> entry : plans.entrySet())
- {
- InetAddressAndPort from = entry.getKey();
- ReconciliationPlan plan = entry.getValue();
- for (InetAddressAndPort to : plan.nodes())
- {
- int syncId = nextSyncId++;
- PendingSync sync = new PendingSync(syncId, from, to, plan.peerReconciliation(to));
- pendingSync.put(syncId, sync);
- syncs++;
- if (to.equals(FBUtilities.getBroadcastAddressAndPort()))
- {
- outstandingMutations.addAll(plan.offsetsFor(to));
- }
- }
- }
-
- if (logger.isTraceEnabled())
- logger.trace("Reconciling {} syncs, {} mutations for {}", syncs, outstandingMutations.idCount(), readId);
-
- this.blockFor = syncs;
- }
-
- @Override
- Status status()
- {
- return Status.RECONCILING;
- }
-
- @Override
- boolean isPurgeable(long nanoTime)
- {
- return nanoTime - expiresAtNanos > 0;
- }
-
- void start()
- {
- ReadRepairMetrics.trackedReconcile.mark();
- ColumnFamilyStore.metricsFor(command.metadata().id).readRepairRequests.mark();
-
- Map<InetAddressAndPort, List<ReadReconcileSend.PeerSync>> peerSync = new HashMap<>();
- pendingSync.values().forEach(pending ->
- peerSync.computeIfAbsent(pending.from, node -> new ArrayList<>()).add(pending.toPeerSync()));
-
- for (Map.Entry<InetAddressAndPort, List<ReadReconcileSend.PeerSync>> entry : peerSync.entrySet())
- {
- Message<ReadReconcileSend> message = Message.out(Verb.READ_RECONCILE_SEND, new ReadReconcileSend(readId, entry.getValue()));
- logger.trace("Sending read reconciliation for {} {} to {}", readId, message.payload, entry.getKey());
- MessagingService.instance().send(message, entry.getKey());
- }
- }
-
- private State maybeComplete()
- {
- if (!pendingSync.isEmpty() || !outstandingMutations.isEmpty())
- return this;
-
- logger.trace("Reconciliation completed for read {}", readId);
- complete(promise, read, command.columnFilter(), consistencyLevel, expiresAtNanos);
- return COMPLETED;
- }
-
- @Override
- State acknowledgeSync(int syncId)
- {
- logger.trace("Reconciliation sync {} received for {}", syncId, readId);
- pendingSync.remove(syncId);
- return maybeComplete();
- }
-
- @Override
- State receiveMutations(List<Mutation> mutations)
- {
- Log2OffsetsMap.Mutable received = new Log2OffsetsMap.Mutable();
- mutations.forEach(mutation -> {
- logger.trace("Received mutation {} for read {}", mutation.id(), readId);
- received.add(mutation.id());
- });
- outstandingMutations.removeAll(received);
-
- if (logger.isTraceEnabled())
- logger.trace("Received {} mutations, {} mutations outstanding for {}", mutations.size(), outstandingMutations.idCount(), readId);
- read.augment(mutations);
- return maybeComplete();
- }
-
- @Override
- State abort()
- {
- read.close();
- return ABORTED;
- }
- }
-
- @Override
- public String toString()
- {
- return "TrackedLocalReadCoordinator{" + readId + ':' + state.status() + '}';
- }
-
- @VisibleForTesting
- public static void processDelta(PartialTrackedRead read, MutationSummary initialSummary, MutationSummary secondarySummary)
- {
- // Compute any mutations that we could've missed during initial read execution.
- ArrayList<ShortMutationId> delta = new ArrayList<>();
- MutationSummary.difference(secondarySummary, initialSummary, delta);
-
- delta.forEach(mutationId -> {
- Mutation mutation = MutationJournal.instance.read(mutationId);
- Preconditions.checkNotNull(mutation);
- read.augment(mutation);
- });
- }
-
- public void startLocalRead(TrackedRead.Id readId, ReadCommand command, ReplicaPlan.AbstractForRead<?, ?> replicaPlan, int[] summaryNodes, long expiresAtNanos, Consumer<PartialTrackedRead> partialReadConsumer)
- {
- synchronized (this)
- {
- if (!(state = state.startLocalRead(readId, promise, command, replicaPlan, summaryNodes, expiresAtNanos)).isReading())
- return;
- }
-
- PartialTrackedRead read;
- MutationSummary secondarySummary;
-
- MutationSummary initialSummary = command.createMutationSummary(false);
- ReadExecutionController controller = command.executionController(false);
- try
- {
- read = command.beginTrackedRead(controller);
- if (partialReadConsumer != null)
- partialReadConsumer.accept(read);
- // Create another summary once initial data has been read fully. We do this to catch
- // any mutations that may have arrived during initial read execution.
- secondarySummary = command.createMutationSummary(true);
- processDelta(read, initialSummary, secondarySummary);
- }
- catch (Exception e)
- {
- controller.close();
- abort();
- throw e;
- }
-
- synchronized (this)
- {
- state = state.receiveInProgressRead(read, secondarySummary);
- }
- }
-
- private static void complete(AsyncPromise<TrackedDataResponse> promise, PartialTrackedRead read, ColumnFilter selection, ConsistencyLevel consistencyLevel, long expiresAtNanos)
- {
- Stage.READ.submit(() -> completeInternal(promise, read, selection, consistencyLevel, expiresAtNanos));
- }
-
- private static void completeInternal(AsyncPromise<TrackedDataResponse> promise, PartialTrackedRead read, ColumnFilter selection, ConsistencyLevel consistencyLevel, long expiresAtNanos)
- {
- try (PartialTrackedRead.CompletedRead completedRead = read.complete())
- {
- TrackedDataResponse response = completedRead.response();
- Future<TrackedDataResponse> followUp = completedRead.followupRead(response, consistencyLevel, expiresAtNanos);
-
- if (followUp != null)
- {
- followUp.addCallback((newResponse, error) -> {
- if (error != null)
- {
- promise.tryFailure(error);
- return;
- }
- promise.trySuccess(newResponse);
- });
- }
- else
- {
- promise.trySuccess(response);
- }
- }
- catch (Exception e)
- {
- promise.tryFailure(e);
- throw e;
- }
- finally
- {
- read.close();
- }
- }
-
- synchronized void receiveSummary(InetAddressAndPort from, MutationSummary summary)
- {
- if (logger.isTraceEnabled())
- logger.trace("Received summary {} from {}, for {}", summary, from, state);
- state = state.receiveSummary(from, summary);
- }
-
- synchronized boolean acknowledgeSync(int syncId)
- {
- return (state = state.acknowledgeSync(syncId)).isCompleted();
- }
-
- synchronized boolean receiveMutations(List<Mutation> mutations)
- {
- return (state = state.receiveMutations(mutations)).isCompleted();
- }
-
- synchronized void abort()
- {
- state = state.abort();
- }
-
- boolean isTimedOutOrComplete(long nanoTime)
- {
- return state.isPurgeable(nanoTime);
- }
-}
diff --git a/src/java/org/apache/cassandra/service/reads/tracked/TrackedLocalReads.java b/src/java/org/apache/cassandra/service/reads/tracked/TrackedLocalReads.java
index 24bc0b5260..4f8bca8f25 100644
--- a/src/java/org/apache/cassandra/service/reads/tracked/TrackedLocalReads.java
+++ b/src/java/org/apache/cassandra/service/reads/tracked/TrackedLocalReads.java
@@ -15,88 +15,69 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.cassandra.service.reads.tracked;
-import org.apache.cassandra.concurrent.ScheduledExecutorPlus;
-import org.apache.cassandra.concurrent.Shutdownable;
-import org.apache.cassandra.config.DatabaseDescriptor;
+import java.util.ArrayList;
+import java.util.Map;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+
+import org.apache.cassandra.concurrent.Stage;
import org.apache.cassandra.db.*;
-import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.locator.ReplicaPlan;
import org.apache.cassandra.locator.ReplicaPlans;
import org.apache.cassandra.service.reads.ReadCoordinator;
+import org.apache.cassandra.metrics.ReadRepairMetrics;
+import org.apache.cassandra.replication.ExpiredStatePurger;
+import org.apache.cassandra.replication.Log2OffsetsMap;
+import org.apache.cassandra.replication.MutationJournal;
+import org.apache.cassandra.replication.MutationSummary;
+import org.apache.cassandra.replication.ShortMutationId;
import org.apache.cassandra.service.reads.SpeculativeRetryPolicy;
import org.apache.cassandra.tcm.ClusterMetadata;
-import org.apache.cassandra.utils.Clock;
+import org.apache.cassandra.utils.concurrent.AsyncPromise;
+import org.apache.cassandra.utils.concurrent.Future;
+import org.jctools.maps.NonBlockingHashMap;
+
+import org.apache.cassandra.transport.Dispatcher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
-import static java.util.concurrent.TimeUnit.MINUTES;
-import static java.util.concurrent.TimeUnit.NANOSECONDS;
-import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
-import static org.apache.cassandra.concurrent.ExecutorFactory.SimulatorSemantics.NORMAL;
-
/**
- * Since the read reconciliations don't use 2 way callbacks, a map of active reconciliations
+ * Since the read reconciliations don't use 2 way callbacks, maps of active reads and reconciliations
* are maintained and expired here.
- *
+ * <p>
* Borrowed heavily from RequestCallbacks
*/
-public class TrackedLocalReads implements Shutdownable
+public class TrackedLocalReads implements ExpiredStatePurger.Expireable
{
private static final Logger logger = LoggerFactory.getLogger(TrackedLocalReads.class);
- private final ConcurrentMap<TrackedRead.Id, TrackedLocalReadCoordinator> reads = new ConcurrentHashMap<>();
- private final ScheduledExecutorPlus executor = executorFactory().scheduled("Reconciliation-Map-Reaper", NORMAL);
+ private final NonBlockingHashMap<TrackedRead.Id, Coordinator> coordinators = new NonBlockingHashMap<>();
public TrackedLocalReads()
{
- long expirationInterval = defaultExpirationInterval();
- executor.scheduleWithFixedDelay(this::expire, expirationInterval, expirationInterval, NANOSECONDS);
+ ExpiredStatePurger.instance.register(this);
}
- private void expire()
- {
- long start = Clock.Global.nanoTime();
- int n = 0;
- for (Map.Entry<TrackedRead.Id, TrackedLocalReadCoordinator> entry : reads.entrySet())
- {
- TrackedLocalReadCoordinator read = entry.getValue();
- if (read.isTimedOutOrComplete(start))
- {
- read.abort();
- if (reads.remove(entry.getKey(), entry.getValue()))
- n++;
- }
- }
- if (n > 0)
- logger.trace("Expired {} entries", n);
- }
-
- private TrackedLocalReadCoordinator getOrCreate(TrackedRead.Id id)
- {
- return reads.computeIfAbsent(id, TrackedLocalReadCoordinator::new);
- }
-
- public void receiveSummary(InetAddressAndPort from, TrackedSummaryResponse summary)
- {
- getOrCreate(summary.readId()).receiveSummary(from, summary.summary());
- }
-
- public TrackedLocalReadCoordinator beginRead(TrackedRead.Id readId, ClusterMetadata metadata, ReadCommand command, ConsistencyLevel consistencyLevel, int[] summaryNodes, long expiresAtNanos, Consumer<PartialTrackedRead> partialReadConsumer)
+ public AsyncPromise<TrackedDataResponse> beginRead(
+ TrackedRead.Id readId,
+ ClusterMetadata metadata,
+ ReadCommand command,
+ ConsistencyLevel consistencyLevel,
+ int[] summaryNodes,
+ Dispatcher.RequestTime requestTime,
+ Consumer<PartialTrackedRead> partialReadConsumer)
{
Keyspace keyspace = Keyspace.open(command.metadata().keyspace);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(command.metadata().id);
SpeculativeRetryPolicy retry = cfs.metadata().params.speculativeRetry;
ReplicaPlan.AbstractForRead<?, ?> replicaPlan;
+
if (command instanceof SinglePartitionReadCommand)
{
replicaPlan = ReplicaPlans.forRead(metadata,
@@ -119,69 +100,170 @@ public class TrackedLocalReads implements Shutdownable
1);
}
// TODO: confirm all summaryNodes are present in the replica plan
-
- TrackedLocalReadCoordinator coordinator = getOrCreate(readId);
- coordinator.startLocalRead(readId, command, replicaPlan, summaryNodes, expiresAtNanos, partialReadConsumer);
- return coordinator;
+ AsyncPromise<TrackedDataResponse> promise = new AsyncPromise<>();
+ beginReadInternal(readId, command, replicaPlan, summaryNodes, requestTime, partialReadConsumer, promise);
+ return promise;
}
- public void acknowledgeSync(TrackedRead.Id readId, int syncId)
+ // TODO (expected): skip local summaries and reconcile when summaryNodes is empty (e.g. for CL.ONE)
+ private void beginReadInternal(
+ TrackedRead.Id readId,
+ ReadCommand command,
+ ReplicaPlan.AbstractForRead<?, ?> replicaPlan,
+ int[] summaryNodes,
+ Dispatcher.RequestTime requestTime,
+ Consumer<PartialTrackedRead> partialReadConsumer,
+ AsyncPromise<TrackedDataResponse> promise)
{
- TrackedLocalReadCoordinator read = reads.get(readId);
- if (read == null)
- return;
+ PartialTrackedRead read = null;
+ MutationSummary secondarySummary;
- if (read.acknowledgeSync(syncId))
- reads.remove(readId);
- }
+ MutationSummary initialSummary = command.createMutationSummary(false);
+ ReadExecutionController controller = command.executionController(false);
- public boolean receiveMutations(TrackedRead.Id readId, int syncId, List<Mutation> mutations)
- {
- TrackedLocalReadCoordinator read = reads.get(readId);
- if (read == null)
- return false;
-
- read.receiveMutations(mutations);
- if (read.acknowledgeSync(syncId))
- reads.remove(readId);
- return true;
- }
+ try
+ {
+ read = command.beginTrackedRead(controller);
+ // Create another summary once initial data has been read fully. We do this to catch
+ // any mutations that may have arrived during initial read execution.
+ secondarySummary = command.createMutationSummary(true);
+ processDelta(read, initialSummary, secondarySummary);
+ }
+ catch (Exception e)
+ {
+ controller.close();
+ logger.trace("Aborting read {}", readId);
+ if (read != null) read.close();
+ throw e;
+ }
- public static long defaultExpirationInterval()
- {
- return DatabaseDescriptor.getMinRpcTimeout(NANOSECONDS) / 2;
+ Coordinator coordinator =
+ new Coordinator(readId, promise, command.columnFilter(), read, replicaPlan.consistencyLevel(), requestTime);
+ coordinators.put(readId, coordinator);
+
+ // TODO (expected): reconsider the approach to tracked mutation metrics
+ ReadRepairMetrics.trackedReconcile.mark();
+
+ // TODO (consider): is it possible to exit right here if this is the last missing summary, and we can tell that
+ // no node needs anything from anyone?
+
+ // pass the final summary to the reconcile service
+ ReadReconciliations.instance.acceptLocalSummary(readId, secondarySummary, summaryNodes);
}
- @Override
- public void shutdown()
+ @VisibleForTesting
+ public static void processDelta(PartialTrackedRead read, MutationSummary initialSummary, MutationSummary secondarySummary)
{
- executor.shutdown();
+ // Compute any mutations that we could've missed during initial read execution.
+ ArrayList<ShortMutationId> delta = new ArrayList<>();
+ MutationSummary.difference(secondarySummary, initialSummary, delta);
+
+ delta.forEach(mutationId -> {
+ Mutation mutation = MutationJournal.instance.read(mutationId);
+ Preconditions.checkNotNull(mutation);
+ read.augment(mutation);
+ });
}
- @Override
- public boolean isTerminated()
+ public void acknowledgeReconcile(TrackedRead.Id readId, Log2OffsetsMap<?> augmentingOffsets)
{
- return executor.isTerminated();
+ Coordinator coordinator = coordinators.remove(readId);
+ if (coordinator != null)
+ coordinator.acknowledgeReconcile(augmentingOffsets);
}
@Override
- public Object shutdownNow()
+ public int expire(long nanoTime)
{
- return executor.shutdownNow();
+ int n = 0;
+ for (Map.Entry<TrackedRead.Id, Coordinator> entry : coordinators.entrySet())
+ {
+ TrackedRead.Id id = entry.getKey();
+ Coordinator coordinator = entry.getValue();
+ if (coordinator.isPurgeable(nanoTime) && coordinators.remove(id, coordinator))
+ {
+ coordinator.abort();
+ n++;
+ }
+ }
+ return n;
}
- public void shutdownBlocking() throws InterruptedException
+ private static class Coordinator
{
- if (executor == null || executor.isTerminated())
- return;
+ private final TrackedRead.Id readId;
+ private final AsyncPromise<TrackedDataResponse> promise;
+ private final ColumnFilter selection;
+ private final PartialTrackedRead read;
+ private final ConsistencyLevel consistencyLevel;
+ private final Dispatcher.RequestTime requestTime;
- executor.shutdown();
- executor.awaitTermination(1, MINUTES);
- }
+ Coordinator(
+ TrackedRead.Id readId,
+ AsyncPromise<TrackedDataResponse> promise,
+ ColumnFilter selection,
+ PartialTrackedRead read,
+ ConsistencyLevel consistencyLevel,
+ Dispatcher.RequestTime requestTime)
+ {
+ this.readId = readId;
+ this.promise = promise;
+ this.selection = selection;
+ this.read = Preconditions.checkNotNull(read);
+ this.consistencyLevel = consistencyLevel;
+ this.requestTime = requestTime;
+ }
- @Override
- public boolean awaitTermination(long timeout, TimeUnit units) throws InterruptedException
- {
- return executor.awaitTermination(timeout, units);
+ boolean isPurgeable(long nanoTime)
+ {
+ long deadline = requestTime.computeDeadline(read.command().verb().expiresAfterNanos());
+ return nanoTime - deadline > 0;
+ }
+
+ void abort()
+ {
+ read.close();
+ }
+
+ void acknowledgeReconcile(Log2OffsetsMap<?> augmentingOffsets)
+ {
+ logger.trace("Reconciliation completed for {}, missing {}", readId, augmentingOffsets);
+ Stage.READ.submit(() -> { read.augment(augmentingOffsets); complete(); });
+ }
+
+ private void complete()
+ {
+ try (PartialTrackedRead.CompletedRead completedRead = read.complete())
+ {
+ TrackedDataResponse response = completedRead.response();
+ Future<TrackedDataResponse> followUp = completedRead.followupRead(response, consistencyLevel, requestTime);
+
+ if (followUp != null)
+ {
+ ReadCommand command = read.command();
+ followUp.addCallback((newResponse, error) -> {
+ if (error != null)
+ {
+ promise.tryFailure(error);
+ return;
+ }
+ promise.trySuccess(newResponse);
+ });
+ }
+ else
+ {
+ promise.trySuccess(response);
+ }
+ }
+ catch (Exception e)
+ {
+ promise.tryFailure(e);
+ throw e;
+ }
+ finally
+ {
+ read.close();
+ }
+ }
}
}
diff --git a/src/java/org/apache/cassandra/service/reads/tracked/TrackedRead.java b/src/java/org/apache/cassandra/service/reads/tracked/TrackedRead.java
index 0d029a554c..267670a63e 100644
--- a/src/java/org/apache/cassandra/service/reads/tracked/TrackedRead.java
+++ b/src/java/org/apache/cassandra/service/reads/tracked/TrackedRead.java
@@ -29,6 +29,8 @@ import org.apache.cassandra.db.rows.RowIterator;
import org.apache.cassandra.exceptions.ReadFailureException;
import org.apache.cassandra.exceptions.ReadTimeoutException;
import org.apache.cassandra.exceptions.RequestFailureReason;
+import org.apache.cassandra.exceptions.UnavailableException;
+import org.apache.cassandra.gms.FailureDetector;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
@@ -58,6 +60,7 @@ import java.util.function.Consumer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.readMetrics;
public abstract class TrackedRead<E extends Endpoints<E>, P extends ReplicaPlan.ForRead<E, P>> implements RequestCallback<TrackedDataResponse>
@@ -140,6 +143,7 @@ public abstract class TrackedRead<E extends Endpoints<E>, P extends ReplicaPlan.
private final ReadCommand command;
private final ReplicaPlan.AbstractForRead<E, P> replicaPlan;
private final ConsistencyLevel consistencyLevel;
+ private final Dispatcher.RequestTime requestTime;
private static class RequestFailure extends Throwable
{
@@ -158,11 +162,12 @@ public abstract class TrackedRead<E extends Endpoints<E>, P extends ReplicaPlan.
}
}
- public TrackedRead(ReadCommand command, ReplicaPlan.AbstractForRead<E, P> replicaPlan, ConsistencyLevel consistencyLevel)
+ public TrackedRead(ReadCommand command, ReplicaPlan.AbstractForRead<E, P> replicaPlan, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
{
this.command = command;
this.replicaPlan = replicaPlan;
this.consistencyLevel = consistencyLevel;
+ this.requestTime = requestTime;
}
@Override
@@ -180,12 +185,12 @@ public abstract class TrackedRead<E extends Endpoints<E>, P extends ReplicaPlan.
public static class Partition extends TrackedRead<EndpointsForToken, ReplicaPlan.ForTokenRead>
{
- private Partition(SinglePartitionReadCommand command, ReplicaPlan.AbstractForRead<EndpointsForToken, ReplicaPlan.ForTokenRead> replicaPlan, ConsistencyLevel consistencyLevel)
+ private Partition(SinglePartitionReadCommand command, ReplicaPlan.AbstractForRead<EndpointsForToken, ReplicaPlan.ForTokenRead> replicaPlan, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
{
- super(command, replicaPlan, consistencyLevel);
+ super(command, replicaPlan, consistencyLevel, requestTime);
}
- public static Partition create(ClusterMetadata metadata, SinglePartitionReadCommand command, ConsistencyLevel consistencyLevel)
+ public static Partition create(ClusterMetadata metadata, SinglePartitionReadCommand command, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
{
Preconditions.checkArgument(command.metadata().replicationType().isTracked());
Keyspace keyspace = Keyspace.open(command.metadata().keyspace);
@@ -199,7 +204,7 @@ public abstract class TrackedRead<E extends Endpoints<E>, P extends ReplicaPlan.
consistencyLevel,
retry,
ReadCoordinator.DEFAULT);
- return new Partition(command, replicaPlan, consistencyLevel);
+ return new Partition(command, replicaPlan, consistencyLevel, requestTime);
}
@Override
@@ -211,15 +216,15 @@ public abstract class TrackedRead<E extends Endpoints<E>, P extends ReplicaPlan.
public static class Range extends TrackedRead<EndpointsForRange, ReplicaPlan.ForRangeRead>
{
- private Range(PartitionRangeReadCommand command, ReplicaPlan.AbstractForRead<EndpointsForRange, ReplicaPlan.ForRangeRead> replicaPlan, ConsistencyLevel consistencyLevel)
+ private Range(PartitionRangeReadCommand command, ReplicaPlan.AbstractForRead<EndpointsForRange, ReplicaPlan.ForRangeRead> replicaPlan, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
{
- super(command, replicaPlan, consistencyLevel);
+ super(command, replicaPlan, consistencyLevel, requestTime);
}
- public static TrackedRead.Range create(PartitionRangeReadCommand command, ReplicaPlan.ForRangeRead replicaPlan)
+ public static TrackedRead.Range create(PartitionRangeReadCommand command, ReplicaPlan.ForRangeRead replicaPlan, Dispatcher.RequestTime requestTime)
{
Preconditions.checkArgument(command.metadata().replicationType().isTracked());
- return new Range(command, replicaPlan, replicaPlan.consistencyLevel());
+ return new Range(command, replicaPlan, replicaPlan.consistencyLevel(), requestTime);
}
@Override
@@ -239,7 +244,13 @@ public abstract class TrackedRead<E extends Endpoints<E>, P extends ReplicaPlan.
return hostids;
}
- private void start(long expiresAt, Consumer<PartialTrackedRead> partialReadConsumer)
+ private static int endpointToHostId(Replica replica)
+ {
+ ClusterMetadata metadata = ClusterMetadata.current();
+ return metadata.directory.peerId(replica.endpoint()).id();
+ }
+
+ private void start(Dispatcher.RequestTime requestTime, Consumer<PartialTrackedRead> partialReadConsumer)
{
// TODO: skip local coordination if this node knows its recovering from an outage
// TODO: read speculation
@@ -252,20 +263,25 @@ public abstract class TrackedRead<E extends Endpoints<E>, P extends ReplicaPlan.
// create an id
// select data node
// select summary nodes
- E selected = replicaPlan.contacts();
- Replica dataNode = localReplica != null && localReplica.isFull()
- ? localReplica
- : Iterables.getOnlyElement(selected.filter(Replica::isFull, 1));
- E summaryNodes = selected.filter(r -> r != dataNode);
- int[] summaryHostIds = endpointsToHostIds(summaryNodes);
-
-
- if (dataNode == localReplica)
+ E selected = replicaPlan.contacts().filter(r -> FailureDetector.instance.isAlive(r.endpoint()));
+ if (selected.size() < replicaPlan.readQuorum())
+ throw new UnavailableException(String.format("Insufficient replicas available for read (%d < %d)", selected.size(), replicaPlan.readQuorum()),
+ replicaPlan.consistencyLevel(), selected.size(), replicaPlan.readQuorum());
+ Replica dataReplica = localReplica != null && localReplica.isFull()
+ ? localReplica
+ : Iterables.getOnlyElement(selected.filter(Replica::isFull, 1));
+ E summaryReplicas = selected.filter(r -> r != dataReplica);
+
+ int dataNode = endpointToHostId(dataReplica);
+ int[] summaryNodes = endpointsToHostIds(summaryReplicas);
+
+ if (dataReplica == localReplica)
{
logger.trace("Locally coordinating {}", readId);
Stage.READ.submit(() -> {
- TrackedLocalReadCoordinator coordinator = MutationTrackingService.instance.localReads().beginRead(readId, ClusterMetadata.current(), command, consistencyLevel, summaryHostIds, expiresAt, partialReadConsumer);
- coordinator.addCallback((response, error) -> {
+ AsyncPromise<TrackedDataResponse> promise =
+ MutationTrackingService.instance.localReads().beginRead(readId, ClusterMetadata.current(), command, consistencyLevel, summaryNodes, requestTime, partialReadConsumer);
+ promise.addCallback((response, error) -> {
if (error != null)
{
// TODO: notify coordinator that read has failed
@@ -279,22 +295,22 @@ public abstract class TrackedRead<E extends Endpoints<E>, P extends ReplicaPlan.
}
else
{
- logger.trace("Sending data request for {} to {}", readId, dataNode.endpoint());
+ logger.trace("Sending data request for {} to {}", readId, dataReplica.endpoint());
Preconditions.checkArgument(partialReadConsumer == null, "Cannot supply read consumer for nonlocal reads");
- DataRequest dataRequest = new DataRequest(readId, command, consistencyLevel, summaryHostIds);
- Message<DataRequest> dataMessage = Message.outWithFlag(verb(), dataRequest, MessageFlag.CALL_BACK_ON_FAILURE);
- MessagingService.instance().sendWithCallback(dataMessage, dataNode.endpoint(), this);
+ DataRequest dataRequest = new DataRequest(readId, command, dataNode, summaryNodes, consistencyLevel);
+ Message<DataRequest> dataMessage = Message.builder(verb(), dataRequest)
+ .withRequestTime(requestTime)
+ .withFlag(MessageFlag.CALL_BACK_ON_FAILURE)
+ .build();
+ MessagingService.instance().sendWithCallback(dataMessage, dataReplica.endpoint(), this);
}
- if (summaryNodes.isEmpty())
+ if (summaryReplicas.isEmpty())
return;
- SummaryRequest summaryRequest = new SummaryRequest(readId, command);
- Message<SummaryRequest> summaryMessage = Message.outWithParam(Verb.TRACKED_SUMMARY_REQ,
- summaryRequest,
- ParamType.RESPOND_TO,
- dataNode.endpoint());
- for (Replica replica : summaryNodes)
+ SummaryRequest summaryRequest = new SummaryRequest(readId, command, dataNode, summaryNodes);
+ Message<SummaryRequest> summaryMessage = Message.outWithRequestTime(Verb.TRACKED_SUMMARY_REQ, summaryRequest, requestTime);
+ for (Replica replica : summaryReplicas)
{
if (localReplica == replica)
{
@@ -309,19 +325,14 @@ public abstract class TrackedRead<E extends Endpoints<E>, P extends ReplicaPlan.
}
}
- public void start(long expiresAt)
- {
- start(expiresAt, null);
- }
-
- public void startLocal(long expiresAt, Consumer<PartialTrackedRead> partialReadConsumer)
+ public void start(Dispatcher.RequestTime requestTime)
{
- start(expiresAt, partialReadConsumer);
+ start(requestTime, null);
}
- public void start(Dispatcher.RequestTime requestTime)
+ public void startLocal(Dispatcher.RequestTime requestTime, Consumer<PartialTrackedRead> partialReadConsumer)
{
- start(requestTime.computeDeadline(verb().expiresAfterNanos()));
+ start(requestTime, partialReadConsumer);
}
private void onResponse(TrackedDataResponse response)
@@ -356,7 +367,7 @@ public abstract class TrackedRead<E extends Endpoints<E>, P extends ReplicaPlan.
{
try
{
- return future.get(command.getTimeout(TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS).makeIterator(command);
+ return future.get(Math.max(0, requestTime.computeDeadline(verb().expiresAfterNanos()) - Clock.Global.nanoTime()), NANOSECONDS).makeIterator(command);
}
catch (InterruptedException e)
{
@@ -407,13 +418,17 @@ public abstract class TrackedRead<E extends Endpoints<E>, P extends ReplicaPlan.
public abstract static class Request
{
- protected final Id readId;
- protected final ReadCommand command;
+ public final Id readId;
+ public final ReadCommand command;
+ public final int dataNode;
+ public final int[] summaryNodes;
- protected Request(Id readId, ReadCommand command)
+ protected Request(Id readId, ReadCommand command, int dataNode, int[] summaryNodes)
{
this.readId = readId;
this.command = command;
+ this.dataNode = dataNode;
+ this.summaryNodes = summaryNodes;
}
public abstract void executeLocally(Message<? extends Request> message, ClusterMetadata metadata);
@@ -422,20 +437,22 @@ public abstract class TrackedRead<E extends Endpoints<E>, P extends ReplicaPlan.
public static class DataRequest extends Request
{
private final ConsistencyLevel consistencyLevel;
- private final int[] summaryNodes;
- public DataRequest(Id readId, ReadCommand command, ConsistencyLevel consistencyLevel, int[] summaryNodes)
+ public DataRequest(Id readId, ReadCommand command, int dataNode, int[] summaryNodes, ConsistencyLevel consistencyLevel)
{
- super(readId, command);
+ super(readId, command, dataNode, summaryNodes);
this.consistencyLevel = consistencyLevel;
- this.summaryNodes = summaryNodes;
}
@Override
public void executeLocally(Message<? extends Request> message, ClusterMetadata metadata)
{
- TrackedLocalReadCoordinator coordinator = MutationTrackingService.instance.localReads().beginRead(readId, metadata, command, consistencyLevel, summaryNodes, message.expiresAtNanos(), null);
- coordinator.addCallback((response, error) -> {
+ Dispatcher.RequestTime requestTime = new Dispatcher.RequestTime(message.createdAtNanos());
+ AsyncPromise<TrackedDataResponse> promise =
+ MutationTrackingService.instance
+ .localReads()
+ .beginRead(readId, metadata, command, consistencyLevel, summaryNodes, requestTime, null);
+ promise.addCallback((response, error) -> {
if (error != null)
{
// TODO: notify coordinator that read has failed
@@ -453,10 +470,11 @@ public abstract class TrackedRead<E extends Endpoints<E>, P extends ReplicaPlan.
{
Id.serializer.serialize(request.readId, out, version);
ReadCommand.serializer.serialize(request.command, out, version);
- out.writeInt(request.consistencyLevel.code);
+ out.writeInt(request.dataNode);
out.writeInt(request.summaryNodes.length);
for (int hostid : request.summaryNodes)
out.writeInt(hostid);
+ out.writeInt(request.consistencyLevel.code);
}
@Override
@@ -464,11 +482,12 @@ public abstract class TrackedRead<E extends Endpoints<E>, P extends ReplicaPlan.
{
Id readId = Id.serializer.deserialize(in, version);
ReadCommand command = ReadCommand.serializer.deserialize(in, version);
- ConsistencyLevel consistencyLevel = ConsistencyLevel.fromCode(in.readInt());
+ int dataNode = in.readInt();
int[] summaryNodes = new int[in.readInt()];
for (int i = 0; i < summaryNodes.length; i++)
summaryNodes[i] = in.readInt();
- return new DataRequest(readId, command, consistencyLevel, summaryNodes);
+ ConsistencyLevel consistencyLevel = ConsistencyLevel.fromCode(in.readInt());
+ return new DataRequest(readId, command, dataNode, summaryNodes, consistencyLevel);
}
@Override
@@ -476,26 +495,30 @@ public abstract class TrackedRead<E extends Endpoints<E>, P extends ReplicaPlan.
{
return Id.serializer.serializedSize(request.readId, version) +
ReadCommand.serializer.serializedSize(request.command, version) +
- TypeSizes.sizeof(request.consistencyLevel.code) +
+ TypeSizes.sizeof(request.dataNode) +
TypeSizes.sizeof(request.summaryNodes.length) +
- ((long) TypeSizes.INT_SIZE * request.summaryNodes.length);
+ ((long) TypeSizes.INT_SIZE * request.summaryNodes.length) +
+ TypeSizes.sizeof(request.consistencyLevel.code);
}
};
}
public static class SummaryRequest extends Request
{
- public SummaryRequest(Id readId, ReadCommand command)
+ public SummaryRequest(Id readId, ReadCommand command, int dataNode, int[] summaryNodes)
{
- super(readId, command);
+ super(readId, command, dataNode, summaryNodes);
}
@Override
public void executeLocally(Message<? extends Request> message, ClusterMetadata metadata)
{
- MutationSummary summary = command.createMutationSummary(false);
- TrackedSummaryResponse response = new TrackedSummaryResponse(readId, summary);
- MessagingService.instance().send(message.responseWith(response), message.respondTo());
+ ReadReconciliations.instance.handleSummaryRequest((SummaryRequest) message.payload);
+ }
+
+ public TrackedSummaryResponse makeResponse(MutationSummary summary)
+ {
+ return new TrackedSummaryResponse(readId, summary, dataNode, summaryNodes);
}
public static final IVersionedSerializer<SummaryRequest> serializer = new IVersionedSerializer<>()
@@ -505,6 +528,10 @@ public abstract class TrackedRead<E extends Endpoints<E>, P extends ReplicaPlan.
{
Id.serializer.serialize(request.readId, out, version);
ReadCommand.serializer.serialize(request.command, out, version);
+ out.writeInt(request.dataNode);
+ out.writeInt(request.summaryNodes.length);
+ for (int hostid : request.summaryNodes)
+ out.writeInt(hostid);
}
@Override
@@ -512,14 +539,21 @@ public abstract class TrackedRead<E extends Endpoints<E>, P extends ReplicaPlan.
{
Id readId = Id.serializer.deserialize(in, version);
ReadCommand command = ReadCommand.serializer.deserialize(in, version);
- return new SummaryRequest(readId, command);
+ int dataNode = in.readInt();
+ int[] summaryNodes = new int[in.readInt()];
+ for (int i = 0; i < summaryNodes.length; i++)
+ summaryNodes[i] = in.readInt();
+ return new SummaryRequest(readId, command, dataNode, summaryNodes);
}
@Override
public long serializedSize(SummaryRequest request, int version)
{
return Id.serializer.serializedSize(request.readId, version) +
- ReadCommand.serializer.serializedSize(request.command, version);
+ ReadCommand.serializer.serializedSize(request.command, version) +
+ TypeSizes.sizeof(request.dataNode) +
+ TypeSizes.sizeof(request.summaryNodes.length) +
+ ((long) TypeSizes.INT_SIZE * request.summaryNodes.length);
}
};
}
diff --git a/src/java/org/apache/cassandra/service/reads/tracked/TrackedSummaryResponse.java b/src/java/org/apache/cassandra/service/reads/tracked/TrackedSummaryResponse.java
index 97f2254eee..ac6a4ddbec 100644
--- a/src/java/org/apache/cassandra/service/reads/tracked/TrackedSummaryResponse.java
+++ b/src/java/org/apache/cassandra/service/reads/tracked/TrackedSummaryResponse.java
@@ -17,38 +17,42 @@
*/
package org.apache.cassandra.service.reads.tracked;
+import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.net.IVerbHandler;
import org.apache.cassandra.replication.MutationSummary;
-import org.apache.cassandra.replication.MutationTrackingService;
import java.io.IOException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
public class TrackedSummaryResponse
{
- private final TrackedRead.Id readId;
- private final MutationSummary summary;
+ private static final Logger logger = LoggerFactory.getLogger(TrackedSummaryResponse.class);
+
+ public final TrackedRead.Id readId;
+ public final MutationSummary summary;
+ public final int dataNode;
+ public final int[] summaryNodes;
- public TrackedSummaryResponse(TrackedRead.Id readId, MutationSummary summary)
+ public TrackedSummaryResponse(TrackedRead.Id readId, MutationSummary summary, int dataNode, int[] summaryNodes)
{
this.readId = readId;
this.summary = summary;
+ this.dataNode = dataNode;
+ this.summaryNodes = summaryNodes;
}
- public TrackedRead.Id readId()
- {
- return readId;
- }
-
- public MutationSummary summary()
+ public static final IVerbHandler<TrackedSummaryResponse> verbHandler = message ->
{
- return summary;
- }
-
- public static final IVerbHandler<TrackedSummaryResponse> verbHandler =
- message -> MutationTrackingService.instance.localReads().receiveSummary(message.from(), message.payload);
+ TrackedSummaryResponse response = message.payload;
+ if (logger.isTraceEnabled())
+ logger.trace("Received summary {} from {}, for {}", response.summary, message.from(), response.readId);
+ ReadReconciliations.instance.acceptRemoteSummary(message.from(), message.payload);
+ };
public static final IVersionedSerializer<TrackedSummaryResponse> serializer = new IVersionedSerializer<>()
{
@@ -57,6 +61,10 @@ public class TrackedSummaryResponse
{
TrackedRead.Id.serializer.serialize(summary.readId, out, version);
MutationSummary.serializer.serialize(summary.summary, out, version);
+ out.writeInt(summary.dataNode);
+ out.writeInt(summary.summaryNodes.length);
+ for (int hostid : summary.summaryNodes)
+ out.writeInt(hostid);
}
@Override
@@ -64,14 +72,21 @@ public class TrackedSummaryResponse
{
TrackedRead.Id id = TrackedRead.Id.serializer.deserialize(in, version);
MutationSummary summary = MutationSummary.serializer.deserialize(in, version);
- return new TrackedSummaryResponse(id, summary);
+ int dataNode = in.readInt();
+ int[] summaryNodes = new int[in.readInt()];
+ for (int i = 0; i < summaryNodes.length; i++)
+ summaryNodes[i] = in.readInt();
+ return new TrackedSummaryResponse(id, summary, dataNode, summaryNodes);
}
@Override
public long serializedSize(TrackedSummaryResponse summary, int version)
{
return TrackedRead.Id.serializer.serializedSize(summary.readId, version) +
- MutationSummary.serializer.serializedSize(summary.summary, version);
+ MutationSummary.serializer.serializedSize(summary.summary, version) +
+ TypeSizes.sizeof(summary.dataNode) +
+ TypeSizes.sizeof(summary.summaryNodes.length) +
+ TypeSizes.INT_SIZE * (long) summary.summaryNodes.length;
}
};
}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTestBase.java b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTestBase.java
index 009399beb5..aa93dd9fec 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTestBase.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTestBase.java
@@ -77,9 +77,9 @@ import static org.apache.cassandra.distributed.api.ConsistencyLevel.QUORUM;
import static org.apache.cassandra.distributed.shared.AssertUtils.assertEquals;
import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows;
import static org.apache.cassandra.distributed.shared.AssertUtils.row;
-import static org.apache.cassandra.net.Verb.READ_RECONCILE_NOTIFY;
-import static org.apache.cassandra.net.Verb.READ_RECONCILE_RCV;
-import static org.apache.cassandra.net.Verb.READ_RECONCILE_SEND;
+import static org.apache.cassandra.net.Verb.PULL_MUTATIONS_REQ;
+import static org.apache.cassandra.net.Verb.PUSH_MUTATION_REQ;
+import static org.apache.cassandra.net.Verb.READ_RECONCILE_ACK;
import static org.apache.cassandra.net.Verb.READ_REPAIR_REQ;
import static org.apache.cassandra.net.Verb.READ_REPAIR_RSP;
import static org.apache.cassandra.net.Verb.READ_REQ;
@@ -128,6 +128,7 @@ public abstract class ReadRepairTestBase extends TestBaseImpl
@Test
public void testBlockingReadRepairAccord() throws Throwable
{
+ MutationTrackingUtils.fixmeSkipIfTracked(replicationType(), "Accord not supported");
testReadRepair(ReadRepairStrategy.BLOCKING, true);
}
@@ -182,7 +183,7 @@ public abstract class ReadRepairTestBase extends TestBaseImpl
cluster.get(1).executeInternal(withTable("INSERT INTO %s (pk, ck, v) VALUES (1, 1, 1)"));
cluster.get(2).executeInternal(withTable("INSERT INTO %s (pk, ck, v) VALUES (1, 1, 1)"));
assertRows(cluster.get(3).executeInternal(withTable("SELECT * FROM %s WHERE pk = 1")));
- cluster.verbs(replicationType().isTracked() ? READ_RECONCILE_NOTIFY : READ_REPAIR_RSP).to(1).drop();
+ cluster.verbs(replicationType().isTracked() ? READ_RECONCILE_ACK : READ_REPAIR_RSP).to(1).drop();
final long start = currentTimeMillis();
try
{
@@ -219,9 +220,10 @@ public abstract class ReadRepairTestBase extends TestBaseImpl
assertRows(cluster.get(3).executeInternal(withTable("SELECT * FROM %s WHERE pk = 1")));
cluster.filters().verbs(READ_REPAIR_REQ.id).to(3).drop();
- cluster.filters().verbs(READ_RECONCILE_SEND.id).to(3).drop();
- cluster.filters().verbs(READ_RECONCILE_RCV.id).to(3).drop();
- assertRows(cluster.coordinator(1).execute(withTable("SELECT * FROM %s WHERE pk = 1"), ConsistencyLevel.QUORUM),
+ cluster.filters().verbs(PULL_MUTATIONS_REQ.id).to(3).drop();
+ cluster.filters().verbs(PUSH_MUTATION_REQ.id).to(3).drop();
+ assertRows(cluster.coordinator(1).execute(withTable("SELECT * FROM %s WHERE pk = 1"),
+ ConsistencyLevel.QUORUM),
row(1, 1, 1));
// Data was not repaired
@@ -268,8 +270,8 @@ public abstract class ReadRepairTestBase extends TestBaseImpl
cluster.filters().verbs(READ_REQ.id).from(4).to(3).drop();
if (replicationType().isTracked())
{
- cluster.filters().verbs(READ_RECONCILE_SEND.id).from(4).to(3).drop();
- cluster.filters().verbs(READ_RECONCILE_RCV.id).from(4).to(3).drop();
+ cluster.filters().verbs(PULL_MUTATIONS_REQ.id).from(4).to(3).drop();
+ cluster.filters().verbs(PUSH_MUTATION_REQ.id).from(4).to(3).drop();
}
else
{
diff --git a/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTester.java b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTester.java
index 9b420b7d22..e599b982a1 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTester.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTester.java
@@ -146,8 +146,13 @@ public abstract class ReadRepairTester<T extends ReadRepairTester<T>>
// verify the number of repaired rows
if (strategy == ReadRepairStrategy.NONE)
expectedRepaired = 0;
- assertEquals(String.format("Expected %d repaired rows, but found %d", expectedRepaired, actualRepaired),
- expectedRepaired, actualRepaired);
+
+ // read repair metric isn't a meaningful one for mutation-tracked replication
+ if (!replicationType.isTracked())
+ {
+ assertEquals(String.format("Expected %d repaired rows, but found %d", expectedRepaired, actualRepaired),
+ expectedRepaired, actualRepaired);
+ }
return self();
}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/tracking/MutationTrackingPendingReadTest.java b/test/distributed/org/apache/cassandra/distributed/test/tracking/MutationTrackingPendingReadTest.java
index 7d859ce110..ae49c00383 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/tracking/MutationTrackingPendingReadTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/tracking/MutationTrackingPendingReadTest.java
@@ -45,7 +45,7 @@ import org.apache.cassandra.replication.MutationTrackingService;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.reads.tracked.PartialTrackedRead;
-import org.apache.cassandra.service.reads.tracked.TrackedLocalReadCoordinator;
+import org.apache.cassandra.service.reads.tracked.TrackedLocalReads;
import org.apache.cassandra.utils.FBUtilities;
import static java.lang.String.format;
@@ -150,7 +150,7 @@ public class MutationTrackingPendingReadTest
// Create another summary once initial data has been read fully. We do this to catch
// any mutations that may have arrived during initial read execution.
secondarySummary = command.createMutationSummary(true);
- TrackedLocalReadCoordinator.processDelta(read, initialSummary, secondarySummary);
+ TrackedLocalReads.processDelta(read, initialSummary, secondarySummary);
}
ColumnFamilyStore cfs = Keyspace.open(keyspaceName).getColumnFamilyStore(tableName);
diff --git a/test/distributed/org/apache/cassandra/distributed/test/tracking/MutationTrackingTest.java b/test/distributed/org/apache/cassandra/distributed/test/tracking/MutationTrackingTest.java
index 21d18c5b08..8e6a8e108f 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/tracking/MutationTrackingTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/tracking/MutationTrackingTest.java
@@ -20,6 +20,7 @@ package org.apache.cassandra.distributed.test.tracking;
import java.util.UUID;
+import org.apache.cassandra.net.Verb;
import org.apache.cassandra.replication.CoordinatorLogId;
import org.apache.cassandra.replication.MutationSummary;
import org.apache.cassandra.replication.Offsets;
@@ -97,8 +98,6 @@ public class MutationTrackingTest extends TestBaseImpl
.set("write_request_timeout", "1000ms"))
.start())
{
-
- String keyspaceName = KEYSPACE;
cluster.schemaChange(withKeyspace("CREATE KEYSPACE %s WITH replication = " +
"{'class': 'SimpleStrategy', 'replication_factor': 3} " +
"AND replication_type='tracked';"));
@@ -124,4 +123,59 @@ public class MutationTrackingTest extends TestBaseImpl
});
}
}
+
+ @Test
+ public void testFailedMutationRedelivery() throws Throwable
+ {
+ try (Cluster cluster = Cluster.build(3)
+ .withConfig(cfg -> cfg.with(Feature.NETWORK)
+ .with(Feature.GOSSIP)
+ .set("mutation_tracking_enabled", "true")
+ .set("write_request_timeout", "1000ms"))
+ .start())
+ {
+ String keyspaceName = KEYSPACE;
+
+ cluster.schemaChange(withKeyspace("CREATE KEYSPACE %s WITH replication = " +
+ "{'class': 'SimpleStrategy', 'replication_factor': 3} " +
+ "AND replication_type='tracked';"));
+
+ cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (k int primary key, v int);"));
+
+ // block writes to node 3
+ cluster.filters().verbs(Verb.MUTATION_REQ.id).to(3).drop();
+
+ // pause reconciler temporarily
+ cluster.get(1).runOnInstance(() -> MutationTrackingService.instance.pauseActiveReconciler());
+
+ // issue a write - should fail on node 3
+ cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.tbl (k, v) VALUES (1, 1)"), ConsistencyLevel.QUORUM);
+
+ Thread.sleep(1000); // wait for write timeout
+
+ cluster.get(1).runOnInstance(() ->
+ {
+ TableMetadata table = Schema.instance.getTableMetadata(keyspaceName, "tbl");
+ DecoratedKey dk = Murmur3Partitioner.instance.decorateKey(ByteBufferUtil.bytes(1));
+ MutationSummary summary = MutationTrackingService.instance.createSummaryForKey(dk, table.id, false);
+ CoordinatorLogId logId = getOnlyLogId(summary);
+ Assert.assertEquals(1, summary.get(logId).unreconciled.offsetCount());
+ Assert.assertEquals(0, summary.get(logId).reconciled.offsetCount());
+ });
+
+ // resume the reconciler
+ cluster.get(1).runOnInstance(() -> MutationTrackingService.instance.resumeActiveReconciler());
+ Thread.sleep(1000); // wait for reconiciler to do its job
+
+ cluster.get(1).runOnInstance(() ->
+ {
+ TableMetadata table = Schema.instance.getTableMetadata(keyspaceName, "tbl");
+ DecoratedKey dk = Murmur3Partitioner.instance.decorateKey(ByteBufferUtil.bytes(1));
+ MutationSummary summary = MutationTrackingService.instance.createSummaryForKey(dk, table.id, false);
+ CoordinatorLogId logId = getOnlyLogId(summary);
+ Assert.assertEquals(0, summary.get(logId).unreconciled.offsetCount());
+ Assert.assertEquals(1, summary.get(logId).reconciled.offsetCount());
+ });
+ }
+ }
}
diff --git a/test/unit/org/apache/cassandra/replication/CoordinatorLogOffsetsTest.java b/test/unit/org/apache/cassandra/replication/CoordinatorLogOffsetsTest.java
index 1d25a1956f..91a2788ddf 100644
--- a/test/unit/org/apache/cassandra/replication/CoordinatorLogOffsetsTest.java
+++ b/test/unit/org/apache/cassandra/replication/CoordinatorLogOffsetsTest.java
@@ -274,8 +274,8 @@ public class CoordinatorLogOffsetsTest
MutationTrackingService.instance.startWriting(mutation);
MutationTrackingService.instance.finishWriting(mutation);
- MutationTrackingService.instance.receivedWriteResponse(ks, mutation.key().getToken(), mutation.id(), addr2);
- MutationTrackingService.instance.receivedWriteResponse(ks, mutation.key().getToken(), mutation.id(), addr3);
+ MutationTrackingService.instance.receivedWriteResponse(mutation.id(), addr2);
+ MutationTrackingService.instance.receivedWriteResponse(mutation.id(), addr3);
ImmutableCoordinatorLogOffsets logOffsets = new ImmutableCoordinatorLogOffsets.Builder()
.add(mutation.id())
@@ -300,8 +300,8 @@ public class CoordinatorLogOffsetsTest
Mutation mutation = MutationTrackingUtils.createMutation(tableMetadata, 3, 3);
MutationTrackingService.instance.startWriting(mutation);
- MutationTrackingService.instance.receivedWriteResponse(ks, mutation.key().getToken(), mutation.id(), addr2);
- MutationTrackingService.instance.receivedWriteResponse(ks, mutation.key().getToken(), mutation.id(), addr3);
+ MutationTrackingService.instance.receivedWriteResponse(mutation.id(), addr2);
+ MutationTrackingService.instance.receivedWriteResponse(mutation.id(), addr3);
ImmutableCoordinatorLogOffsets logOffsets = new ImmutableCoordinatorLogOffsets.Builder()
.add(mutation.id())
@@ -315,8 +315,8 @@ public class CoordinatorLogOffsetsTest
MutationTrackingService.instance.startWriting(mutation);
MutationTrackingService.instance.finishWriting(mutation);
- MutationTrackingService.instance.receivedWriteResponse(ks, mutation.key().getToken(), mutation.id(), addr2);
- MutationTrackingService.instance.receivedWriteResponse(ks, mutation.key().getToken(), mutation.id(), addr3);
+ MutationTrackingService.instance.receivedWriteResponse(mutation.id(), addr2);
+ MutationTrackingService.instance.receivedWriteResponse(mutation.id(), addr3);
MutationId fakeMutationId = new MutationId(CoordinatorLogId.asLong(111, 222), MutationId.sequenceId(333, 444));
Assertions.assertThat(metadata.directory.version(new NodeId(fakeMutationId.hostId()))).isNull();
diff --git a/test/unit/org/apache/cassandra/replication/OffsetsTest.java b/test/unit/org/apache/cassandra/replication/OffsetsTest.java
index d7c4d4fcb1..5d10ae5e45 100644
--- a/test/unit/org/apache/cassandra/replication/OffsetsTest.java
+++ b/test/unit/org/apache/cassandra/replication/OffsetsTest.java
@@ -105,7 +105,7 @@ public class OffsetsTest
public TestConsumer assertOffsetsConsumed(int... expected)
{
- Assert.assertTrue(expected.length % 2 == 0);
+ assertEquals(0, expected.length % 2);
TestConsumer expectedConsumer = new TestConsumer();
for (int i = 0; i < expected.length; i+=2)
expectedConsumer.consumerOffsets(expected[i], expected[i+1]);
@@ -134,7 +134,7 @@ public class OffsetsTest
private static Offsets.Mutable offsets(int... bounds)
{
- Assert.assertTrue(bounds.length % 2 == 0);
+ assertEquals(0, bounds.length % 2);
Offsets.Mutable ids = new Offsets.Mutable(LOG_ID);
int keys = 0;
int last = 0;
@@ -506,6 +506,41 @@ public class OffsetsTest
}
}
+ @Test
+ public void testMultiMerge()
+ {
+ Supplier<Offsets.Mutable> sequenceIds = () -> {
+ Offsets.Mutable ids0 = new Offsets.Mutable(LOG_ID);
+ ids0.add(0, 3);
+ ids0.add(7, 10);
+ ids0.add(15, 17);
+
+ assertEquals(3, ids0.rangeCount());
+ assertEquals(11, ids0.offsetCount());
+ return ids0;
+ };
+
+ {
+ // extends on the end
+ Offsets.Mutable offsets = sequenceIds.get();
+ TestConsumer consumer = new TestConsumer();
+
+ assertTrue(offsets.add(0, 12, consumer));
+ consumer.assertOffsetsConsumed(4, 6, 11, 12).clear();
+ assertEquals(offsets(0, 12, 15, 17), offsets);
+ }
+
+ {
+ // extends on the start
+ Offsets.Mutable offsets = sequenceIds.get();
+ TestConsumer consumer = new TestConsumer();
+
+ assertTrue(offsets.add(5, 17, consumer));
+ consumer.assertOffsetsConsumed(5, 6, 11, 14).clear();
+ assertEquals(offsets(0, 3, 5, 17), offsets);
+ }
+ }
+
@Test
public void addTest()
{
@@ -747,4 +782,60 @@ public class OffsetsTest
Assert.assertEquals(ids(1, 2, 3), ids(offsets(1, 3)));
Assert.assertEquals(ids(1, 2, 3, 5, 6, 7), ids(offsets(1, 3, 5, 7)));
}
+
+ @Test
+ public void testRemoveFromEmpty()
+ {
+ testRemove(offsets(), 5, offsets());
+ }
+
+ @Test
+ public void testRemoveFromSingleRange()
+ {
+ // before all ranges
+ testRemove(offsets(5, 7), 4, offsets(5, 7));
+ // after all ranges
+ testRemove(offsets(5, 7), 8, offsets(5, 7));
+ // start of range
+ testRemove(offsets(5, 7), 5, offsets(6, 7));
+ // end of range
+ testRemove(offsets(5, 7), 7, offsets(5, 6));
+ // middle of range
+ testRemove(offsets(5, 7), 6, offsets(5, 5, 7, 7));
+ // single element range
+ testRemove(offsets(5, 5), 5, offsets());
+ }
+
+ @Test
+ public void testRemoveGeneric()
+ {
+ // before all ranges
+ testRemove(offsets(5, 7, 9, 9, 11, 13), 4, offsets(5, 7, 9, 9, 11, 13));
+ // after all ranges
+ testRemove(offsets(5, 7, 9, 9, 11, 13), 14, offsets(5, 7, 9, 9, 11, 13));
+ // between two ranges
+ testRemove(offsets(5, 7, 9, 9, 11, 13), 8, offsets(5, 7, 9, 9, 11, 13));
+ // start of a range
+ testRemove(offsets(5, 7, 9, 9, 11, 13), 11, offsets(5, 7, 9, 9, 12, 13));
+ // end of a range
+ testRemove(offsets(5, 7, 9, 9, 11, 13), 7, offsets(5, 6, 9, 9, 11, 13));
+ // middle of a range
+ testRemove(offsets(5, 7, 9, 9, 11, 13), 12, offsets(5, 7, 9, 9, 11, 11, 13, 13));
+ // single element range
+ testRemove(offsets(5, 7, 9, 9, 11, 13), 9, offsets(5, 7, 11, 13));
+
+ // single element ranges at different positions
+ testRemove(offsets(5, 5, 10, 12), 5, offsets(10, 12));
+ testRemove(offsets(1, 3, 5, 5, 7, 9), 5, offsets(1, 3, 7, 9));
+
+ // range splitting at boundaries
+ testRemove(offsets(5, 10), 7, offsets(5, 6, 8, 10));
+ testRemove(offsets(5, 10, 15, 17), 7, offsets(5, 6, 8, 10, 15, 17));
+ }
+
+ private void testRemove(Offsets.Mutable from, int toRemove, Offsets.Mutable expectedAfter)
+ {
+ from.remove(toRemove);
+ assertOffsetsEqual(expectedAfter, from);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org