You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by sa...@apache.org on 2023/03/14 16:45:23 UTC
[cassandra] 07/25: [CEP-21] Initial transforms for modifying ownership
This is an automated email from the ASF dual-hosted git repository.
samt pushed a commit to branch cep-21-tcm
in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit ee17642571f40891dfa7b6f058a240774a47ce0a
Author: Sam Tunnicliffe <sa...@apache.org>
AuthorDate: Thu Mar 2 11:29:07 2023 +0000
[CEP-21] Initial transforms for modifying ownership
Introduces transformations to modify ClusterMetadata with the affect of
modifying data ownership and placement i.e. join, replace, move &
decommission. These operations do not simply modify metadata however,
so simple atomic updates to cluster metadata are not sufficient.
Streaming data in and out of nodes must also occur and obviously read
and write operations must continue whilst this is in progress. These
operations then are performed in phases, planned in advance and properly
linearized using the global log. e.g. to join an new node, the full set
of phased range movements required is calculated, generating an
actionable plan which is then serialised into ClusterMetadata.
Concurrent operations are permitted as long as they only affect disjoint
token ranges, ensuring that concurrent range movements remain safe and
cluster invariants are preserved at all times, including in the case of
the failure to complete any operation (i.e. failed bootstrap).
This commit only adds the transformations and supporting TCM components
(LockedRanges, InProgressSequences etc), the implementation of actually
performing the operations follows in subsequent commits.
Co-authored-by: Marcus Eriksson <ma...@apache.org>
Co-authored-by: Alex Petrov <ol...@gmail.com>
Co-authored-by: Sam Tunnicliffe <sa...@apache.org>
---
.../config/CassandraRelevantProperties.java | 12 +
.../cassandra/locator/EndpointsForRange.java | 7 +-
.../apache/cassandra/service/StorageService.java | 8 +-
.../org/apache/cassandra/tcm/ClusterMetadata.java | 123 ++++++-
.../cassandra/tcm/ClusterMetadataService.java | 49 ++-
.../apache/cassandra/tcm/InProgressSequence.java | 27 +-
.../org/apache/cassandra/tcm/MetadataKeys.java | 8 +-
src/java/org/apache/cassandra/tcm/Startup.java | 4 +-
.../org/apache/cassandra/tcm/Transformation.java | 39 ++-
.../cassandra/tcm/compatibility/GossipHelper.java | 6 +
.../tcm/listeners/PaxosRepairListener.java | 47 +++
.../org/apache/cassandra/tcm/log/LocalLog.java | 2 +
.../apache/cassandra/tcm/membership/Directory.java | 7 +-
.../cassandra/tcm/sequences/BootstrapAndJoin.java | 360 +++++++++++++++++++
.../tcm/sequences/BootstrapAndReplace.java | 320 +++++++++++++++++
.../tcm/sequences/InProgressSequences.java | 240 +++++++++++++
.../org/apache/cassandra/tcm/sequences/Move.java | 360 +++++++++++++++++++
.../cassandra/tcm/sequences/ProgressBarrier.java | 224 ++++++++++++
.../tcm/sequences/UnbootstrapAndLeave.java | 349 +++++++++++++++++++
.../cassandra/tcm/transformations/AlterSchema.java | 20 +-
.../tcm/transformations/ApplyPlacementDeltas.java | 121 +++++++
.../transformations/CancelInProgressSequence.java | 99 ++++++
.../tcm/transformations/CustomTransformation.java | 11 +-
.../tcm/transformations/ForceSnapshot.java | 5 +-
.../cassandra/tcm/transformations/PrepareJoin.java | 307 ++++++++++++++++
.../tcm/transformations/PrepareLeave.java | 303 ++++++++++++++++
.../cassandra/tcm/transformations/PrepareMove.java | 289 +++++++++++++++
.../tcm/transformations/PrepareReplace.java | 387 +++++++++++++++++++++
.../cassandra/tcm/transformations/Register.java | 9 +-
.../cassandra/tcm/transformations/SealPeriod.java | 8 +-
.../cassandra/tcm/transformations/UnsafeJoin.java | 104 ++++++
.../tcm/transformations/cms/EntireRange.java | 3 +
.../tcm/transformations/cms/Initialize.java | 4 +-
.../tcm/transformations/cms/PreInitialize.java | 3 +-
.../cassandra/tcm/StubClusterMetadataService.java | 4 +-
.../tcm/log/LogListenerNotificationTest.java | 12 +-
.../cassandra/tcm/sequences/SequencesUtils.java | 21 ++
37 files changed, 3851 insertions(+), 51 deletions(-)
diff --git a/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java b/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
index 979fc7a247..3a219fc6e7 100644
--- a/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
+++ b/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
@@ -362,6 +362,18 @@ public enum CassandraRelevantProperties
TOMBSTONE_HISTOGRAM_TTL_ROUND_SECONDS("cassandra.streaminghistogram.roundseconds", "60"),
// transactional cluster metadata relevant properties
+
+ /**
+ * should replica groups in data placements be sorted to ensure the primary replica is first in the list
+ */
+ SORT_REPLICA_GROUPS("cassandra.sorted_replica_groups_enabled", "true"),
+
+ /**
+ * for obtaining acknowlegement from peers to make progress in multi-step operations
+ */
+ PROGRESS_BARRIER_TIMEOUT_MILLIS("cassandra.progress_barrier_timeout_ms", "3600000"),
+ PROGRESS_BARRIER_BACKOFF_MILLIS("cassandra.progress_barrier_backoff_ms", "1000"),
+
/**
* size of in-memory index of max epoch -> sealed period
*/
diff --git a/src/java/org/apache/cassandra/locator/EndpointsForRange.java b/src/java/org/apache/cassandra/locator/EndpointsForRange.java
index 7039df055f..f7aefb8dfd 100644
--- a/src/java/org/apache/cassandra/locator/EndpointsForRange.java
+++ b/src/java/org/apache/cassandra/locator/EndpointsForRange.java
@@ -71,8 +71,11 @@ public class EndpointsForRange extends Endpoints<EndpointsForRange>
{
if (newList.isEmpty()) return empty(range);
ReplicaMap<InetAddressAndPort> byEndpoint = null;
- if (this.byEndpoint != null && list.isSubList(newList))
- byEndpoint = this.byEndpoint.forSubList(newList);
+ if (this.byEndpoint != null)
+ if (list.isSubList(newList))
+ byEndpoint = this.byEndpoint.forSubList(newList);
+ else
+ byEndpoint = endpointMap(newList);
return new EndpointsForRange(range, newList, byEndpoint);
}
diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java
index 0e9c175546..6d472b7600 100644
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@ -2145,7 +2145,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
/**
* All MVs have been created during bootstrap, so mark them as built
*/
- private void markViewsAsBuilt() {
+ public void markViewsAsBuilt() {
for (String keyspace : Schema.instance.getUserKeyspaces().names())
{
for (ViewMetadata view: Schema.instance.getKeyspaceMetadata(keyspace).views)
@@ -4707,7 +4707,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
}
}
- private void repairPaxosForTopologyChange(String reason)
+ public void repairPaxosForTopologyChange(String reason)
{
if (getSkipPaxosRepairOnTopologyChange() || !Paxos.useV2())
{
@@ -5273,7 +5273,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
onFinish.run();
}
- private Future streamHints()
+ public Future streamHints()
{
return HintsService.instance.transferHints(this::getPreferredHintsStreamTarget);
}
@@ -6129,7 +6129,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
* @param rangesToStreamByKeyspace keyspaces and data ranges with endpoints included for each
* @return async Future for whether stream was success
*/
- private Future<StreamState> streamRanges(Map<String, EndpointsByReplica> rangesToStreamByKeyspace)
+ public Future<StreamState> streamRanges(Map<String, EndpointsByReplica> rangesToStreamByKeyspace)
{
// First, we build a list of ranges to stream to each host, per table
Map<String, RangesByEndpoint> sessionsToStreamByKeyspace = new HashMap<>();
diff --git a/src/java/org/apache/cassandra/tcm/ClusterMetadata.java b/src/java/org/apache/cassandra/tcm/ClusterMetadata.java
index bbccd3af2b..a89f3c6dd9 100644
--- a/src/java/org/apache/cassandra/tcm/ClusterMetadata.java
+++ b/src/java/org/apache/cassandra/tcm/ClusterMetadata.java
@@ -31,6 +31,7 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
+import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Token;
@@ -47,7 +48,11 @@ import org.apache.cassandra.tcm.membership.NodeAddresses;
import org.apache.cassandra.tcm.membership.NodeId;
import org.apache.cassandra.tcm.membership.NodeState;
import org.apache.cassandra.tcm.membership.NodeVersion;
+import org.apache.cassandra.tcm.ownership.DataPlacements;
+import org.apache.cassandra.tcm.ownership.PrimaryRangeComparator;
import org.apache.cassandra.tcm.ownership.TokenMap;
+import org.apache.cassandra.tcm.sequences.InProgressSequences;
+import org.apache.cassandra.tcm.sequences.LockedRanges;
import org.apache.cassandra.tcm.serialization.MetadataSerializer;
import org.apache.cassandra.tcm.serialization.Version;
import org.apache.cassandra.tcm.transformations.cms.EntireRange;
@@ -70,6 +75,9 @@ public class ClusterMetadata
public final DistributedSchema schema;
public final Directory directory;
public final TokenMap tokenMap;
+ public final DataPlacements placements;
+ public final LockedRanges lockedRanges;
+ public final InProgressSequences inProgressSequences;
public final EndpointsForRange cmsReplicas;
public final ImmutableSet<InetAddressAndPort> cmsMembers;
@@ -93,6 +101,9 @@ public class ClusterMetadata
schema,
directory,
new TokenMap(partitioner),
+ DataPlacements.EMPTY,
+ LockedRanges.EMPTY,
+ InProgressSequences.EMPTY,
ImmutableSet.of(),
ImmutableMap.of());
}
@@ -104,6 +115,9 @@ public class ClusterMetadata
DistributedSchema schema,
Directory directory,
TokenMap tokenMap,
+ DataPlacements placements,
+ LockedRanges lockedRanges,
+ InProgressSequences inProgressSequences,
Set<InetAddressAndPort> cmsMembers,
Map<ExtensionKey<?, ?>, ExtensionValue<?>> extensions)
{
@@ -118,6 +132,9 @@ public class ClusterMetadata
this.schema = schema;
this.directory = directory;
this.tokenMap = tokenMap;
+ this.placements = placements;
+ this.lockedRanges = lockedRanges;
+ this.inProgressSequences = inProgressSequences;
this.cmsMembers = ImmutableSet.copyOf(cmsMembers);
this.extensions = ImmutableMap.copyOf(extensions);
@@ -157,6 +174,9 @@ public class ClusterMetadata
schema,
directory,
tokenMap,
+ placements,
+ lockedRanges,
+ inProgressSequences,
cmsMembers,
extensions);
}
@@ -181,6 +201,9 @@ public class ClusterMetadata
private DistributedSchema schema;
private Directory directory;
private TokenMap tokenMap;
+ private DataPlacements placements;
+ private LockedRanges lockedRanges;
+ private InProgressSequences inProgressSequences;
private final Set<InetAddressAndPort> cmsMembers;
private final Map<ExtensionKey<?, ?>, ExtensionValue<?>> extensions;
private final Set<MetadataKey> modifiedKeys;
@@ -195,6 +218,9 @@ public class ClusterMetadata
this.schema = metadata.schema;
this.directory = metadata.directory;
this.tokenMap = metadata.tokenMap;
+ this.placements = metadata.placements;
+ this.lockedRanges = metadata.lockedRanges;
+ this.inProgressSequences = metadata.inProgressSequences;
this.cmsMembers = new HashSet<>(metadata.cmsMembers);
extensions = new HashMap<>(metadata.extensions);
modifiedKeys = new HashSet<>();
@@ -228,6 +254,7 @@ public class ClusterMetadata
public Transformer unproposeTokens(NodeId nodeId)
{
tokenMap = tokenMap.unassignTokens(nodeId);
+ directory = directory.withoutRackAndDC(nodeId);
return this;
}
@@ -237,6 +264,53 @@ public class ClusterMetadata
return this;
}
+ public Transformer join(NodeId nodeId)
+ {
+ directory = directory.withNodeState(nodeId, NodeState.JOINED);
+ return this;
+ }
+
+ public Transformer replaced(NodeId replaced, NodeId replacement)
+ {
+ Collection<Token> transferringTokens = tokenMap.tokens(replaced);
+ tokenMap = tokenMap.unassignTokens(replaced)
+ .assignTokens(replacement, transferringTokens);
+ directory = directory.without(replaced)
+ .withNodeState(replacement, NodeState.JOINED);
+ return this;
+ }
+
+ public Transformer proposeRemoveNode(NodeId id)
+ {
+ tokenMap = tokenMap.unassignTokens(id);
+ return this;
+ }
+
+ public Transformer left(NodeId id)
+ {
+ tokenMap = tokenMap.unassignTokens(id);
+ directory = directory.withNodeState(id, NodeState.LEFT)
+ .withoutRackAndDC(id);
+ return this;
+ }
+
+ public Transformer with(DataPlacements placements)
+ {
+ this.placements = placements;
+ return this;
+ }
+
+ public Transformer with(LockedRanges lockedRanges)
+ {
+ this.lockedRanges = lockedRanges;
+ return this;
+ }
+
+ public Transformer with(InProgressSequences sequences)
+ {
+ this.inProgressSequences = sequences;
+ return this;
+ }
public Transformer withCMSMember(InetAddressAndPort member)
{
cmsMembers.add(member);
@@ -311,6 +385,30 @@ public class ClusterMetadata
tokenMap = tokenMap.withLastModified(epoch);
}
+ if (placements != base.placements)
+ {
+ modifiedKeys.add(MetadataKeys.DATA_PLACEMENTS);
+ // sort all endpoint lists to preserve primary replica
+ if (CassandraRelevantProperties.SORT_REPLICA_GROUPS.getBoolean())
+ {
+ PrimaryRangeComparator comparator = new PrimaryRangeComparator(tokenMap, directory);
+ placements = DataPlacements.sortReplicaGroups(placements, comparator);
+ }
+ placements = placements.withLastModified(epoch);
+ }
+
+ if (lockedRanges != base.lockedRanges)
+ {
+ modifiedKeys.add(MetadataKeys.LOCKED_RANGES);
+ lockedRanges = lockedRanges.withLastModified(epoch);
+ }
+
+ if (inProgressSequences != base.inProgressSequences)
+ {
+ modifiedKeys.add(MetadataKeys.IN_PROGRESS_SEQUENCES);
+ inProgressSequences = inProgressSequences.withLastModified(epoch);
+ }
+
return new Transformed(new ClusterMetadata(epoch,
period,
lastInPeriod,
@@ -318,6 +416,9 @@ public class ClusterMetadata
schema,
directory,
tokenMap,
+ placements,
+ lockedRanges,
+ inProgressSequences,
cmsMembers,
extensions),
ImmutableSet.copyOf(modifiedKeys));
@@ -334,6 +435,9 @@ public class ClusterMetadata
", schema=" + schema +
", directory=" + schema +
", tokenMap=" + tokenMap +
+ ", placement=" + placements +
+ ", lockedRanges=" + lockedRanges +
+ ", inProgressSequences=" + inProgressSequences +
", extensions=" + extensions +
", cmsMembers=" + cmsMembers +
", modifiedKeys=" + modifiedKeys +
@@ -372,13 +476,16 @@ public class ClusterMetadata
schema.equals(that.schema) &&
directory.equals(that.directory) &&
tokenMap.equals(that.tokenMap) &&
+ placements.equals(that.placements) &&
+ lockedRanges.equals(that.lockedRanges) &&
+ inProgressSequences.equals(that.inProgressSequences) &&
extensions.equals(that.extensions);
}
@Override
public int hashCode()
{
- return Objects.hash(epoch, lastInPeriod, schema, directory, tokenMap, extensions);
+ return Objects.hash(epoch, lastInPeriod, schema, directory, tokenMap, placements, lockedRanges, inProgressSequences, extensions);
}
public static ClusterMetadata current()
@@ -423,6 +530,9 @@ public class ClusterMetadata
DistributedSchema.serializer.serialize(metadata.schema, out, version);
Directory.serializer.serialize(metadata.directory, out, version);
TokenMap.serializer.serialize(metadata.tokenMap, out, version);
+ DataPlacements.serializer.serialize(metadata.placements, out, version);
+ LockedRanges.serializer.serialize(metadata.lockedRanges, out, version);
+ InProgressSequences.serializer.serialize(metadata.inProgressSequences, out, version);
out.writeInt(metadata.extensions.size());
for (Map.Entry<ExtensionKey<?, ?>, ExtensionValue<?>> entry : metadata.extensions.entrySet())
{
@@ -447,6 +557,9 @@ public class ClusterMetadata
DistributedSchema schema = DistributedSchema.serializer.deserialize(in, version);
Directory dir = Directory.serializer.deserialize(in, version);
TokenMap tokenMap = TokenMap.serializer.deserialize(in, version);
+ DataPlacements placements = DataPlacements.serializer.deserialize(in, version);
+ LockedRanges lockedRanges = LockedRanges.serializer.deserialize(in, version);
+ InProgressSequences ips = InProgressSequences.serializer.deserialize(in, version);
int items = in.readInt();
Map<ExtensionKey<?, ?>, ExtensionValue<?>> extensions = new HashMap<>(items);
for (int i = 0; i < items; i++)
@@ -467,6 +580,9 @@ public class ClusterMetadata
schema,
dir,
tokenMap,
+ placements,
+ lockedRanges,
+ ips,
members,
extensions);
}
@@ -485,7 +601,10 @@ public class ClusterMetadata
sizeof(metadata.partitioner.getClass().getCanonicalName()) +
DistributedSchema.serializer.serializedSize(metadata.schema, version) +
Directory.serializer.serializedSize(metadata.directory, version) +
- TokenMap.serializer.serializedSize(metadata.tokenMap, version);
+ TokenMap.serializer.serializedSize(metadata.tokenMap, version) +
+ DataPlacements.serializer.serializedSize(metadata.placements, version) +
+ LockedRanges.serializer.serializedSize(metadata.lockedRanges, version) +
+ InProgressSequences.serializer.serializedSize(metadata.inProgressSequences, version);
size += TypeSizes.INT_SIZE;
for (InetAddressAndPort member : metadata.cmsMembers)
diff --git a/src/java/org/apache/cassandra/tcm/ClusterMetadataService.java b/src/java/org/apache/cassandra/tcm/ClusterMetadataService.java
index 4a7464c3a3..b5e4c1fb95 100644
--- a/src/java/org/apache/cassandra/tcm/ClusterMetadataService.java
+++ b/src/java/org/apache/cassandra/tcm/ClusterMetadataService.java
@@ -40,6 +40,8 @@ import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.tcm.log.Entry;
import org.apache.cassandra.tcm.log.LocalLog;
import org.apache.cassandra.tcm.log.Replication;
+import org.apache.cassandra.tcm.ownership.PlacementProvider;
+import org.apache.cassandra.tcm.ownership.UniformRangePlacement;
import org.apache.cassandra.tcm.transformations.SealPeriod;
import org.apache.cassandra.utils.FBUtilities;
@@ -94,6 +96,7 @@ public class ClusterMetadataService
ClusterMetadata replayAndWait();
}
+ private final PlacementProvider placementProvider;
private final Processor processor;
private final LocalLog log;
private final Commit.Replicator replicator;
@@ -120,10 +123,12 @@ public class ClusterMetadataService
return State.REMOTE;
}
- ClusterMetadataService(ClusterMetadata initial,
+ ClusterMetadataService(PlacementProvider placementProvider,
+ ClusterMetadata initial,
Function<Processor, Processor> wrapProcessor,
Supplier<State> cmsStateSupplier)
{
+ this.placementProvider = placementProvider;
this.snapshots = new MetadataSnapshots.SystemKeyspaceMetadataSnapshots();
log = LocalLog.async(initial);
@@ -142,12 +147,14 @@ public class ClusterMetadataService
}
@VisibleForTesting
- public ClusterMetadataService(MetadataSnapshots snapshots,
+ public ClusterMetadataService(PlacementProvider placementProvider,
+ MetadataSnapshots snapshots,
LocalLog log,
Processor processor,
Commit.Replicator replicator,
boolean isMemberOfOwnershipGroup)
{
+ this.placementProvider = placementProvider;
this.log = log;
this.processor = processor;
this.replicator = replicator;
@@ -161,6 +168,29 @@ public class ClusterMetadataService
commitRequestHandler = isMemberOfOwnershipGroup ? new Commit.Handler(processor, replicator) : null;
}
+ private ClusterMetadataService(PlacementProvider placementProvider,
+ MetadataSnapshots snapshots,
+ LocalLog log,
+ Processor processor,
+ Commit.Replicator replicator,
+ Replication.ReplicationHandler replicationHandler,
+ Replication.LogNotifyHandler logNotifyHandler,
+ CurrentEpochRequestHandler currentEpochHandler,
+ Replay.Handler replayRequestHandler,
+ Commit.Handler commitRequestHandler)
+ {
+ this.placementProvider = placementProvider;
+ this.snapshots = snapshots;
+ this.log = log;
+ this.processor = processor;
+ this.replicator = replicator;
+ this.replicationHandler = replicationHandler;
+ this.logNotifyHandler = logNotifyHandler;
+ this.currentEpochHandler = currentEpochHandler;
+ this.replayRequestHandler = replayRequestHandler;
+ this.commitRequestHandler = commitRequestHandler;
+ }
+
@SuppressWarnings("resource")
public static void initializeForTools(boolean loadSSTables)
{
@@ -170,16 +200,20 @@ public class ClusterMetadataService
emptyFromSystemTables.schema.initializeKeyspaceInstances(DistributedSchema.empty(), loadSSTables);
emptyFromSystemTables = emptyFromSystemTables.forceEpoch(Epoch.EMPTY);
LocalLog log = LocalLog.sync(emptyFromSystemTables, new AtomicLongBackedProcessor.InMemoryStorage(), true);
- ClusterMetadataService cms = new ClusterMetadataService(MetadataSnapshots.NO_OP,
+ ClusterMetadataService cms = new ClusterMetadataService(new UniformRangePlacement(),
+ MetadataSnapshots.NO_OP,
log,
new AtomicLongBackedProcessor(log),
Commit.Replicator.NO_OP,
- false);
+ new Replication.ReplicationHandler(log),
+ new Replication.LogNotifyHandler(log),
+ new CurrentEpochRequestHandler(),
+ null,
+ null);
log.bootstrap(FBUtilities.getBroadcastAddressAndPort());
ClusterMetadataService.setInstance(cms);
}
-
public boolean isCurrentMember(InetAddressAndPort peer)
{
return ClusterMetadata.current().isCMSMember(peer);
@@ -291,6 +325,11 @@ public class ClusterMetadataService
return ClusterMetadataService.instance().currentEpochHandler;
}
+ public PlacementProvider placementProvider()
+ {
+ return this.placementProvider;
+ }
+
@VisibleForTesting
public Processor processor()
{
diff --git a/test/unit/org/apache/cassandra/tcm/sequences/SequencesUtils.java b/src/java/org/apache/cassandra/tcm/InProgressSequence.java
similarity index 51%
copy from test/unit/org/apache/cassandra/tcm/sequences/SequencesUtils.java
copy to src/java/org/apache/cassandra/tcm/InProgressSequence.java
index 1d7e08fd7c..2b006016c3 100644
--- a/test/unit/org/apache/cassandra/tcm/sequences/SequencesUtils.java
+++ b/src/java/org/apache/cassandra/tcm/InProgressSequence.java
@@ -16,16 +16,29 @@
* limitations under the License.
*/
-package org.apache.cassandra.tcm.sequences;
+package org.apache.cassandra.tcm;
-import java.util.Random;
+import org.apache.cassandra.tcm.sequences.InProgressSequences;
+import org.apache.cassandra.tcm.sequences.ProgressBarrier;
-import org.apache.cassandra.tcm.Epoch;
-
-public class SequencesUtils
+public interface InProgressSequence<T extends InProgressSequence<T>>
{
- public static Epoch epoch(Random random)
+ public InProgressSequences.Kind kind();
+
+ public ProgressBarrier barrier();
+
+ /**
+ * Returns a kind of the next step
+ */
+ public Transformation.Kind nextStep();
+ public boolean executeNext();
+
+ public T advance(Epoch waitForWatermark, Transformation.Kind next);
+
+ // TODO rename this. It really provides the result of undoing any steps in the sequence
+ // which have already been executed
+ default ClusterMetadata.Transformer cancel(ClusterMetadata metadata)
{
- return Epoch.create(random.nextLong());
+ throw new UnsupportedOperationException();
}
}
diff --git a/src/java/org/apache/cassandra/tcm/MetadataKeys.java b/src/java/org/apache/cassandra/tcm/MetadataKeys.java
index d1bde221cc..fda509186b 100644
--- a/src/java/org/apache/cassandra/tcm/MetadataKeys.java
+++ b/src/java/org/apache/cassandra/tcm/MetadataKeys.java
@@ -29,10 +29,16 @@ public class MetadataKeys
public static final MetadataKey SCHEMA = make(CORE_NS, "schema", "dist_schema");
public static final MetadataKey NODE_DIRECTORY = make(CORE_NS, "membership", "node_directory");
public static final MetadataKey TOKEN_MAP = make(CORE_NS, "ownership", "token_map");
+ public static final MetadataKey DATA_PLACEMENTS = make(CORE_NS, "ownership", "data_placements");
+ public static final MetadataKey LOCKED_RANGES = make(CORE_NS, "sequences", "locked_ranges");
+ public static final MetadataKey IN_PROGRESS_SEQUENCES = make(CORE_NS, "sequences", "in_progress");
public static final ImmutableSet<MetadataKey> CORE_METADATA = ImmutableSet.of(SCHEMA,
NODE_DIRECTORY,
- TOKEN_MAP);
+ TOKEN_MAP,
+ DATA_PLACEMENTS,
+ LOCKED_RANGES,
+ IN_PROGRESS_SEQUENCES);
public static MetadataKey make(String...parts)
{
diff --git a/src/java/org/apache/cassandra/tcm/Startup.java b/src/java/org/apache/cassandra/tcm/Startup.java
index f3e6e5c4be..6d5c8d037b 100644
--- a/src/java/org/apache/cassandra/tcm/Startup.java
+++ b/src/java/org/apache/cassandra/tcm/Startup.java
@@ -29,6 +29,7 @@ package org.apache.cassandra.tcm;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.tcm.log.SystemKeyspaceStorage;
+ import org.apache.cassandra.tcm.ownership.UniformRangePlacement;
import org.apache.cassandra.tcm.transformations.cms.Initialize;
import org.apache.cassandra.utils.FBUtilities;
@@ -86,7 +87,8 @@ package org.apache.cassandra.tcm;
public static void initializeAsNonCmsNode(Function<ClusterMetadataService.Processor, ClusterMetadataService.Processor> wrapProcessor)
{
ClusterMetadata initial = new ClusterMetadata(DatabaseDescriptor.getPartitioner());
- ClusterMetadataService.setInstance(new ClusterMetadataService(initial,
+ ClusterMetadataService.setInstance(new ClusterMetadataService(new UniformRangePlacement(),
+ initial,
wrapProcessor,
ClusterMetadataService::state));
ClusterMetadataService.instance().log().replayPersisted();
diff --git a/src/java/org/apache/cassandra/tcm/Transformation.java b/src/java/org/apache/cassandra/tcm/Transformation.java
index 8f7afe0ebd..b81a793ee1 100644
--- a/src/java/org/apache/cassandra/tcm/Transformation.java
+++ b/src/java/org/apache/cassandra/tcm/Transformation.java
@@ -30,14 +30,21 @@ import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.tcm.sequences.LockedRanges;
import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
import org.apache.cassandra.tcm.serialization.VerboseMetadataSerializer;
import org.apache.cassandra.tcm.serialization.Version;
import org.apache.cassandra.tcm.transformations.AlterSchema;
+import org.apache.cassandra.tcm.transformations.CancelInProgressSequence;
import org.apache.cassandra.tcm.transformations.CustomTransformation;
import org.apache.cassandra.tcm.transformations.ForceSnapshot;
+import org.apache.cassandra.tcm.transformations.PrepareJoin;
+import org.apache.cassandra.tcm.transformations.PrepareLeave;
+import org.apache.cassandra.tcm.transformations.PrepareMove;
+import org.apache.cassandra.tcm.transformations.PrepareReplace;
import org.apache.cassandra.tcm.transformations.Register;
import org.apache.cassandra.tcm.transformations.SealPeriod;
+import org.apache.cassandra.tcm.transformations.UnsafeJoin;
import org.apache.cassandra.tcm.transformations.cms.Initialize;
import org.apache.cassandra.tcm.transformations.cms.PreInitialize;
@@ -49,10 +56,10 @@ public interface Transformation
Result execute(ClusterMetadata metadata);
- default Success success(ClusterMetadata.Transformer transformer)
+ default Success success(ClusterMetadata.Transformer transformer, LockedRanges.AffectedRanges affectedRanges)
{
ClusterMetadata.Transformer.Transformed transformed = transformer.build();
- return new Success(transformed.metadata, transformed.modifiedKeys);
+ return new Success(transformed.metadata, affectedRanges, transformed.modifiedKeys);
}
interface Result
@@ -67,11 +74,13 @@ public interface Transformation
final class Success implements Result
{
public final ClusterMetadata metadata;
+ public final LockedRanges.AffectedRanges affectedRanges;
public final ImmutableSet<MetadataKey> affectedMetadata;
- public Success(ClusterMetadata metadata, ImmutableSet<MetadataKey> affectedKeys)
+ public Success(ClusterMetadata metadata, LockedRanges.AffectedRanges affectedRanges, ImmutableSet<MetadataKey> affectedKeys)
{
this.metadata = metadata;
+ this.affectedRanges = affectedRanges;
this.affectedMetadata = affectedKeys;
}
@@ -141,6 +150,30 @@ public interface Transformation
SEAL_PERIOD(() -> SealPeriod.serializer),
SCHEMA_CHANGE(() -> AlterSchema.serializer),
REGISTER(() -> Register.serializer),
+
+ UNSAFE_JOIN(() -> UnsafeJoin.serializer),
+ PREPARE_JOIN(() -> PrepareJoin.serializer),
+ START_JOIN(() -> PrepareJoin.StartJoin.serializer),
+ MID_JOIN(() -> PrepareJoin.MidJoin.serializer),
+ FINISH_JOIN(() -> PrepareJoin.FinishJoin.serializer),
+
+ PREPARE_MOVE(() -> PrepareMove.serializer),
+ START_MOVE(() -> PrepareMove.StartMove.serializer),
+ MID_MOVE(() -> PrepareMove.MidMove.serializer),
+ FINISH_MOVE(() -> PrepareMove.FinishMove.serializer),
+
+ PREPARE_LEAVE(() -> PrepareLeave.serializer),
+ START_LEAVE(() -> PrepareLeave.StartLeave.serializer),
+ MID_LEAVE(() -> PrepareLeave.MidLeave.serializer),
+ FINISH_LEAVE(() -> PrepareLeave.FinishLeave.serializer),
+
+ PREPARE_REPLACE(() -> PrepareReplace.serializer),
+ START_REPLACE(() -> PrepareReplace.StartReplace.serializer),
+ MID_REPLACE(() -> PrepareReplace.MidReplace.serializer),
+ FINISH_REPLACE(() -> PrepareReplace.FinishReplace.serializer),
+
+ CANCEL_SEQUENCE(() -> CancelInProgressSequence.serializer),
+
CUSTOM(() -> CustomTransformation.serializer);
private final Supplier<AsymmetricMetadataSerializer<Transformation, ? extends Transformation>> serializer;
diff --git a/src/java/org/apache/cassandra/tcm/compatibility/GossipHelper.java b/src/java/org/apache/cassandra/tcm/compatibility/GossipHelper.java
index 477a1ea2d8..7ff6b87c59 100644
--- a/src/java/org/apache/cassandra/tcm/compatibility/GossipHelper.java
+++ b/src/java/org/apache/cassandra/tcm/compatibility/GossipHelper.java
@@ -31,7 +31,10 @@ import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.Epoch;
import org.apache.cassandra.tcm.Period;
import org.apache.cassandra.tcm.membership.Directory;
+import org.apache.cassandra.tcm.ownership.DataPlacements;
import org.apache.cassandra.tcm.ownership.TokenMap;
+import org.apache.cassandra.tcm.sequences.InProgressSequences;
+import org.apache.cassandra.tcm.sequences.LockedRanges;
public class GossipHelper
{
@@ -50,6 +53,9 @@ public class GossipHelper
DistributedSchema.fromSystemTables(SchemaKeyspace.fetchNonSystemKeyspaces()),
Directory.EMPTY,
new TokenMap(DatabaseDescriptor.getPartitioner()),
+ DataPlacements.empty(),
+ LockedRanges.EMPTY,
+ InProgressSequences.EMPTY,
Collections.emptySet(),
Collections.emptyMap());
}
diff --git a/src/java/org/apache/cassandra/tcm/listeners/PaxosRepairListener.java b/src/java/org/apache/cassandra/tcm/listeners/PaxosRepairListener.java
new file mode 100644
index 0000000000..43c756efb5
--- /dev/null
+++ b/src/java/org/apache/cassandra/tcm/listeners/PaxosRepairListener.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tcm.listeners;
+
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.membership.NodeId;
+import org.apache.cassandra.tcm.membership.NodeState;
+
+import static org.apache.cassandra.tcm.membership.NodeState.LEFT;
+
+public class PaxosRepairListener implements ChangeListener
+{
+ @Override
+ public void notifyPostCommit(ClusterMetadata prev, ClusterMetadata next)
+ {
+ if (!next.directory.lastModified().equals(prev.directory.lastModified()))
+ {
+ for (NodeId node : next.directory.peerIds())
+ {
+ NodeState oldState = prev.directory.peerState(node);
+ NodeState newState = next.directory.peerState(node);
+ if (oldState != newState && newState == LEFT)
+ {
+ StorageService.instance.repairPaxosForTopologyChange("decommission");
+ break;
+ }
+ }
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/tcm/log/LocalLog.java b/src/java/org/apache/cassandra/tcm/log/LocalLog.java
index d6165386ba..469cc8a0e4 100644
--- a/src/java/org/apache/cassandra/tcm/log/LocalLog.java
+++ b/src/java/org/apache/cassandra/tcm/log/LocalLog.java
@@ -49,6 +49,7 @@ import org.apache.cassandra.tcm.listeners.InitializationListener;
import org.apache.cassandra.tcm.listeners.LogListener;
import org.apache.cassandra.tcm.listeners.MetadataSnapshotListener;
import org.apache.cassandra.tcm.listeners.SchemaListener;
+import org.apache.cassandra.tcm.listeners.PaxosRepairListener;
import org.apache.cassandra.tcm.transformations.ForceSnapshot;
import org.apache.cassandra.tcm.transformations.cms.PreInitialize;
import org.apache.cassandra.utils.Closeable;
@@ -545,6 +546,7 @@ public abstract class LocalLog implements Closeable
addListener(new InitializationListener());
addListener(new SchemaListener());
addListener(new MetadataSnapshotListener());
+ addListener(new PaxosRepairListener());
}
private LogListener snapshotListener()
diff --git a/src/java/org/apache/cassandra/tcm/membership/Directory.java b/src/java/org/apache/cassandra/tcm/membership/Directory.java
index 842c78f95c..94a67c2864 100644
--- a/src/java/org/apache/cassandra/tcm/membership/Directory.java
+++ b/src/java/org/apache/cassandra/tcm/membership/Directory.java
@@ -213,9 +213,14 @@ public class Directory implements MetadataValue<Directory>, AsEndpoints, AsLocat
Location location = locations.get(id);
BTreeMultimap<String, InetAddressAndPort> rackEP = (BTreeMultimap<String, InetAddressAndPort>) racksByDC.get(location.datacenter);
rackEP = rackEP.without(location.rack, endpoint);
+ BTreeMap<String, Multimap<String, InetAddressAndPort>> newRacksByDC;
+ if (rackEP.isEmpty())
+ newRacksByDC = racksByDC.without(location.datacenter);
+ else
+ newRacksByDC = racksByDC.withForce(location.datacenter, rackEP);
return new Directory(nextId, lastModified, peers, locations, states, versions, hostIds, addresses,
endpointsByDC.without(location.datacenter, endpoint),
- racksByDC.withForce(location.datacenter, rackEP));
+ newRacksByDC);
}
public Directory without(NodeId id)
diff --git a/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndJoin.java b/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndJoin.java
new file mode 100644
index 0000000000..f97be0228c
--- /dev/null
+++ b/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndJoin.java
@@ -0,0 +1,360 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tcm.sequences;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Objects;
+import java.util.stream.StreamSupport;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.streaming.StreamState;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.ClusterMetadataService;
+import org.apache.cassandra.tcm.Epoch;
+import org.apache.cassandra.tcm.InProgressSequence;
+import org.apache.cassandra.tcm.Transformation;
+import org.apache.cassandra.tcm.membership.NodeId;
+import org.apache.cassandra.tcm.membership.NodeState;
+import org.apache.cassandra.tcm.ownership.DataPlacements;
+import org.apache.cassandra.tcm.ownership.PlacementDeltas;
+import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
+import org.apache.cassandra.tcm.serialization.Version;
+import org.apache.cassandra.tcm.transformations.PrepareJoin;
+import org.apache.cassandra.utils.concurrent.Future;
+import org.apache.cassandra.utils.concurrent.ImmediateFuture;
+import org.apache.cassandra.utils.vint.VIntCoding;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+
+public class BootstrapAndJoin implements InProgressSequence<BootstrapAndJoin>
+{
+ private static final Logger logger = LoggerFactory.getLogger(BootstrapAndJoin.class);
+ public static final Serializer serializer = new Serializer();
+
+ public final ProgressBarrier barrier;
+ public final LockedRanges.Key lockKey;
+ public final PlacementDeltas toSplitRanges;
+ public final PrepareJoin.StartJoin startJoin;
+ public final PrepareJoin.MidJoin midJoin;
+ public final PrepareJoin.FinishJoin finishJoin;
+ public final Transformation.Kind next;
+
+ public final boolean finishJoiningRing;
+ public final boolean streamData;
+
+ public BootstrapAndJoin(ProgressBarrier barrier,
+ LockedRanges.Key lockKey,
+ Transformation.Kind next,
+ PlacementDeltas toSplitRanges,
+ PrepareJoin.StartJoin startJoin,
+ PrepareJoin.MidJoin midJoin,
+ PrepareJoin.FinishJoin finishJoin,
+ boolean finishJoiningRing,
+ boolean streamData)
+ {
+ this.barrier = barrier;
+ this.lockKey = lockKey;
+ this.next = next;
+
+ this.toSplitRanges = toSplitRanges;
+ this.startJoin = startJoin;
+ this.midJoin = midJoin;
+ this.finishJoin = finishJoin;
+
+ this.finishJoiningRing = finishJoiningRing;
+ this.streamData = streamData;
+ }
+
+ // TODO this is reused by BootstrapAndReplace, should we move it somewhere common?
+ public static boolean bootstrap(final Collection<Token> tokens,
+ long bootstrapTimeoutMillis,
+ ClusterMetadata metadata,
+ InetAddressAndPort replacingEndpoint){
+ SystemKeyspace.updateTokens(tokens); // DON'T use setToken, that makes us part of the ring locally which is incorrect until we are done bootstrapping
+
+ if (Boolean.getBoolean("cassandra.reset_bootstrap_progress"))
+ {
+ logger.info("Resetting bootstrap progress to start fresh");
+ SystemKeyspace.resetAvailableStreamedRanges();
+ }
+
+// Future<StreamState> bootstrapStream = StorageService.instance.startBootstrap(tokens, metadata, replacingEndpoint);
+ Future<StreamState> bootstrapStream = ImmediateFuture.failure(new UnsupportedOperationException("Not implemented"));
+ try
+ {
+ if (bootstrapTimeoutMillis > 0)
+ bootstrapStream.get(bootstrapTimeoutMillis, MILLISECONDS);
+ else
+ bootstrapStream.get();
+ StorageService.instance.markViewsAsBuilt();
+ logger.info("Bootstrap completed for tokens {}", tokens);
+ return true;
+ }
+ catch (Throwable e)
+ {
+ logger.error("Error while waiting on bootstrap to complete. Bootstrap will have to be restarted.", e);
+ return false;
+ }
+ }
+
+ @Override
+ public ProgressBarrier barrier()
+ {
+ return barrier;
+ }
+
+ @Override
+ public Transformation.Kind nextStep()
+ {
+ return next;
+ }
+
+ @Override
+ public BootstrapAndJoin advance(Epoch waitForWatermark, Transformation.Kind next)
+ {
+ return new BootstrapAndJoin(barrier().withNewEpoch(waitForWatermark), lockKey, next,
+ toSplitRanges, startJoin, midJoin, finishJoin,
+ finishJoiningRing, streamData);
+ }
+
+
+ @Override
+ public InProgressSequences.Kind kind()
+ {
+ return InProgressSequences.Kind.JOIN;
+ }
+
+ @Override
+ public boolean executeNext()
+ {
+ switch (next)
+ {
+ case START_JOIN:
+ try
+ {
+ NodeId nodeId = ClusterMetadata.current().myNodeId();
+ SystemKeyspace.updateTokens(ClusterMetadata.current().tokenMap.tokens(nodeId));
+
+ ClusterMetadataService.instance().commit(startJoin);
+ }
+ catch (Throwable e)
+ {
+ return true;
+ }
+
+ break;
+ case MID_JOIN:
+ try
+ {
+ Collection<Token> bootstrapTokens = SystemKeyspace.getSavedTokens();
+
+ if (streamData)
+ {
+ boolean dataAvailable = bootstrap(bootstrapTokens,
+ StorageService.INDEFINITE,
+ ClusterMetadata.current(),
+ null);
+
+ if (!dataAvailable)
+ {
+ logger.warn("Some data streaming failed. Use nodetool to check bootstrap state and resume. " +
+ "For more, see `nodetool help bootstrap`. {}", SystemKeyspace.getBootstrapState());
+ return false;
+ }
+ }
+
+ SystemKeyspace.setBootstrapState(SystemKeyspace.BootstrapState.COMPLETED);
+ StreamSupport.stream(ColumnFamilyStore.all().spliterator(), false)
+ .filter(cfs -> Schema.instance.getUserKeyspaces().names().contains(cfs.keyspace.getName()))
+ .forEach(cfs -> cfs.indexManager.executePreJoinTasksBlocking(true));
+
+ ClusterMetadataService.instance().commit(midJoin);
+ }
+ catch (Throwable e)
+ {
+ return true;
+ }
+
+ break;
+ case FINISH_JOIN:
+ try
+ {
+ if (!finishJoiningRing)
+ {
+ logger.info("Startup complete, but write survey mode is active, not becoming an active ring member. Use JMX (StorageService->joinRing()) to finalize ring joining.");
+ return false;
+ }
+
+ ClusterMetadataService.instance().commit(finishJoin);
+ }
+ catch (Throwable e)
+ {
+ return true;
+ }
+ break;
+ default:
+ throw new IllegalStateException("Can't proceed with join from " + next);
+ }
+ return true;
+ }
+
+ @Override
+ public ClusterMetadata.Transformer cancel(ClusterMetadata metadata)
+ {
+ DataPlacements placements = metadata.placements;
+ switch (next)
+ {
+ // need to undo MID_JOIN and START_JOIN, then merge the ranges split by PrepareJoin
+ case FINISH_JOIN:
+ placements = midJoin.inverseDelta().apply(placements);
+ case MID_JOIN:
+ placements = startJoin.inverseDelta().apply(placements);
+ case START_JOIN:
+ placements = toSplitRanges.invert().apply(placements);
+ break;
+ default:
+ throw new IllegalStateException("Can't revert join from " + next);
+ }
+ LockedRanges newLockedRanges = metadata.lockedRanges.unlock(lockKey);
+ return metadata.transformer()
+ .unproposeTokens(startJoin.nodeId())
+ .withNodeState(startJoin.nodeId(), NodeState.REGISTERED)
+ .with(placements)
+ .with(newLockedRanges);
+ }
+
+ public BootstrapAndJoin finishJoiningRing()
+ {
+ return new BootstrapAndJoin(barrier(), lockKey, next, toSplitRanges, startJoin, midJoin, finishJoin,
+ true, streamData);
+ }
+
+ public String toString()
+ {
+ return "BootstrapAndJoinPlan{" +
+ "barrier=" + barrier +
+ ", lockKey=" + lockKey +
+ ", toSplitRanges=" + toSplitRanges +
+ ", startJoin=" + startJoin +
+ ", midJoin=" + midJoin +
+ ", finishJoin=" + finishJoin +
+ ", next=" + next +
+ '}';
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ BootstrapAndJoin that = (BootstrapAndJoin) o;
+ return finishJoiningRing == that.finishJoiningRing &&
+ streamData == that.streamData &&
+ Objects.equals(barrier, that.barrier) &&
+ Objects.equals(lockKey, that.lockKey) &&
+ Objects.equals(toSplitRanges, that.toSplitRanges) &&
+ Objects.equals(startJoin, that.startJoin) &&
+ Objects.equals(midJoin, that.midJoin) &&
+ Objects.equals(finishJoin, that.finishJoin) &&
+ next == that.next;
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(barrier, lockKey, toSplitRanges, startJoin, midJoin, finishJoin, next, finishJoiningRing, streamData);
+ }
+
+ public static class Serializer implements AsymmetricMetadataSerializer<InProgressSequence<?>, BootstrapAndJoin>
+ {
+ public void serialize(InProgressSequence<?> t, DataOutputPlus out, Version version) throws IOException
+ {
+ BootstrapAndJoin plan = (BootstrapAndJoin) t;
+ out.writeBoolean(plan.finishJoiningRing);
+ out.writeBoolean(plan.streamData);
+
+ ProgressBarrier.serializer.serialize(plan.barrier(), out, version);
+ LockedRanges.Key.serializer.serialize(plan.lockKey, out, version);
+ PlacementDeltas.serializer.serialize(plan.toSplitRanges, out, version);
+ VIntCoding.writeUnsignedVInt32(plan.next.ordinal(), out);
+
+ if (plan.next.ordinal() >= Transformation.Kind.START_JOIN.ordinal())
+ PrepareJoin.StartJoin.serializer.serialize(plan.startJoin, out, version);
+ if (plan.next.ordinal() >= Transformation.Kind.MID_JOIN.ordinal())
+ PrepareJoin.MidJoin.serializer.serialize(plan.midJoin, out, version);
+ if (plan.next.ordinal() >= Transformation.Kind.FINISH_JOIN.ordinal())
+ PrepareJoin.FinishJoin.serializer.serialize(plan.finishJoin, out, version);
+ }
+
+ public BootstrapAndJoin deserialize(DataInputPlus in, Version version) throws IOException
+ {
+ boolean finishJoiningRing = in.readBoolean();
+ boolean streamData = in.readBoolean();
+
+ ProgressBarrier barrier = ProgressBarrier.serializer.deserialize(in, version);
+ LockedRanges.Key lockKey = LockedRanges.Key.serializer.deserialize(in, version);
+ PlacementDeltas toSplitRanges = PlacementDeltas.serializer.deserialize(in, version);
+ Transformation.Kind next = Transformation.Kind.values()[VIntCoding.readUnsignedVInt32(in)];
+ PrepareJoin.StartJoin startJoin = null;
+ if (next.ordinal() >= Transformation.Kind.START_JOIN.ordinal())
+ startJoin = PrepareJoin.StartJoin.serializer.deserialize(in, version);
+ PrepareJoin.MidJoin midJoin = null;
+ if (next.ordinal() >= Transformation.Kind.MID_JOIN.ordinal())
+ midJoin = PrepareJoin.MidJoin.serializer.deserialize(in, version);
+ PrepareJoin.FinishJoin finishJoin = null;
+ if (next.ordinal() >= Transformation.Kind.FINISH_JOIN.ordinal())
+ finishJoin = PrepareJoin.FinishJoin.serializer.deserialize(in, version);
+
+ return new BootstrapAndJoin(barrier, lockKey, next, toSplitRanges, startJoin, midJoin, finishJoin, finishJoiningRing, streamData);
+ }
+
+ public long serializedSize(InProgressSequence<?> t, Version version)
+ {
+ BootstrapAndJoin plan = (BootstrapAndJoin) t;
+ long size = (TypeSizes.BOOL_SIZE * 2);
+
+ size += ProgressBarrier.serializer.serializedSize(plan.barrier(), version);
+ size += LockedRanges.Key.serializer.serializedSize(plan.lockKey, version);
+ size += PlacementDeltas.serializer.serializedSize(plan.toSplitRanges, version);
+
+ size += VIntCoding.computeVIntSize(plan.kind().ordinal());
+
+ if (plan.kind().ordinal() >= Transformation.Kind.START_JOIN.ordinal())
+ size += PrepareJoin.StartJoin.serializer.serializedSize(plan.startJoin, version);
+ if (plan.kind().ordinal() >= Transformation.Kind.MID_JOIN.ordinal())
+ size += PrepareJoin.MidJoin.serializer.serializedSize(plan.midJoin, version);
+ if (plan.kind().ordinal() >= Transformation.Kind.FINISH_JOIN.ordinal())
+ size += PrepareJoin.FinishJoin.serializer.serializedSize(plan.finishJoin, version);
+
+ return size;
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndReplace.java b/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndReplace.java
new file mode 100644
index 0000000000..382c8caf19
--- /dev/null
+++ b/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndReplace.java
@@ -0,0 +1,320 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tcm.sequences;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.StreamSupport;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.ClusterMetadataService;
+import org.apache.cassandra.tcm.Epoch;
+import org.apache.cassandra.tcm.InProgressSequence;
+import org.apache.cassandra.tcm.Transformation;
+import org.apache.cassandra.tcm.membership.NodeState;
+import org.apache.cassandra.tcm.ownership.DataPlacements;
+import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
+import org.apache.cassandra.tcm.serialization.Version;
+import org.apache.cassandra.tcm.transformations.PrepareReplace;
+import org.apache.cassandra.utils.vint.VIntCoding;
+
+import static org.apache.cassandra.tcm.sequences.BootstrapAndJoin.bootstrap;
+
+public class BootstrapAndReplace implements InProgressSequence<BootstrapAndReplace>
+{
+ private static final Logger logger = LoggerFactory.getLogger(BootstrapAndReplace.class);
+ public static final Serializer serializer = new Serializer();
+
+ public final ProgressBarrier barrier;
+ public final LockedRanges.Key lockKey;
+ public final Set<Token> bootstrapTokens;
+ public final PrepareReplace.StartReplace startReplace;
+ public final PrepareReplace.MidReplace midReplace;
+ public final PrepareReplace.FinishReplace finishReplace;
+ public final Transformation.Kind next;
+
+ public final boolean finishJoiningRing;
+ public final boolean streamData;
+
+ public BootstrapAndReplace(ProgressBarrier barrier,
+ LockedRanges.Key lockKey,
+ Transformation.Kind next,
+ Set<Token> bootstrapTokens,
+ PrepareReplace.StartReplace startReplace,
+ PrepareReplace.MidReplace midReplace,
+ PrepareReplace.FinishReplace finishReplace,
+ boolean finishJoiningRing,
+ boolean streamData)
+ {
+ this.barrier = barrier;
+ this.lockKey = lockKey;
+ this.bootstrapTokens = bootstrapTokens;
+ this.startReplace = startReplace;
+ this.midReplace = midReplace;
+ this.finishReplace = finishReplace;
+ this.next = next;
+ this.finishJoiningRing = finishJoiningRing;
+ this.streamData = streamData;
+ }
+
+ public BootstrapAndReplace advance(Epoch waitForWatermark, Transformation.Kind next)
+ {
+ return new BootstrapAndReplace(barrier.withNewEpoch(waitForWatermark),
+ lockKey,
+ next,
+ bootstrapTokens,
+ startReplace, midReplace, finishReplace,
+ finishJoiningRing,
+ streamData);
+ }
+
+ public InProgressSequences.Kind kind()
+ {
+ return InProgressSequences.Kind.REPLACE;
+ }
+
+ public ProgressBarrier barrier()
+ {
+ return barrier;
+ }
+
+ public Transformation.Kind nextStep()
+ {
+ return next;
+ }
+
+ @Override
+ public boolean executeNext()
+ {
+ switch (next)
+ {
+ // Last transformation is register, so we move to PrepareReplace
+ case START_REPLACE:
+ try
+ {
+ ClusterMetadataService.instance().commit(startReplace);
+ }
+ catch (Throwable e)
+ {
+ return true;
+ }
+
+ case MID_REPLACE:
+ try
+ {
+ ClusterMetadata metadata = ClusterMetadata.current();
+
+ if (streamData)
+ {
+ boolean dataAvailable = bootstrap(bootstrapTokens,
+ StorageService.INDEFINITE,
+ metadata,
+ // TODO: do we need replaced or replacement here?
+ metadata.directory.endpoint(startReplace.replaced()));
+
+ if (!dataAvailable)
+ {
+ logger.warn("Some data streaming failed. Use nodetool to check bootstrap state and resume. " +
+ "For more, see `nodetool help bootstrap`. {}", SystemKeyspace.getBootstrapState());
+ return false;
+ }
+ }
+
+ SystemKeyspace.setBootstrapState(SystemKeyspace.BootstrapState.COMPLETED);
+ StreamSupport.stream(ColumnFamilyStore.all().spliterator(), false)
+ .filter(cfs -> Schema.instance.getUserKeyspaces().names().contains(cfs.keyspace.getName()))
+ .forEach(cfs -> cfs.indexManager.executePreJoinTasksBlocking(true));
+
+ ClusterMetadataService.instance().commit(midReplace);
+ }
+ catch (Throwable e)
+ {
+ return true;
+ }
+ case FINISH_REPLACE:
+ try
+ {
+ if (!finishJoiningRing)
+ {
+ logger.info("Startup complete, but write survey mode is active, not becoming an active ring member. Use JMX (StorageService->joinRing()) to finalize ring joining.");
+ return false;
+ }
+ ClusterMetadataService.instance().commit(finishReplace);
+ }
+ catch (Throwable e)
+ {
+ return true;
+ }
+ break;
+ default:
+ throw new IllegalStateException("Can't proceed with replacement from " + next);
+ }
+
+ return true;
+ }
+
+ @Override
+ public ClusterMetadata.Transformer cancel(ClusterMetadata metadata)
+ {
+ DataPlacements placements = metadata.placements;
+ switch (next)
+ {
+ // need to undo MID_REPLACE and START_REPLACE, but PREPARE_REPLACE doesn't affect placements
+ case FINISH_REPLACE:
+ placements = midReplace.inverseDelta().apply(placements);
+ case MID_REPLACE:
+ case START_REPLACE:
+ placements = startReplace.inverseDelta().apply(placements);
+ break;
+ default:
+ throw new IllegalStateException("Can't revert replacement from " + next);
+ }
+
+ LockedRanges newLockedRanges = metadata.lockedRanges.unlock(lockKey);
+ return metadata.transformer()
+ .withNodeState(startReplace.replacement(), NodeState.REGISTERED)
+ .with(placements)
+ .with(newLockedRanges);
+ }
+
+ public String toString()
+ {
+ return "BootstrapAndReplacePlan{" +
+ "barrier=" + barrier() +
+ "lockKey=" + lockKey +
+ ", bootstrapTokens=" + bootstrapTokens +
+ ", startReplace=" + startReplace +
+ ", midReplace=" + midReplace +
+ ", finishReplace=" + finishReplace +
+ ", next=" + next +
+ ", finishJoiningRing=" + finishJoiningRing +
+ '}';
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ BootstrapAndReplace that = (BootstrapAndReplace) o;
+ return finishJoiningRing == that.finishJoiningRing &&
+ streamData == that.streamData &&
+ Objects.equals(barrier, that.barrier) &&
+ Objects.equals(lockKey, that.lockKey) &&
+ Objects.equals(bootstrapTokens, that.bootstrapTokens) &&
+ Objects.equals(startReplace, that.startReplace) &&
+ Objects.equals(midReplace, that.midReplace) &&
+ Objects.equals(finishReplace, that.finishReplace) && next == that.next;
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(barrier, lockKey, bootstrapTokens, startReplace, midReplace, finishReplace, next, finishJoiningRing, streamData);
+ }
+
+ public static class Serializer implements AsymmetricMetadataSerializer<InProgressSequence<?>, BootstrapAndReplace>
+ {
+ public void serialize(InProgressSequence<?> t, DataOutputPlus out, Version version) throws IOException
+ {
+ BootstrapAndReplace plan = (BootstrapAndReplace) t;
+ out.writeBoolean(plan.finishJoiningRing);
+ out.writeBoolean(plan.streamData);
+
+ out.writeUnsignedVInt32(plan.bootstrapTokens.size());
+ for (Token token : plan.bootstrapTokens)
+ Token.metadataSerializer.serialize(token, out, version);
+
+ ProgressBarrier.serializer.serialize(t.barrier(), out, version);
+ LockedRanges.Key.serializer.serialize(plan.lockKey, out, version);
+
+ VIntCoding.writeUnsignedVInt32(plan.next.ordinal(), out);
+ if (plan.next.ordinal() >= Transformation.Kind.START_REPLACE.ordinal())
+ PrepareReplace.StartReplace.serializer.serialize(plan.startReplace, out, version);
+ if (plan.next.ordinal() >= Transformation.Kind.MID_REPLACE.ordinal())
+ PrepareReplace.MidReplace.serializer.serialize(plan.midReplace, out, version);
+ if (plan.next.ordinal() >= Transformation.Kind.FINISH_REPLACE.ordinal())
+ PrepareReplace.FinishReplace.serializer.serialize(plan.finishReplace, out, version);
+ }
+
+ public BootstrapAndReplace deserialize(DataInputPlus in, Version version) throws IOException
+ {
+ boolean finishJoiningRing = in.readBoolean();
+ boolean streamData = in.readBoolean();
+
+ Set<Token> tokens = new HashSet<>();
+ int tokenCount = VIntCoding.readUnsignedVInt32(in);
+ for (int i = 0; i < tokenCount; i++)
+ tokens.add(Token.metadataSerializer.deserialize(in, version));
+
+ ProgressBarrier barrier = ProgressBarrier.serializer.deserialize(in, version);
+ LockedRanges.Key lockKey = LockedRanges.Key.serializer.deserialize(in, version);
+
+ Transformation.Kind next = Transformation.Kind.values()[VIntCoding.readUnsignedVInt32(in)];
+ PrepareReplace.StartReplace startReplace = null;
+ if (next.ordinal() >= Transformation.Kind.START_REPLACE.ordinal())
+ startReplace = PrepareReplace.StartReplace.serializer.deserialize(in, version);
+ PrepareReplace.MidReplace midReplace = null;
+ if (next.ordinal() >= Transformation.Kind.MID_REPLACE.ordinal())
+ midReplace = PrepareReplace.MidReplace.serializer.deserialize(in, version);
+ PrepareReplace.FinishReplace finishReplace = null;
+ if (next.ordinal() >= Transformation.Kind.FINISH_REPLACE.ordinal())
+ finishReplace = PrepareReplace.FinishReplace.serializer.deserialize(in, version);
+
+ return new BootstrapAndReplace(barrier, lockKey, next, tokens, startReplace, midReplace, finishReplace, finishJoiningRing, streamData);
+ }
+
+ public long serializedSize(InProgressSequence<?> t, Version version)
+ {
+ BootstrapAndReplace plan = (BootstrapAndReplace) t;
+ long size = (TypeSizes.BOOL_SIZE * 2);
+
+ size += VIntCoding.computeVIntSize(plan.bootstrapTokens.size());
+ for (Token token : plan.bootstrapTokens)
+ size += Token.metadataSerializer.serializedSize(token, version);
+
+ size += ProgressBarrier.serializer.serializedSize(plan.barrier(), version);
+ size += LockedRanges.Key.serializer.serializedSize(plan.lockKey, version);
+
+ size += VIntCoding.computeVIntSize(plan.kind().ordinal());
+
+ if (plan.kind().ordinal() >= Transformation.Kind.START_REPLACE.ordinal())
+ size += PrepareReplace.StartReplace.serializer.serializedSize(plan.startReplace, version);
+ if (plan.kind().ordinal() >= Transformation.Kind.MID_REPLACE.ordinal())
+ size += PrepareReplace.MidReplace.serializer.serializedSize(plan.midReplace, version);
+ if (plan.kind().ordinal() >= Transformation.Kind.FINISH_REPLACE.ordinal())
+ size += PrepareReplace.FinishReplace.serializer.serializedSize(plan.finishReplace, version);
+
+ return size;
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/tcm/sequences/InProgressSequences.java b/src/java/org/apache/cassandra/tcm/sequences/InProgressSequences.java
new file mode 100644
index 0000000000..32e78c5fd5
--- /dev/null
+++ b/src/java/org/apache/cassandra/tcm/sequences/InProgressSequences.java
@@ -0,0 +1,240 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tcm.sequences;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableMap;
+
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.tcm.Epoch;
+import org.apache.cassandra.tcm.InProgressSequence;
+import org.apache.cassandra.tcm.MetadataValue;
+import org.apache.cassandra.tcm.membership.NodeId;
+
+import org.apache.cassandra.tcm.serialization.MetadataSerializer;
+import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
+import org.apache.cassandra.tcm.serialization.Version;
+import org.apache.cassandra.tcm.transformations.PrepareLeave;
+
+import static org.apache.cassandra.db.TypeSizes.sizeof;
+
+public class InProgressSequences implements MetadataValue<InProgressSequences>
+{
+ public static final Serializer serializer = new Serializer();
+
+ public static InProgressSequences EMPTY = new InProgressSequences(Epoch.EMPTY, ImmutableMap.of());
+ private final ImmutableMap<NodeId, InProgressSequence<?>> state;
+ private final Epoch lastModified;
+
+ private InProgressSequences(Epoch lastModified, ImmutableMap<NodeId, InProgressSequence<?>> state)
+ {
+ this.lastModified = lastModified;
+ this.state = state;
+ }
+
+ @Override
+ public InProgressSequences withLastModified(Epoch epoch)
+ {
+ return new InProgressSequences(epoch, state);
+ }
+
+ @Override
+ public Epoch lastModified()
+ {
+ return lastModified;
+ }
+
+ public boolean contains(NodeId nodeId)
+ {
+ return state.containsKey(nodeId);
+ }
+
+ public InProgressSequence<?> get(NodeId nodeId)
+ {
+ return state.get(nodeId);
+ }
+
+ public InProgressSequences with(NodeId nodeId, InProgressSequence<?> sequence)
+ {
+ ImmutableMap.Builder<NodeId, InProgressSequence<?>> builder = ImmutableMap.builder();
+ builder.put(nodeId, sequence);
+ for (Map.Entry<NodeId, InProgressSequence<?>> e : state.entrySet())
+ {
+ if (e.getKey().equals(nodeId))
+ continue;
+ builder.put(e.getKey(), e.getValue());
+ }
+ return new InProgressSequences(lastModified, builder.build());
+ }
+
+ public InProgressSequences with(NodeId nodeId, Function<InProgressSequence<?>, InProgressSequence<?>> update)
+ {
+ ImmutableMap.Builder<NodeId, InProgressSequence<?>> builder = ImmutableMap.builder();
+
+ for (Map.Entry<NodeId, InProgressSequence<?>> e : state.entrySet())
+ {
+ if (e.getKey().equals(nodeId))
+ builder.put(e.getKey(), update.apply(e.getValue()));
+ else
+ builder.put(e.getKey(), e.getValue());
+ }
+ return new InProgressSequences(lastModified, builder.build());
+ }
+
+ public InProgressSequences without(NodeId nodeId)
+ {
+ ImmutableMap.Builder<NodeId, InProgressSequence<?>> builder = ImmutableMap.builder();
+ boolean removed = false;
+ for (Map.Entry<NodeId, InProgressSequence<?>> e : state.entrySet())
+ {
+ if (e.getKey().equals(nodeId))
+ removed = true;
+ else
+ builder.put(e.getKey(), e.getValue());
+ }
+ assert removed : String.format("Expected to remove node %s, but it wasn't found in in-progress sequences", nodeId);
+ return new InProgressSequences(lastModified, builder.build());
+
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ InProgressSequences that = (InProgressSequences) o;
+ return Objects.equals(state, that.state) && Objects.equals(lastModified, that.lastModified);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(state, lastModified);
+ }
+
+ public enum Kind
+ {
+ JOIN(BootstrapAndJoin.serializer),
+ MOVE(Move.serializer),
+ REPLACE(BootstrapAndReplace.serializer),
+ LEAVE(UnbootstrapAndLeave.serializer),
+ ;
+
+ public final AsymmetricMetadataSerializer<InProgressSequence<?>, ? extends InProgressSequence<?>> serializer;
+
+ Kind(AsymmetricMetadataSerializer<InProgressSequence<?>, ? extends InProgressSequence<?>> serializer)
+ {
+ this.serializer = serializer;
+ }
+ }
+
+ @VisibleForTesting
+ public enum SequenceState { BLOCKED, CONTINUING, HALTED}
+
+ @VisibleForTesting
+ public static BiFunction<InProgressSequence<?>, SequenceState, SequenceState> listener = (s, o) -> o;
+
+ @VisibleForTesting
+ public static BiFunction<InProgressSequence<?>, SequenceState, SequenceState> replaceListener(BiFunction<InProgressSequence<?>, SequenceState, SequenceState> newListener)
+ {
+ BiFunction<InProgressSequence<?>, SequenceState, SequenceState> prev = listener;
+ listener = newListener;
+ return prev;
+ }
+
+ public static boolean resume(InProgressSequence<?> sequence)
+ {
+ SequenceState state;
+ if (sequence.barrier().await())
+ state = listener.apply(sequence, sequence.executeNext() ? SequenceState.CONTINUING : SequenceState.HALTED);
+ else
+ state = listener.apply(sequence, SequenceState.BLOCKED);
+ return state == SequenceState.CONTINUING;
+ }
+
+ public static boolean isLeave(InProgressSequence<?> sequence)
+ {
+ // TODO only the first is an InProgressSequence
+ return sequence instanceof UnbootstrapAndLeave ||
+ sequence instanceof PrepareLeave.StartLeave ||
+ sequence instanceof PrepareLeave.MidLeave ||
+ sequence instanceof PrepareLeave.FinishLeave;
+ }
+
+ public static class Serializer implements MetadataSerializer<InProgressSequences>
+ {
+ public void serialize(InProgressSequences t, DataOutputPlus out, Version version) throws IOException
+ {
+ Epoch.serializer.serialize(t.lastModified, out, version);
+ out.writeInt(t.state.size());
+ for (Map.Entry<NodeId, InProgressSequence<?>> entry : t.state.entrySet())
+ {
+ NodeId.serializer.serialize(entry.getKey(), out, version);
+ InProgressSequence<?> seq = entry.getValue();
+ out.writeUTF(seq.kind().name());
+ entry.getValue().kind().serializer.serialize(seq, out, version);
+ }
+ }
+
+ public InProgressSequences deserialize(DataInputPlus in, Version version) throws IOException
+ {
+ Epoch lastModified = Epoch.serializer.deserialize(in, version);
+ int ipsSize = in.readInt();
+ ImmutableMap.Builder<NodeId, InProgressSequence<?>> res = ImmutableMap.builder();
+ for (int i = 0; i < ipsSize; i++)
+ {
+ NodeId nodeId = NodeId.serializer.deserialize(in, version);
+ Kind kind = Kind.valueOf(in.readUTF());
+ InProgressSequence<?> ips = kind.serializer.deserialize(in, version);
+ res.put(nodeId, ips);
+ }
+ return new InProgressSequences(lastModified, res.build());
+ }
+
+ public long serializedSize(InProgressSequences t, Version version)
+ {
+ long size = Epoch.serializer.serializedSize(t.lastModified, version);
+ size += sizeof(t.state.size());
+ for (Map.Entry<NodeId, InProgressSequence<?>> entry : t.state.entrySet())
+ {
+ size += NodeId.serializer.serializedSize(entry.getKey(), version);
+ InProgressSequence<?> seq = entry.getValue();
+ size += sizeof(seq.kind().name());
+ size += entry.getValue().kind().serializer.serializedSize(seq, version);
+ }
+ return size;
+ }
+ }
+
+ @Override
+ public String toString()
+ {
+ return "InProgressSequences{" +
+ "lastModified=" + lastModified +
+ ", state=" + state +
+ '}';
+ }
+}
diff --git a/src/java/org/apache/cassandra/tcm/sequences/Move.java b/src/java/org/apache/cassandra/tcm/sequences/Move.java
new file mode 100644
index 0000000000..650eadb329
--- /dev/null
+++ b/src/java/org/apache/cassandra/tcm/sequences/Move.java
@@ -0,0 +1,360 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tcm.sequences;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.ExecutionException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.gms.FailureDetector;
+import org.apache.cassandra.gms.IFailureDetector;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.EndpointsByReplica;
+import org.apache.cassandra.locator.EndpointsForRange;
+import org.apache.cassandra.locator.RangesAtEndpoint;
+import org.apache.cassandra.locator.RangesByEndpoint;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.Keyspaces;
+import org.apache.cassandra.schema.ReplicationParams;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.streaming.StreamOperation;
+import org.apache.cassandra.streaming.StreamPlan;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.ClusterMetadataService;
+import org.apache.cassandra.tcm.Epoch;
+import org.apache.cassandra.tcm.InProgressSequence;
+import org.apache.cassandra.tcm.Transformation;
+import org.apache.cassandra.tcm.membership.NodeState;
+import org.apache.cassandra.tcm.ownership.DataPlacements;
+import org.apache.cassandra.tcm.ownership.MovementMap;
+import org.apache.cassandra.tcm.ownership.PlacementDeltas;
+import org.apache.cassandra.tcm.ownership.PlacementForRange;
+import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
+import org.apache.cassandra.tcm.serialization.Version;
+import org.apache.cassandra.tcm.transformations.PrepareMove;
+import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.apache.cassandra.utils.vint.VIntCoding;
+
+public class Move implements InProgressSequence<Move>
+{
+ private static final Logger logger = LoggerFactory.getLogger(Move.class);
+ public static final Serializer serializer = new Serializer();
+
+ public final Collection<Token> tokens;
+ public final ProgressBarrier barrier;
+ public final LockedRanges.Key lockKey;
+ public final Transformation.Kind next;
+
+ public final PlacementDeltas toSplitRanges;
+ public final PrepareMove.StartMove startMove;
+ public final PrepareMove.MidMove midMove;
+ public final PrepareMove.FinishMove finishMove;
+ public final boolean streamData;
+
+
+ public Move(Collection<Token> tokens,
+ ProgressBarrier barrier,
+ LockedRanges.Key lockKey,
+ Transformation.Kind next,
+ PlacementDeltas toSplitRanges,
+ PrepareMove.StartMove startMove,
+ PrepareMove.MidMove midMove,
+ PrepareMove.FinishMove finishMove,
+ boolean streamData)
+ {
+ this.tokens = tokens;
+ this.barrier = barrier;
+ this.lockKey = lockKey;
+ this.next = next;
+ this.toSplitRanges = toSplitRanges;
+ this.startMove = startMove;
+ this.midMove = midMove;
+ this.finishMove = finishMove;
+
+ this.streamData = streamData;
+ }
+
+ @Override
+ public InProgressSequences.Kind kind()
+ {
+ return InProgressSequences.Kind.MOVE;
+ }
+
+ @Override
+ public ProgressBarrier barrier()
+ {
+ return barrier;
+ }
+
+ @Override
+ public Transformation.Kind nextStep()
+ {
+ return next;
+ }
+
+ @Override
+ public boolean executeNext()
+ {
+ switch (next)
+ {
+ case START_MOVE:
+ try
+ {
+ ClusterMetadata metadata = ClusterMetadata.current();
+ logger.info("Moving {} from {} to {}.",
+ metadata.directory.endpoint(startMove.nodeId()),
+ metadata.tokenMap.tokens(startMove.nodeId()),
+ finishMove.newTokens);
+ ClusterMetadataService.instance().commit(startMove);
+ }
+ catch (Throwable t)
+ {
+ JVMStabilityInspector.inspectThrowable(t);
+ return true;
+ }
+ break;
+ case MID_MOVE:
+ try
+ {
+ logger.info("fetching new ranges and streaming old ranges");
+ StreamPlan streamPlan = new StreamPlan(StreamOperation.RELOCATION);
+ Keyspaces keyspaces = Schema.instance.getNonLocalStrategyKeyspaces();
+ Map<ReplicationParams, EndpointsByReplica> movementMap = movementMap(FailureDetector.instance,
+ ClusterMetadata.current().placements,
+ startMove.delta())
+ .asMap();
+ for (KeyspaceMetadata ks : keyspaces)
+ {
+ ReplicationParams replicationParams = ks.params.replication;
+ if (replicationParams.isMeta())
+ continue;
+ EndpointsByReplica endpoints = movementMap.get(replicationParams);
+ for (Map.Entry<Replica, Replica> e : endpoints.flattenEntries())
+ {
+ Replica source = e.getKey();
+ Replica destination = e.getValue();
+ assert !source.endpoint().equals(destination.endpoint()) : String.format("Source should not be the same as destionation", source, destination);
+ if (source.isSelf())
+ streamPlan.transferRanges(destination.endpoint(), ks.name, RangesAtEndpoint.of(destination));
+ else if (destination.isSelf())
+ streamPlan.requestRanges(source.endpoint(), ks.name, RangesAtEndpoint.of(destination), RangesAtEndpoint.empty(destination.endpoint()));
+ else
+ throw new IllegalStateException("Node should be either source or desintation in the movement map " + endpoints);
+ }
+ }
+
+ streamPlan.execute().get();
+ StorageService.instance.repairPaxosForTopologyChange("move");
+ }
+ catch (InterruptedException e)
+ {
+ return true;
+ }
+ catch (ExecutionException e)
+ {
+ throw new RuntimeException("Unable to move", e);
+ }
+
+ try
+ {
+ ClusterMetadataService.instance().commit(midMove);
+ }
+ catch (Throwable t)
+ {
+ JVMStabilityInspector.inspectThrowable(t);
+ return true;
+ }
+ break;
+ case FINISH_MOVE:
+ try
+ {
+ SystemKeyspace.updateTokens(tokens);
+ ClusterMetadataService.instance().commit(finishMove);
+ }
+ catch (Throwable t)
+ {
+ JVMStabilityInspector.inspectThrowable(t);
+ return true;
+ }
+
+ break;
+ default:
+ throw new IllegalStateException("Can't proceed with join from " + next);
+ }
+
+ return true;
+ }
+
+ private MovementMap movementMap(IFailureDetector fd, DataPlacements placements, PlacementDeltas toStart)
+ {
+ MovementMap.Builder allMovements = MovementMap.builder();
+ toStart.forEach((params, delta) -> {
+ RangesByEndpoint targets = delta.writes.additions;
+ PlacementForRange oldOwners = placements.get(params).reads;
+ EndpointsByReplica.Builder movements = new EndpointsByReplica.Builder();
+ targets.flattenValues().forEach(destination -> {
+ EndpointsForRange candidates = oldOwners.forRange(destination.range());
+ Optional<Replica> maybeSelf = candidates.stream().filter(Replica::isSelf).findFirst();
+ if (maybeSelf.isPresent())
+ {
+ movements.put(maybeSelf.get(), destination);
+ return;
+ }
+
+ assert destination.isSelf();
+ for (Replica source : candidates)
+ {
+ if ( fd.isAlive(source.endpoint()))
+ {
+ movements.put(source, destination);
+ return;
+ }
+ }
+ throw new IllegalStateException(String.format("No live sources for the range %s. Tried: %s",
+ destination.range(), oldOwners.forRange(destination.range())));
+ });
+ allMovements.put(params, movements.build());
+ });
+
+ return allMovements.build();
+ }
+
+ @Override
+ public ClusterMetadata.Transformer cancel(ClusterMetadata metadata)
+ {
+ DataPlacements placements = metadata.placements;
+
+ switch (next)
+ {
+ case FINISH_MOVE:
+ placements = midMove.inverseDelta().apply(placements);
+ case MID_MOVE:
+ placements = startMove.inverseDelta().apply(placements);
+ case START_MOVE:
+ placements = toSplitRanges.invert().apply(placements);
+ break;
+ default:
+ throw new IllegalStateException("Can't revert move from " + next);
+ }
+
+ LockedRanges newLockedRanges = metadata.lockedRanges.unlock(lockKey);
+ return metadata.transformer()
+ .unproposeTokens(startMove.nodeId(), tokens)
+ .withNodeState(startMove.nodeId(), NodeState.JOINED)
+ .with(placements)
+ .with(newLockedRanges);
+ }
+
+ public Move advance(Epoch waitForWatermark, Transformation.Kind next)
+ {
+ return new Move(tokens,
+ barrier().withNewEpoch(waitForWatermark), lockKey, next,
+ toSplitRanges, startMove, midMove, finishMove,
+ streamData);
+ }
+
+ public static class Serializer implements AsymmetricMetadataSerializer<InProgressSequence<?>, Move>
+ {
+ public void serialize(InProgressSequence<?> t, DataOutputPlus out, Version version) throws IOException
+ {
+ Move plan = (Move) t;
+ out.writeBoolean(plan.streamData);
+
+ ProgressBarrier.serializer.serialize(plan.barrier(), out, version);
+ LockedRanges.Key.serializer.serialize(plan.lockKey, out, version);
+ PlacementDeltas.serializer.serialize(plan.toSplitRanges, out, version);
+ VIntCoding.writeUnsignedVInt32(plan.next.ordinal(), out);
+
+ if (plan.next.ordinal() >= Transformation.Kind.START_MOVE.ordinal())
+ PrepareMove.StartMove.serializer.serialize(plan.startMove, out, version);
+ if (plan.next.ordinal() >= Transformation.Kind.MID_MOVE.ordinal())
+ PrepareMove.MidMove.serializer.serialize(plan.midMove, out, version);
+ if (plan.next.ordinal() >= Transformation.Kind.FINISH_MOVE.ordinal())
+ PrepareMove.FinishMove.serializer.serialize(plan.finishMove, out, version);
+
+ out.writeUnsignedVInt32(plan.tokens.size());
+ for (Token token : plan.tokens)
+ Token.metadataSerializer.serialize(token, out, version);
+ }
+
+ public Move deserialize(DataInputPlus in, Version version) throws IOException
+ {
+ boolean streamData = in.readBoolean();
+
+ ProgressBarrier barrier = ProgressBarrier.serializer.deserialize(in, version);
+ LockedRanges.Key lockKey = LockedRanges.Key.serializer.deserialize(in, version);
+ PlacementDeltas toSplitRanges = PlacementDeltas.serializer.deserialize(in, version);
+ Transformation.Kind next = Transformation.Kind.values()[VIntCoding.readUnsignedVInt32(in)];
+
+ PrepareMove.StartMove startMove = null;
+ if (next.ordinal() >= Transformation.Kind.START_MOVE.ordinal())
+ startMove = PrepareMove.StartMove.serializer.deserialize(in, version);
+ PrepareMove.MidMove midMove = null;
+ if (next.ordinal() >= Transformation.Kind.MID_MOVE.ordinal())
+ midMove = PrepareMove.MidMove.serializer.deserialize(in, version);
+ PrepareMove.FinishMove finishMove = null;
+ if (next.ordinal() >= Transformation.Kind.FINISH_MOVE.ordinal())
+ finishMove = PrepareMove.FinishMove.serializer.deserialize(in, version);
+
+ int numTokens = in.readUnsignedVInt32();
+ List<Token> tokens = new ArrayList<>();
+ for (int i = 0; i < numTokens; i++)
+ tokens.add(Token.metadataSerializer.deserialize(in, version));
+ return new Move(tokens, barrier, lockKey, next,
+ toSplitRanges, startMove, midMove, finishMove, streamData);
+ }
+
+ public long serializedSize(InProgressSequence<?> t, Version version)
+ {
+ Move plan = (Move) t;
+ long size = TypeSizes.BOOL_SIZE;
+
+ size += ProgressBarrier.serializer.serializedSize(plan.barrier(), version);
+ size += LockedRanges.Key.serializer.serializedSize(plan.lockKey, version);
+ size += PlacementDeltas.serializer.serializedSize(plan.toSplitRanges, version);
+
+ size += VIntCoding.computeVIntSize(plan.kind().ordinal());
+
+ if (plan.kind().ordinal() >= Transformation.Kind.START_MOVE.ordinal())
+ size += PrepareMove.StartMove.serializer.serializedSize(plan.startMove, version);
+ if (plan.kind().ordinal() >= Transformation.Kind.MID_MOVE.ordinal())
+ size += PrepareMove.MidMove.serializer.serializedSize(plan.midMove, version);
+ if (plan.kind().ordinal() >= Transformation.Kind.FINISH_MOVE.ordinal())
+ size += PrepareMove.FinishMove.serializer.serializedSize(plan.finishMove, version);
+
+ size += TypeSizes.sizeofUnsignedVInt(plan.tokens.size());
+ for (Token token : plan.tokens)
+ size += Token.metadataSerializer.serializedSize(token, version);
+ return size;
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/tcm/sequences/ProgressBarrier.java b/src/java/org/apache/cassandra/tcm/sequences/ProgressBarrier.java
new file mode 100644
index 0000000000..5e4e03cad9
--- /dev/null
+++ b/src/java/org/apache/cassandra/tcm/sequences/ProgressBarrier.java
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tcm.sequences;
+
+import java.io.IOException;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.ImmutableSet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.exceptions.RequestFailureReason;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.tcm.Epoch;
+import org.apache.cassandra.tcm.serialization.MetadataSerializer;
+import org.apache.cassandra.tcm.serialization.Version;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.NoPayload;
+import org.apache.cassandra.net.RequestCallback;
+import org.apache.cassandra.net.RequestCallbackWithFailure;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.membership.Directory;
+import org.apache.cassandra.tcm.membership.NodeId;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.concurrent.CountDownLatch;
+
+public class ProgressBarrier
+{
+ private static final Logger logger = LoggerFactory.getLogger(ProgressBarrier.class);
+ private static final long TIMEOUT_MILLIS = CassandraRelevantProperties.PROGRESS_BARRIER_TIMEOUT_MILLIS.getLong();
+ private static final long BACKOFF_MILLIS = CassandraRelevantProperties.PROGRESS_BARRIER_BACKOFF_MILLIS.getLong();
+
+ public static final ProgressBarrier NONE = new ProgressBarrier(Epoch.EMPTY, ImmutableSet.of(), false)
+ {
+ @Override
+ public boolean await()
+ {
+ return true;
+ }
+ };
+
+ public static final Serializer serializer = new Serializer();
+
+ public final Epoch epoch;
+ public final ImmutableSet<NodeId> affectedPeers;
+ private final boolean adjustBlockForDuringReplacements;
+
+ // For use by multistep operations where the initial step has no prerequisite, but subsequent ones do. In contrast
+ // to ops where no step has a prerequisite and so no gating is required which can use ProgressBarrier.NONE, in this
+ // case we want to be able to pass the affected ranges, while adding a "real" epoch, when advancing the operation.
+ public static ProgressBarrier immediate(ImmutableSet<NodeId> affectedPeers)
+ {
+ return new ProgressBarrier(Epoch.EMPTY, affectedPeers, false);
+ }
+
+ public ProgressBarrier(Epoch epoch, ImmutableSet<NodeId> affectedPeers, boolean adjustBlockForDuringReplacements)
+ {
+ this.epoch = epoch;
+ this.affectedPeers = affectedPeers;
+ this.adjustBlockForDuringReplacements = adjustBlockForDuringReplacements;
+ }
+
+ public ProgressBarrier withNewEpoch(Epoch newEpoch)
+ {
+ return new ProgressBarrier(newEpoch, affectedPeers, adjustBlockForDuringReplacements);
+ }
+
+ public boolean await()
+ {
+ assert epoch != null : "Found unexpected null epoch in progress barrier";
+ if (epoch.isBefore(Epoch.FIRST) || affectedPeers.isEmpty())
+ {
+ logger.debug("Empty epoch or no affected peers for barrier, returning immediately");
+ return true;
+ }
+ logger.info("Waiting for required acks of epoch {}", epoch);
+ // TODO: is this even going to work each quorum and not local quorum?
+ // TODO: this can be optimised quite a bit
+ // TODO: make this DC-aware!
+ Directory directory = ClusterMetadata.current().directory;
+ Set<InetAddressAndPort> endpoints = affectedPeers.stream().map(directory::endpoint).collect(Collectors.toSet());
+ int blockFor = endpoints.size() / 2 + 1;
+
+ if (adjustBlockForDuringReplacements && blockFor == endpoints.size())
+ {
+ logger.warn("Reducing the number of replicas to block for from {} to allow replacement to go through.", blockFor);
+ blockFor--;
+ }
+
+ logger.info("Waiting for {} required acks of epoch {}", blockFor, epoch);
+ // TODO replace latch + boolean with something more capable for handling DC awareness etc
+ CountDownLatch latch = CountDownLatch.newCountDownLatch(blockFor);
+ AtomicBoolean active = new AtomicBoolean(true);
+ for (InetAddressAndPort peer : endpoints)
+ sendRequest(peer, active, latch);
+
+ boolean acked = latch.awaitUninterruptibly(TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);
+ active.set(false);
+ if (!acked)
+ {
+ logger.info("Timed out waiting for acks of epoch {} from {}", epoch, endpoints);
+ return false;
+ }
+ return true;
+ }
+
+ private void sendRequest(InetAddressAndPort to, AtomicBoolean active, CountDownLatch blockingFor)
+ {
+ logger.debug("Sending watermark request to {}, expecting at least {}", to, epoch);
+ RequestCallback<NoPayload> callback = new RequestCallbackWithFailure<NoPayload>()
+ {
+ @Override
+ public void onResponse(Message<NoPayload> msg)
+ {
+ Epoch remote = msg.epoch();
+ logger.debug("Received watermark response from {} with epoch {}", msg.from(), remote);
+ if (remote.isEqualOrAfter(epoch))
+ blockingFor.decrement();
+ else
+ retry(to, active, blockingFor);
+ }
+
+ @Override
+ public void onFailure(InetAddressAndPort from, RequestFailureReason failureReason)
+ {
+ logger.debug("Watermark failure from {} with reason {}", from, failureReason);
+ retry(to, active, blockingFor);
+ }
+
+ private void retry(InetAddressAndPort to, AtomicBoolean active, CountDownLatch blockingFor)
+ {
+ FBUtilities.sleepQuietly(BACKOFF_MILLIS);
+ sendRequest(to, active, blockingFor);
+ }
+ };
+
+ if (active.get())
+ MessagingService.instance().sendWithCallback(Message.out(Verb.TCM_CURRENT_EPOCH_REQ, NoPayload.noPayload), to, callback);
+ }
+
+ @Override
+ public String toString()
+ {
+ return "ProgressBarrier{" +
+ "epoch=" + epoch +
+ ", affectedPeers=" + affectedPeers +
+ '}';
+ }
+
+ public static final class Serializer implements MetadataSerializer<ProgressBarrier>
+ {
+ @Override
+ public void serialize(ProgressBarrier t, DataOutputPlus out, Version version) throws IOException
+ {
+ boolean hasCondition = !t.equals(NONE);
+ out.writeBoolean(hasCondition);
+ if (hasCondition)
+ {
+ Epoch.serializer.serialize(t.epoch, out, version);
+ out.writeUnsignedVInt32(t.affectedPeers.size());
+ out.writeBoolean(t.adjustBlockForDuringReplacements);
+ for (NodeId id : t.affectedPeers)
+ NodeId.serializer.serialize(id, out, version);
+ }
+ }
+
+ @Override
+ public ProgressBarrier deserialize(DataInputPlus in, Version version) throws IOException
+ {
+ boolean hasCondition = !in.readBoolean();
+ if (hasCondition)
+ {
+ Epoch epoch = Epoch.serializer.deserialize(in);
+ boolean adjustBlockForDuringReplacements = in.readBoolean();
+ int size = in.readUnsignedVInt32();
+ ImmutableSet.Builder<NodeId> peers = ImmutableSet.builderWithExpectedSize(size);
+ while (size-- > 0)
+ peers.add(NodeId.serializer.deserialize(in, version));
+ return new ProgressBarrier(epoch, peers.build(), adjustBlockForDuringReplacements);
+ }
+ return NONE;
+ }
+
+ @Override
+ public long serializedSize(ProgressBarrier t, Version version)
+ {
+ long size = TypeSizes.BOOL_SIZE;
+ if (!t.equals(NONE))
+ {
+ size += Epoch.serializer.serializedSize(t.epoch, version);
+ size += TypeSizes.BOOL_SIZE;
+ size += TypeSizes.sizeofUnsignedVInt(t.affectedPeers.size());
+ for(NodeId id : t.affectedPeers)
+ size += NodeId.serializer.serializedSize(id, version);
+ }
+ return size;
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/tcm/sequences/UnbootstrapAndLeave.java b/src/java/org/apache/cassandra/tcm/sequences/UnbootstrapAndLeave.java
new file mode 100644
index 0000000000..3d1d49ed8b
--- /dev/null
+++ b/src/java/org/apache/cassandra/tcm/sequences/UnbootstrapAndLeave.java
@@ -0,0 +1,349 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tcm.sequences;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.batchlog.BatchlogManager;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.EndpointsByReplica;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.RangesByEndpoint;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.locator.SystemStrategy;
+import org.apache.cassandra.schema.Keyspaces;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.streaming.StreamState;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.ClusterMetadataService;
+import org.apache.cassandra.tcm.Epoch;
+import org.apache.cassandra.tcm.InProgressSequence;
+import org.apache.cassandra.tcm.Transformation;
+import org.apache.cassandra.tcm.membership.NodeState;
+import org.apache.cassandra.tcm.ownership.DataPlacements;
+import org.apache.cassandra.tcm.ownership.MovementMap;
+import org.apache.cassandra.tcm.ownership.PlacementDeltas;
+import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
+import org.apache.cassandra.tcm.serialization.Version;
+import org.apache.cassandra.tcm.transformations.PrepareLeave;
+import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.apache.cassandra.utils.vint.VIntCoding;
+
+public class UnbootstrapAndLeave implements InProgressSequence<UnbootstrapAndLeave>
+{
+ private static final Logger logger = LoggerFactory.getLogger(UnbootstrapAndLeave.class);
+ public static final Serializer serializer = new Serializer();
+
+ public final PrepareLeave.StartLeave startLeave;
+ public final PrepareLeave.MidLeave midLeave;
+ public final PrepareLeave.FinishLeave finishLeave;
+
+ public final ProgressBarrier barrier;
+ public final LockedRanges.Key lockKey;
+ public final Transformation.Kind next;
+
+ public UnbootstrapAndLeave(ProgressBarrier barrier,
+ LockedRanges.Key lockKey,
+ Transformation.Kind next,
+ PrepareLeave.StartLeave startLeave,
+ PrepareLeave.MidLeave midLeave,
+ PrepareLeave.FinishLeave finishLeave)
+ {
+ this.barrier = barrier;
+ this.lockKey = lockKey;
+ this.next = next;
+ this.startLeave = startLeave;
+ this.midLeave = midLeave;
+ this.finishLeave = finishLeave;
+ }
+
+ public UnbootstrapAndLeave advance(Epoch waitForWatermark, Transformation.Kind next)
+ {
+ return new UnbootstrapAndLeave(barrier().withNewEpoch(waitForWatermark), lockKey, next,
+ startLeave, midLeave, finishLeave);
+ }
+
+ public ProgressBarrier barrier()
+ {
+ return barrier;
+ }
+
+ public Transformation.Kind nextStep()
+ {
+ return next;
+ }
+
+ public InProgressSequences.Kind kind()
+ {
+ return InProgressSequences.Kind.LEAVE;
+ }
+
+ public boolean executeNext()
+ {
+ switch (next)
+ {
+ case START_LEAVE:
+ try
+ {
+ ClusterMetadataService.instance().commit(startLeave);
+ }
+ catch (Throwable t)
+ {
+ JVMStabilityInspector.inspectThrowable(t);
+ return true;
+ }
+ break;
+ case MID_LEAVE:
+ try
+ {
+ unbootstrap(Schema.instance.getNonLocalStrategyKeyspaces(),
+ movementMap(ClusterMetadata.current().directory.endpoint(startLeave.nodeId()),
+ startLeave.delta(), finishLeave.delta()));
+ }
+ catch (InterruptedException e)
+ {
+ return true;
+ }
+ catch (ExecutionException e)
+ {
+ JVMStabilityInspector.inspectThrowable(e);
+ throw new RuntimeException("Unable to unbootstrap", e);
+ }
+
+ try
+ {
+ ClusterMetadataService.instance().commit(midLeave);
+ }
+ catch (Throwable t)
+ {
+ JVMStabilityInspector.inspectThrowable(t);
+ return true;
+ }
+ break;
+ case FINISH_LEAVE:
+ try
+ {
+ ClusterMetadataService.instance().commit(finishLeave);
+ }
+ catch (Throwable t)
+ {
+ JVMStabilityInspector.inspectThrowable(t);
+ return true;
+ }
+// StorageService.instance.finishLeaving();
+ break;
+ default:
+ throw new IllegalStateException("Can't proceed with leave from " + next);
+ }
+
+ return true;
+ }
+
+ private MovementMap movementMap(InetAddressAndPort leaving, PlacementDeltas startDelta, PlacementDeltas finishDelta)
+ {
+ MovementMap.Builder allMovements = MovementMap.builder();
+ // map of src->dest movements, keyed by replication settings. During unbootstrap, this will be used to construct
+ // a stream plan for each keyspace, based on their replication params.
+ finishDelta.forEach((params, delta) -> {
+ // no streaming for LocalStrategy and friends
+ if (SystemStrategy.class.isAssignableFrom(params.klass))
+ return;
+
+ // first identify ranges to be migrated off the leaving node
+ Map<Range<Token>, Replica> oldReplicas = delta.writes.removals.get(leaving).byRange();
+
+ // next go through the additions to the write groups that will be applied during the
+ // first step of the plan. These represent the ranges moving to new replicas so in
+ // order to construct a streaming plan we can match these up with the corresponding
+ // removals to produce a src->dest mapping.
+ EndpointsByReplica.Builder movements = new EndpointsByReplica.Builder();
+ RangesByEndpoint startWriteAdditions = startDelta.get(params).writes.additions;
+ startWriteAdditions.flattenValues()
+ .forEach(newReplica -> {
+ movements.put(oldReplicas.get(newReplica.range()), newReplica);
+ });
+
+ allMovements.put(params, movements.build());
+ });
+ return allMovements.build();
+ }
+
+ @Override
+ public ClusterMetadata.Transformer cancel(ClusterMetadata metadata)
+ {
+ DataPlacements placements = metadata.placements;
+ switch (next)
+ {
+ // need to undo MID_LEAVE and START_LEAVE, but PrepareLeave doesn't affect placement
+ case FINISH_LEAVE:
+ placements = midLeave.inverseDelta().apply(placements);
+ case MID_LEAVE:
+ case START_LEAVE:
+ placements = startLeave.inverseDelta().apply(placements);
+ break;
+ default:
+ throw new IllegalStateException("Can't revert leave from " + next);
+ }
+ LockedRanges newLockedRanges = metadata.lockedRanges.unlock(lockKey);
+ return metadata.transformer()
+ .with(placements)
+ .with(newLockedRanges)
+ .withNodeState(startLeave.nodeId(), NodeState.JOINED);
+ }
+
+ private static void unbootstrap(Keyspaces keyspaces, MovementMap movements) throws ExecutionException, InterruptedException
+ {
+ Supplier<Future<StreamState>> startStreaming = prepareUnbootstrapStreaming(keyspaces, movements);
+
+ logger.info("replaying batch log and streaming data to other nodes");
+ // Start with BatchLog replay, which may create hints but no writes since this is no longer a valid endpoint.
+ Future<?> batchlogReplay = BatchlogManager.instance.startBatchlogReplay();
+ Future<StreamState> streamSuccess = startStreaming.get();
+
+ // Wait for batch log to complete before streaming hints.
+ logger.debug("waiting for batch log processing.");
+ batchlogReplay.get();
+
+ logger.info("streaming hints to other nodes");
+
+ Future<?> hintsSuccess = StorageService.instance.streamHints();
+
+ // wait for the transfer runnables to signal the latch.
+ logger.debug("waiting for stream acks.");
+ streamSuccess.get();
+ hintsSuccess.get();
+ logger.debug("stream acks all received.");
+ }
+
+ private static Supplier<Future<StreamState>> prepareUnbootstrapStreaming(Keyspaces keyspaces,
+ MovementMap movements)
+ {
+ // PrepareLeave transformation gives us a map of range movements for unbootstrap, keyed on replication settings.
+ // The movements themselves are maps of leavingReplica -> newReplica(s). Here we just "inflate" the outer
+ // map to a set of movements per-keyspace, duplicating where keyspaces share the same replication params
+ Map<String, EndpointsByReplica> byKeyspace =
+ keyspaces.stream()
+ .collect(Collectors.toMap(k -> k.name,
+ k -> movements.get(k.params.replication)));
+
+ return () -> StorageService.instance.streamRanges(byKeyspace);
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ UnbootstrapAndLeave that = (UnbootstrapAndLeave) o;
+ return Objects.equals(startLeave, that.startLeave) &&
+ Objects.equals(midLeave, that.midLeave) &&
+ Objects.equals(finishLeave, that.finishLeave) &&
+ Objects.equals(barrier, that.barrier) &&
+ Objects.equals(lockKey, that.lockKey) &&
+ next == that.next;
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(startLeave, midLeave, finishLeave, barrier, lockKey, next);
+ }
+
+ @Override
+ public String toString()
+ {
+ return "UnbootstrapAndLeavePlan{" +
+ "barrier=" + barrier +
+ ", lockKey=" + lockKey +
+ ", startLeave=" + startLeave +
+ ", midLeave=" + midLeave +
+ ", finishLeave=" + finishLeave +
+ ", next=" + next +
+ '}';
+ }
+
+ public static class Serializer implements AsymmetricMetadataSerializer<InProgressSequence<?>, UnbootstrapAndLeave>
+ {
+ public void serialize(InProgressSequence<?> t, DataOutputPlus out, Version version) throws IOException
+ {
+ UnbootstrapAndLeave plan = (UnbootstrapAndLeave) t;
+
+ ProgressBarrier.serializer.serialize(t.barrier(), out, version);
+ LockedRanges.Key.serializer.serialize(plan.lockKey, out, version);
+
+ VIntCoding.writeUnsignedVInt32(plan.next.ordinal(), out);
+ if (plan.next.ordinal() >= Transformation.Kind.START_LEAVE.ordinal())
+ PrepareLeave.StartLeave.serializer.serialize(plan.startLeave, out, version);
+ if (plan.next.ordinal() >= Transformation.Kind.MID_LEAVE.ordinal())
+ PrepareLeave.MidLeave.serializer.serialize(plan.midLeave, out, version);
+ if (plan.next.ordinal() >= Transformation.Kind.FINISH_LEAVE.ordinal())
+ PrepareLeave.FinishLeave.serializer.serialize(plan.finishLeave, out, version);
+ }
+
+ public UnbootstrapAndLeave deserialize(DataInputPlus in, Version version) throws IOException
+ {
+ ProgressBarrier barrier = ProgressBarrier.serializer.deserialize(in, version);
+ LockedRanges.Key lockKey = LockedRanges.Key.serializer.deserialize(in, version);
+
+ Transformation.Kind next = Transformation.Kind.values()[VIntCoding.readUnsignedVInt32(in)];
+ PrepareLeave.StartLeave startLeave = null;
+ if (next.ordinal() >= Transformation.Kind.START_LEAVE.ordinal())
+ startLeave = PrepareLeave.StartLeave.serializer.deserialize(in, version);
+ PrepareLeave.MidLeave midLeave = null;
+ if (next.ordinal() >= Transformation.Kind.MID_LEAVE.ordinal())
+ midLeave = PrepareLeave.MidLeave.serializer.deserialize(in, version);
+ PrepareLeave.FinishLeave finishLeave = null;
+ if (next.ordinal() >= Transformation.Kind.FINISH_LEAVE.ordinal())
+ finishLeave = PrepareLeave.FinishLeave.serializer.deserialize(in, version);
+
+ return new UnbootstrapAndLeave(barrier, lockKey, next,
+ startLeave, midLeave, finishLeave);
+ }
+
+ public long serializedSize(InProgressSequence<?> t, Version version)
+ {
+ UnbootstrapAndLeave plan = (UnbootstrapAndLeave) t;
+ long size = ProgressBarrier.serializer.serializedSize(plan.barrier(), version);
+ size += LockedRanges.Key.serializer.serializedSize(plan.lockKey, version);
+
+ size += VIntCoding.computeVIntSize(plan.kind().ordinal());
+
+ if (plan.next.ordinal() >= Transformation.Kind.START_LEAVE.ordinal())
+ size += PrepareLeave.StartLeave.serializer.serializedSize(plan.startLeave, version);
+ if (plan.next.ordinal() >= Transformation.Kind.MID_LEAVE.ordinal())
+ size += PrepareLeave.StartLeave.serializer.serializedSize(plan.midLeave, version);
+ if (plan.next.ordinal() >= Transformation.Kind.FINISH_LEAVE.ordinal())
+ size += PrepareLeave.StartLeave.serializer.serializedSize(plan.finishLeave, version);
+ return size;
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/tcm/transformations/AlterSchema.java b/src/java/org/apache/cassandra/tcm/transformations/AlterSchema.java
index af96eba01c..0e0dec2bf0 100644
--- a/src/java/org/apache/cassandra/tcm/transformations/AlterSchema.java
+++ b/src/java/org/apache/cassandra/tcm/transformations/AlterSchema.java
@@ -30,7 +30,10 @@ import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.SchemaProvider;
import org.apache.cassandra.schema.SchemaTransformation;
import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.ClusterMetadataService;
import org.apache.cassandra.tcm.Transformation;
+import org.apache.cassandra.tcm.ownership.DataPlacements;
+import org.apache.cassandra.tcm.sequences.LockedRanges;
import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
import org.apache.cassandra.tcm.serialization.Version;
@@ -57,6 +60,10 @@ public class AlterSchema implements Transformation
@Override
public final Result execute(ClusterMetadata prev)
{
+ // TODO: this not necessarily should be the case, we should optimise this, just be careful not to override
+ if (!prev.lockedRanges.locked.isEmpty())
+ return new Rejected("Can't have schema changes during ring movements: " + prev.lockedRanges.locked);
+
Keyspaces newKeyspaces;
try
@@ -69,9 +76,18 @@ public class AlterSchema implements Transformation
}
DistributedSchema snapshotAfter = new DistributedSchema(newKeyspaces);
- ClusterMetadata.Transformer next = prev.transformer().with(snapshotAfter);
- return success(next);
+ // state.schema is a DistributedSchema, so doesn't include local keyspaces. If we don't explicitly include those
+ // here, their placements won't be calculated, effectively dropping them from the new versioned state
+ Keyspaces allKeyspaces = prev.schema.getKeyspaces().withAddedOrReplaced(snapshotAfter.getKeyspaces());
+
+ DataPlacements newPlacement = ClusterMetadataService.instance()
+ .placementProvider()
+ .calculatePlacements(prev.tokenMap.toRanges(), prev, allKeyspaces);
+
+ ClusterMetadata.Transformer next = prev.transformer().with(snapshotAfter).with(newPlacement);
+
+ return success(next, LockedRanges.AffectedRanges.EMPTY);
}
static class Serializer implements AsymmetricMetadataSerializer<Transformation, AlterSchema>
diff --git a/src/java/org/apache/cassandra/tcm/transformations/ApplyPlacementDeltas.java b/src/java/org/apache/cassandra/tcm/transformations/ApplyPlacementDeltas.java
new file mode 100644
index 0000000000..500a96c06d
--- /dev/null
+++ b/src/java/org/apache/cassandra/tcm/transformations/ApplyPlacementDeltas.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tcm.transformations;
+
+import java.io.IOException;
+
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.tcm.Transformation;
+import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
+import org.apache.cassandra.tcm.serialization.Version;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.sequences.LockedRanges;
+import org.apache.cassandra.tcm.membership.NodeId;
+import org.apache.cassandra.tcm.ownership.PlacementDeltas;
+
+public abstract class ApplyPlacementDeltas implements Transformation
+{
+ protected final NodeId nodeId;
+ protected final PlacementDeltas delta;
+ protected final LockedRanges.Key lockKey;
+ protected final boolean unlock;
+
+ ApplyPlacementDeltas(NodeId nodeId, PlacementDeltas delta, LockedRanges.Key lockKey, boolean unlock)
+ {
+ this.nodeId = nodeId;
+ this.delta = delta;
+ this.lockKey = lockKey;
+ this.unlock = unlock;
+ }
+
+ public abstract Kind kind();
+ public abstract ClusterMetadata.Transformer transform(ClusterMetadata prev, ClusterMetadata.Transformer transformer);
+
+ public PlacementDeltas inverseDelta()
+ {
+ return delta.invert();
+ }
+
+ public PlacementDeltas delta()
+ {
+ return delta;
+ }
+
+ @Override
+ public final Result execute(ClusterMetadata prev)
+ {
+ ClusterMetadata.Transformer next = prev.transformer();
+
+ if (!delta.isEmpty())
+ next = next.with(delta.apply(prev.placements));
+
+ next = transform(prev, next);
+
+ if (unlock)
+ next = next.with(prev.lockedRanges.unlock(lockKey));
+
+ return success(next, prev.lockedRanges.locked.get(lockKey));
+ }
+
+ @Override
+ public String toString()
+ {
+ return getClass().getName() +
+ "{" +
+ "id=" + nodeId +
+ ", delta=" + delta +
+ '}';
+ }
+
+ public NodeId nodeId()
+ {
+ return nodeId;
+ }
+
+ abstract static class SerializerBase<T extends ApplyPlacementDeltas> implements AsymmetricMetadataSerializer<Transformation, T>
+ {
+ public void serialize(Transformation t, DataOutputPlus out, Version version) throws IOException
+ {
+ ApplyPlacementDeltas change = (T) t;
+ NodeId.serializer.serialize(change.nodeId, out, version);
+ PlacementDeltas.serializer.serialize(change.delta, out, version);
+ LockedRanges.Key.serializer.serialize(change.lockKey, out, version);
+ }
+
+ public T deserialize(DataInputPlus in, Version version) throws IOException
+ {
+ NodeId nodeId = NodeId.serializer.deserialize(in, version);
+ PlacementDeltas delta = PlacementDeltas.serializer.deserialize(in, version);
+ LockedRanges.Key lockKey = LockedRanges.Key.serializer.deserialize(in, version);
+ return construct(nodeId, delta, lockKey);
+ }
+
+ public long serializedSize(Transformation t, Version version)
+ {
+ ApplyPlacementDeltas change = (T) t;
+
+ return NodeId.serializer.serializedSize(change.nodeId, version) +
+ PlacementDeltas.serializer.serializedSize(change.delta, version) +
+ LockedRanges.Key.serializer.serializedSize(change.lockKey, version);
+ }
+
+ abstract T construct(NodeId nodeId, PlacementDeltas delta, LockedRanges.Key lockKey);
+ }
+}
diff --git a/src/java/org/apache/cassandra/tcm/transformations/CancelInProgressSequence.java b/src/java/org/apache/cassandra/tcm/transformations/CancelInProgressSequence.java
new file mode 100644
index 0000000000..b85490b385
--- /dev/null
+++ b/src/java/org/apache/cassandra/tcm/transformations/CancelInProgressSequence.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tcm.transformations;
+
+import java.io.IOException;
+
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.InProgressSequence;
+import org.apache.cassandra.tcm.Transformation;
+import org.apache.cassandra.tcm.membership.NodeId;
+import org.apache.cassandra.tcm.sequences.LockedRanges;
+import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
+import org.apache.cassandra.tcm.serialization.Version;
+
+public class CancelInProgressSequence implements Transformation
+{
+ public static final Serializer serializer = new Serializer();
+
+ private final NodeId nodeId;
+
+ public CancelInProgressSequence(NodeId nodeId)
+ {
+ this.nodeId = nodeId;
+ }
+
+ @Override
+ public Kind kind()
+ {
+ return Kind.CANCEL_SEQUENCE;
+ }
+
+ @Override
+ public Result execute(ClusterMetadata prev)
+ {
+ InProgressSequence<?> sequence = prev.inProgressSequences.get(nodeId);
+ if (null == sequence)
+ return new Rejected(String.format("No in-progress sequence found for node id %s at epoch %s",
+ nodeId, prev.epoch));
+
+ // TODO unlock - requires IPS to have/provide locked ranges - or maybe this is just part of cancel?
+ // Doesn't really need to remove the sequence here, could encapsulate that in cancel() - makes renaming
+ // it more important b/c it's really a "recipe" or plan for reverting affects - maybe rename to
+ // "revert(Applied)Effects"?
+ ClusterMetadata.Transformer transformer = sequence.cancel(prev)
+ .with(prev.inProgressSequences.without(nodeId));
+ return success(transformer, LockedRanges.AffectedRanges.EMPTY);
+ }
+
+ @Override
+ public String toString()
+ {
+ return "CancelInProgressSequence{" +
+ "nodeId=" + nodeId +
+ '}';
+ }
+
+ public static final class Serializer implements AsymmetricMetadataSerializer<Transformation, CancelInProgressSequence>
+ {
+ @Override
+ public void serialize(Transformation t, DataOutputPlus out, Version version) throws IOException
+ {
+ assert t instanceof CancelInProgressSequence;
+ CancelInProgressSequence cancel = (CancelInProgressSequence) t;
+ NodeId.serializer.serialize(cancel.nodeId, out, version);
+ }
+
+ @Override
+ public CancelInProgressSequence deserialize(DataInputPlus in, Version version) throws IOException
+ {
+ NodeId nodeId = NodeId.serializer.deserialize(in, version);
+ return new CancelInProgressSequence(nodeId);
+ }
+
+ @Override
+ public long serializedSize(Transformation t, Version version)
+ {
+ assert t instanceof CancelInProgressSequence;
+ return NodeId.serializer.serializedSize(((CancelInProgressSequence)t).nodeId, version);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/tcm/transformations/CustomTransformation.java b/src/java/org/apache/cassandra/tcm/transformations/CustomTransformation.java
index 1c28cb1374..28cefd4793 100644
--- a/src/java/org/apache/cassandra/tcm/transformations/CustomTransformation.java
+++ b/src/java/org/apache/cassandra/tcm/transformations/CustomTransformation.java
@@ -19,19 +19,18 @@
package org.apache.cassandra.tcm.transformations;
import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
+import java.util.*;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.Transformation;
import org.apache.cassandra.tcm.extensions.ExtensionKey;
import org.apache.cassandra.tcm.extensions.IntValue;
import org.apache.cassandra.tcm.extensions.StringValue;
import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
import org.apache.cassandra.tcm.serialization.Version;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.sequences.LockedRanges;
public class CustomTransformation implements Transformation
{
@@ -175,7 +174,7 @@ public class CustomTransformation implements Transformation
public Result execute(ClusterMetadata prev)
{
StringValue value = StringValue.create(str);
- return success(prev.transformer().with(METADATA_KEY, value));
+ return success(prev.transformer().with(METADATA_KEY, value), LockedRanges.AffectedRanges.EMPTY);
}
public String toString()
@@ -233,7 +232,7 @@ public class CustomTransformation implements Transformation
public Result execute(ClusterMetadata prev)
{
IntValue value = IntValue.create(v);
- return success(prev.transformer().with(METADATA_KEY, value));
+ return success(prev.transformer().with(METADATA_KEY, value), LockedRanges.AffectedRanges.EMPTY);
}
public String toString()
diff --git a/src/java/org/apache/cassandra/tcm/transformations/ForceSnapshot.java b/src/java/org/apache/cassandra/tcm/transformations/ForceSnapshot.java
index 7e8288fd0d..be9fde80db 100644
--- a/src/java/org/apache/cassandra/tcm/transformations/ForceSnapshot.java
+++ b/src/java/org/apache/cassandra/tcm/transformations/ForceSnapshot.java
@@ -25,6 +25,7 @@ import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.MetadataKeys;
import org.apache.cassandra.tcm.Transformation;
+import org.apache.cassandra.tcm.sequences.LockedRanges;
import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
import org.apache.cassandra.tcm.serialization.Version;
@@ -48,9 +49,9 @@ public class ForceSnapshot implements Transformation
return Kind.FORCE_SNAPSHOT;
}
- public Result execute(ClusterMetadata clusterMetadata)
+ public Result execute(ClusterMetadata metadata)
{
- return new Success(baseState, MetadataKeys.CORE_METADATA);
+ return new Success(baseState, LockedRanges.AffectedRanges.EMPTY, MetadataKeys.CORE_METADATA);
}
public String toString()
diff --git a/src/java/org/apache/cassandra/tcm/transformations/PrepareJoin.java b/src/java/org/apache/cassandra/tcm/transformations/PrepareJoin.java
new file mode 100644
index 0000000000..12fc33eca2
--- /dev/null
+++ b/src/java/org/apache/cassandra/tcm/transformations/PrepareJoin.java
@@ -0,0 +1,307 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tcm.transformations;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.tcm.Transformation;
+import org.apache.cassandra.tcm.ClusterMetadataService;
+import org.apache.cassandra.tcm.membership.NodeState;
+import org.apache.cassandra.tcm.sequences.BootstrapAndJoin;
+import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
+import org.apache.cassandra.tcm.serialization.Version;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.ownership.DataPlacements;
+import org.apache.cassandra.tcm.sequences.LockedRanges;
+import org.apache.cassandra.tcm.membership.NodeId;
+import org.apache.cassandra.tcm.ownership.PlacementDeltas;
+import org.apache.cassandra.tcm.ownership.PlacementProvider;
+import org.apache.cassandra.tcm.ownership.PlacementTransitionPlan;
+import org.apache.cassandra.tcm.sequences.ProgressBarrier;
+
+/**
+ * Create a plan for adding a new node and bootstrapping it, then start to execute that plan.
+ * Creating the plan involves adding the joining node's tokens to the current tokenmap and generating the set of
+ * required operations to be applied to existing data placements.
+ * Specifically, this involves splitting the ranges containing the new node's tokens and adding the new node as a
+ * write replica for those ranges. After the new node has completed streaming for bootstrap, it is made a read
+ * replica for those ranges.
+ *
+ * For example, if we start with a (subset of a) ring where A has token 100, B has token 200 and C has token 300
+ * with RF=2 then both the read and write placements will contain:
+ *
+ * (0, 100] : {A,B}
+ * (100, 200] : {B,C}
+ *
+ * If we then begin bootstrap X with token 150, the first step is to range split the existing ranges in line without
+ * changing any ownership. At this point, both the read and write placements will contain:
+ *
+ * (0, 100] : {A,B}
+ * (100, 150] : {B,C}
+ * (150, 200] : {B,C}
+ *
+ * Next, the new node is added to the write groups for the ranges it is acquiring. After this step, the read placement
+ * is unchanged, while the write placement will contain:
+ *
+ * (0, 100] : {A,B,X}
+ * (100, 150] : {B,C,X}
+ * (150, 200] : {B,C}
+ *
+ * Once X completes bootstrapping, it is added to the corresponding read groups, replacing nodes which are no longer
+ * replicas for those ranges. Now both the read and write placements will contain:
+ *
+ * (0, 100] : {A,X}
+ * (100, 150] : {X,B}
+ * (150, 200] : {B,C}
+ *
+ * Currently, we do not automatically clean up unowned ranges after bootstrap, so C will still hold data for
+ * (100, 150] on disk even though it is no longer a replica for that range.
+ */
+public class PrepareJoin implements Transformation
+{
+ private static final Logger logger = LoggerFactory.getLogger(PrepareJoin.class);
+
+ public static final Serializer<PrepareJoin> serializer = new Serializer<PrepareJoin>()
+ {
+ public PrepareJoin construct(NodeId nodeId, Set<Token> tokens, PlacementProvider placementProvider, boolean joinTokenRing, boolean streamData)
+ {
+ return new PrepareJoin(nodeId, tokens, placementProvider, joinTokenRing, streamData);
+ }
+ };
+
+ protected final NodeId nodeId;
+ protected final Set<Token> tokens;
+ protected final PlacementProvider placementProvider;
+ protected final boolean joinTokenRing;
+ protected final boolean streamData;
+
+ public PrepareJoin(NodeId nodeId,
+ Set<Token> tokens,
+ PlacementProvider placementProvider,
+ boolean joinTokenRing,
+ boolean streamData)
+ {
+ this.nodeId = nodeId;
+ this.tokens = tokens;
+ this.placementProvider = placementProvider;
+ this.joinTokenRing = joinTokenRing;
+ this.streamData = streamData;
+ }
+
+ @Override
+ public Kind kind()
+ {
+ return Kind.PREPARE_JOIN;
+ }
+
+ public NodeId nodeId()
+ {
+ return nodeId;
+ }
+
+ @Override
+ public Result execute(ClusterMetadata prev)
+ {
+ ClusterMetadata.Transformer proposed = prev.transformer().proposeToken(nodeId, tokens);
+ PlacementTransitionPlan transitionPlan = placementProvider.planForJoin(prev, nodeId, tokens, prev.schema.getKeyspaces());
+
+ LockedRanges.AffectedRanges rangesToLock = transitionPlan.affectedRanges();
+ LockedRanges.Key alreadyLockedBy = prev.lockedRanges.intersects(rangesToLock);
+ if (!alreadyLockedBy.equals(LockedRanges.NOT_LOCKED))
+ {
+ return new Rejected(String.format("Rejecting this plan as it interacts with a range locked by %s (locked: %s, new: %s)",
+ alreadyLockedBy, prev.lockedRanges, rangesToLock));
+ }
+
+ LockedRanges.Key lockKey = LockedRanges.keyFor(prev.nextEpoch());
+ StartJoin startJoin = new StartJoin(nodeId, transitionPlan.addToWrites(), lockKey);
+ MidJoin midJoin = new MidJoin(nodeId, transitionPlan.moveReads(), lockKey);
+ FinishJoin finishJoin = new FinishJoin(nodeId, transitionPlan.removeFromWrites(), lockKey);
+
+ ProgressBarrier barrier = ProgressBarrier.immediate(rangesToLock.toPeers(prev.placements, prev.directory));
+ BootstrapAndJoin plan = new BootstrapAndJoin(barrier,
+ lockKey,
+ Kind.START_JOIN,
+ transitionPlan.toSplit,
+ startJoin, midJoin, finishJoin,
+ joinTokenRing,
+ streamData);
+
+ LockedRanges newLockedRanges = prev.lockedRanges.lock(lockKey, rangesToLock);
+ DataPlacements startingPlacements = transitionPlan.toSplit.apply(prev.placements);
+ proposed = proposed.with(newLockedRanges)
+ .with(startingPlacements)
+ .with(prev.inProgressSequences.with(nodeId, plan));
+
+ return success(proposed, rangesToLock);
+ }
+
+ public static abstract class Serializer<T extends PrepareJoin> implements AsymmetricMetadataSerializer<Transformation, T>
+ {
+ public void serialize(Transformation t, DataOutputPlus out, Version version) throws IOException
+ {
+ T transformation = (T) t;
+ NodeId.serializer.serialize(transformation.nodeId, out, version);
+ out.writeInt(transformation.tokens.size());
+ for(Token token : transformation.tokens)
+ Token.metadataSerializer.serialize(token, out, version);
+ out.writeBoolean(transformation.joinTokenRing);
+ out.writeBoolean(transformation.streamData);
+ }
+
+ public T deserialize(DataInputPlus in, Version version) throws IOException
+ {
+ NodeId id = NodeId.serializer.deserialize(in, version);
+ int numTokens = in.readInt();
+ Set<Token> tokens = new HashSet<>(numTokens);
+ for (int i=0;i<numTokens;i++)
+ tokens.add(Token.metadataSerializer.deserialize(in, version));
+ boolean joinTokenRing = in.readBoolean();
+ boolean streamData = in.readBoolean();
+ return construct(id, tokens, ClusterMetadataService.instance().placementProvider(), joinTokenRing, streamData);
+ }
+
+ public long serializedSize(Transformation t, Version version)
+ {
+ T transformation = (T) t;
+ long size = NodeId.serializer.serializedSize(transformation.nodeId, version);
+ size += TypeSizes.INT_SIZE;
+ for (Token token : transformation.tokens)
+ size += Token.metadataSerializer.serializedSize(token, version);
+ size += TypeSizes.BOOL_SIZE * 2;
+ return size;
+ }
+
+ public abstract T construct(NodeId nodeId,
+ Set<Token> tokens,
+ PlacementProvider placementProvider,
+ boolean joinTokenRing,
+ boolean streamData);
+
+ }
+
+ public String toString()
+ {
+ return "PrepareJoin{" +
+ "id=" + nodeId +
+ ", tokens=" + tokens +
+ '}';
+ }
+
+
+ public static class StartJoin extends ApplyPlacementDeltas
+ {
+ public static final Serializer serializer = new Serializer();
+
+ public StartJoin(NodeId nodeId, PlacementDeltas delta, LockedRanges.Key lockKey)
+ {
+ super(nodeId, delta, lockKey, false);
+ }
+
+ @Override
+ public Kind kind()
+ {
+ return Kind.START_JOIN;
+ }
+
+ @Override
+ public ClusterMetadata.Transformer transform(ClusterMetadata prev, ClusterMetadata.Transformer transformer)
+ {
+ return transformer.withNodeState(nodeId, NodeState.BOOTSTRAPPING)
+ .with(prev.inProgressSequences.with(nodeId, (plan) -> plan.advance(prev.nextEpoch(), Kind.MID_JOIN)));
+ }
+
+ public static final class Serializer extends SerializerBase<StartJoin>
+ {
+ StartJoin construct(NodeId nodeId, PlacementDeltas delta, LockedRanges.Key lockKey)
+ {
+ return new StartJoin(nodeId, delta, lockKey);
+ }
+ }
+ }
+
+ public static class MidJoin extends ApplyPlacementDeltas
+ {
+ public static final Serializer serializer = new Serializer();
+
+ public MidJoin(NodeId nodeId, PlacementDeltas delta, LockedRanges.Key lockKey)
+ {
+ super(nodeId, delta, lockKey, false);
+ }
+
+ @Override
+ public Kind kind()
+ {
+ return Kind.MID_JOIN;
+ }
+
+ @Override
+ public ClusterMetadata.Transformer transform(ClusterMetadata prev, ClusterMetadata.Transformer transformer)
+ {
+ return transformer.with(prev.inProgressSequences.with(nodeId, (plan) -> plan.advance(prev.nextEpoch(), Kind.FINISH_JOIN)));
+ }
+
+ public static final class Serializer extends SerializerBase<MidJoin>
+ {
+ MidJoin construct(NodeId nodeId, PlacementDeltas delta, LockedRanges.Key lockKey)
+ {
+ return new MidJoin(nodeId, delta, lockKey);
+ }
+ }
+ }
+
+ public static class FinishJoin extends ApplyPlacementDeltas
+ {
+ public static final Serializer serializer = new Serializer();
+
+ public FinishJoin(NodeId nodeId, PlacementDeltas delta, LockedRanges.Key unlockKey)
+ {
+ super(nodeId, delta, unlockKey, true);
+ }
+
+ @Override
+ public Kind kind()
+ {
+ return Kind.FINISH_JOIN;
+ }
+
+ public ClusterMetadata.Transformer transform(ClusterMetadata prev, ClusterMetadata.Transformer transformer)
+ {
+ return transformer.join(nodeId)
+ .with(prev.inProgressSequences.without(nodeId));
+ }
+
+ public static final class Serializer extends SerializerBase<FinishJoin>
+ {
+ FinishJoin construct(NodeId nodeId, PlacementDeltas delta, LockedRanges.Key lockKey)
+ {
+ return new FinishJoin(nodeId, delta, lockKey);
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/tcm/transformations/PrepareLeave.java b/src/java/org/apache/cassandra/tcm/transformations/PrepareLeave.java
new file mode 100644
index 0000000000..b1363d4b8c
--- /dev/null
+++ b/src/java/org/apache/cassandra/tcm/transformations/PrepareLeave.java
@@ -0,0 +1,303 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tcm.transformations;
+
+import java.io.IOException;
+import java.util.Collection;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.NetworkTopologyStrategy;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.ClusterMetadataService;
+import org.apache.cassandra.tcm.Transformation;
+import org.apache.cassandra.tcm.membership.Directory;
+import org.apache.cassandra.tcm.membership.NodeId;
+import org.apache.cassandra.tcm.membership.NodeState;
+import org.apache.cassandra.tcm.ownership.PlacementDeltas;
+import org.apache.cassandra.tcm.ownership.PlacementProvider;
+import org.apache.cassandra.tcm.ownership.PlacementTransitionPlan;
+import org.apache.cassandra.tcm.sequences.LockedRanges;
+import org.apache.cassandra.tcm.sequences.ProgressBarrier;
+import org.apache.cassandra.tcm.sequences.UnbootstrapAndLeave;
+import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
+import org.apache.cassandra.tcm.serialization.Version;
+
+public class PrepareLeave implements Transformation
+{
+ private static final Logger logger = LoggerFactory.getLogger(PrepareLeave.class);
+ public static final Serializer serializer = new Serializer();
+
+ private final NodeId nodeId;
+ private final boolean force;
+ private final PlacementProvider placementProvider;
+
+ public PrepareLeave(NodeId nodeId, boolean force, PlacementProvider placementProvider)
+ {
+ this.nodeId = nodeId;
+ this.force = force;
+ this.placementProvider = placementProvider;
+ }
+
+ @Override
+ public Kind kind()
+ {
+ return Kind.PREPARE_LEAVE;
+ }
+
+ public NodeId nodeId()
+ {
+ return nodeId;
+ }
+
+ @Override
+ public Result execute(ClusterMetadata prev)
+ {
+ ClusterMetadata proposed = prev.transformer().proposeRemoveNode(nodeId).build().metadata;
+
+ if (!force && !validateReplicationForDecommission(proposed))
+ return new Rejected("Not enough live nodes to maintain replication factor after decomission.");
+
+ if (proposed.directory.isEmpty())
+ return new Rejected("No peers registered, at least local node should be");
+
+ PlacementTransitionPlan transitionPlan = placementProvider.planForDecommission(prev,
+ nodeId,
+ prev.schema.getKeyspaces());
+
+ LockedRanges.AffectedRanges rangesToLock = transitionPlan.affectedRanges();
+ LockedRanges.Key alreadyLockedBy = prev.lockedRanges.intersects(rangesToLock);
+ if (!alreadyLockedBy.equals(LockedRanges.NOT_LOCKED))
+ {
+ return new Rejected(String.format("Rejecting this plan as it interacts with a range locked by %s (locked: %s, new: %s)",
+ alreadyLockedBy, prev.lockedRanges, rangesToLock));
+ }
+
+ PlacementDeltas startDelta = transitionPlan.addToWrites();
+ PlacementDeltas midDelta = transitionPlan.moveReads();
+ PlacementDeltas finishDelta = transitionPlan.removeFromWrites();
+
+ LockedRanges.Key unlockKey = LockedRanges.keyFor(proposed.epoch);
+ InetAddressAndPort leaving = prev.directory.endpoint(nodeId);
+
+ StartLeave start = new StartLeave(nodeId, startDelta, unlockKey);
+ MidLeave mid = new MidLeave(nodeId, midDelta, unlockKey);
+ FinishLeave leave = new FinishLeave(nodeId, finishDelta, unlockKey);
+
+ ProgressBarrier barrier = new ProgressBarrier(prev.nextEpoch(),
+ rangesToLock.toPeers(prev.placements, prev.directory),
+ false);
+ UnbootstrapAndLeave plan = new UnbootstrapAndLeave(barrier,
+ unlockKey,
+ Kind.START_LEAVE,
+ start,
+ mid,
+ leave);
+
+ // note: we throw away the state with the leaving node's tokens removed. It's only
+ // used to produce the operation plan.
+ ClusterMetadata.Transformer next = prev.transformer()
+ .with(prev.lockedRanges.lock(unlockKey, rangesToLock))
+ .with(prev.inProgressSequences.with(nodeId, plan));
+ return success(next, rangesToLock);
+ }
+
+ private boolean validateReplicationForDecommission(ClusterMetadata proposed)
+ {
+ String dc = proposed.directory.location(nodeId).datacenter;
+ int rf, numNodes;
+ for (KeyspaceMetadata ksm : proposed.schema.getKeyspaces())
+ {
+ if (ksm.replicationStrategy instanceof NetworkTopologyStrategy)
+ {
+ NetworkTopologyStrategy strategy = (NetworkTopologyStrategy) ksm.replicationStrategy;
+ rf = strategy.getReplicationFactor(dc).allReplicas;
+ numNodes = joinedNodeCount(proposed.directory, proposed.directory.allDatacenterEndpoints().get(dc));
+
+ if (numNodes <= rf)
+ {
+ logger.warn("Not enough live nodes to maintain replication factor for keyspace {}. " +
+ "Replication factor in {} is {}, live nodes = {}. " +
+ "Perform a forceful decommission to ignore.", ksm, dc, rf, numNodes);
+ return false;
+ }
+ }
+ else if (ksm.params.replication.isMeta())
+ {
+ // TODO: usually we should not allow decommissioning of CMS node
+ // from what i understand this is not necessarily decomissioning of the cms node; every node has this ks now
+ continue;
+ }
+ else
+ {
+ numNodes = joinedNodeCount(proposed.directory, proposed.directory.allAddresses());
+ rf = ksm.replicationStrategy.getReplicationFactor().allReplicas;
+ if (numNodes <= rf)
+ {
+ logger.warn("Not enough live nodes to maintain replication factor in keyspace "
+ + ksm + " (RF = " + rf + ", N = " + numNodes + ")."
+ + " Perform a forceful decommission to ignore.");
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ private static int joinedNodeCount(Directory directory, Collection<InetAddressAndPort> endpoints)
+ {
+ return (int)endpoints.stream().filter(i -> directory.peerState(i) == NodeState.JOINED).count();
+ }
+
+ public static final class Serializer implements AsymmetricMetadataSerializer<Transformation, PrepareLeave>
+ {
+ public void serialize(Transformation t, DataOutputPlus out, Version version) throws IOException
+ {
+ assert t instanceof PrepareLeave;
+ PrepareLeave transformation = (PrepareLeave) t;
+ NodeId.serializer.serialize(transformation.nodeId, out, version);
+ out.writeBoolean(transformation.force);
+ }
+
+ public PrepareLeave deserialize(DataInputPlus in, Version version) throws IOException
+ {
+ NodeId id = NodeId.serializer.deserialize(in, version);
+ boolean force = in.readBoolean();
+
+ return new PrepareLeave(id,
+ force,
+ ClusterMetadataService.instance().placementProvider());
+ }
+
+ public long serializedSize(Transformation t, Version version)
+ {
+ assert t instanceof PrepareLeave;
+ PrepareLeave transformation = (PrepareLeave) t;
+ return NodeId.serializer.serializedSize(transformation.nodeId, version)
+ + TypeSizes.sizeof(transformation.force);
+ }
+ }
+
+ public String toString()
+ {
+ return "PrepareLeave{" +
+ "id=" + nodeId +
+ ", force=" + force +
+ '}';
+ }
+
+ public static class StartLeave extends ApplyPlacementDeltas
+ {
+ public static final Serializer serializer = new Serializer();
+
+ public StartLeave(NodeId nodeId, PlacementDeltas delta, LockedRanges.Key lockKey)
+ {
+ super(nodeId, delta, lockKey, false);
+ }
+
+ @Override
+ public Kind kind()
+ {
+ return Kind.START_LEAVE;
+ }
+
+ @Override
+ public ClusterMetadata.Transformer transform(ClusterMetadata prev, ClusterMetadata.Transformer transformer)
+ {
+ return transformer
+ .with(prev.inProgressSequences.with(nodeId, (plan) -> plan.advance(prev.nextEpoch(), Kind.MID_LEAVE)))
+ .withNodeState(nodeId, NodeState.LEAVING);
+ }
+
+ public static final class Serializer extends SerializerBase<StartLeave>
+ {
+ StartLeave construct(NodeId nodeId, PlacementDeltas delta, LockedRanges.Key lockKey)
+ {
+ return new StartLeave(nodeId, delta, lockKey);
+ }
+ }
+ }
+
+ public static class MidLeave extends ApplyPlacementDeltas
+ {
+ public static final Serializer serializer = new Serializer();
+
+ public MidLeave(NodeId nodeId, PlacementDeltas delta, LockedRanges.Key lockKey)
+ {
+ super(nodeId, delta, lockKey, false);
+ }
+
+ @Override
+ public Kind kind()
+ {
+ return Kind.MID_LEAVE;
+ }
+
+ @Override
+ public ClusterMetadata.Transformer transform(ClusterMetadata prev, ClusterMetadata.Transformer transformer)
+ {
+ return transformer.with(prev.inProgressSequences.with(nodeId, (plan) -> plan.advance(prev.nextEpoch(), Kind.FINISH_LEAVE)));
+ }
+
+ public static final class Serializer extends SerializerBase<MidLeave>
+ {
+ MidLeave construct(NodeId nodeId, PlacementDeltas delta, LockedRanges.Key lockKey)
+ {
+ return new MidLeave(nodeId, delta, lockKey);
+ }
+ }
+ }
+
+ public static class FinishLeave extends ApplyPlacementDeltas
+ {
+ public static final Serializer serializer = new Serializer();
+
+ public FinishLeave(NodeId nodeId, PlacementDeltas delta, LockedRanges.Key lockKey)
+ {
+ super(nodeId, delta, lockKey, true);
+ }
+
+ @Override
+ public Kind kind()
+ {
+ return Kind.FINISH_LEAVE;
+ }
+
+ @Override
+ public ClusterMetadata.Transformer transform(ClusterMetadata prev, ClusterMetadata.Transformer transformer)
+ {
+ return transformer.left(nodeId)
+ .with(prev.inProgressSequences.without(nodeId));
+ }
+
+ public static class Serializer extends SerializerBase<FinishLeave>
+ {
+ FinishLeave construct(NodeId nodeId, PlacementDeltas delta, LockedRanges.Key lockKey)
+ {
+ return new FinishLeave(nodeId, delta, lockKey);
+ }
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/tcm/transformations/PrepareMove.java b/src/java/org/apache/cassandra/tcm/transformations/PrepareMove.java
new file mode 100644
index 0000000000..f8e178150c
--- /dev/null
+++ b/src/java/org/apache/cassandra/tcm/transformations/PrepareMove.java
@@ -0,0 +1,289 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tcm.transformations;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.ClusterMetadataService;
+import org.apache.cassandra.tcm.Transformation;
+import org.apache.cassandra.tcm.membership.NodeId;
+import org.apache.cassandra.tcm.membership.NodeState;
+import org.apache.cassandra.tcm.ownership.PlacementDeltas;
+import org.apache.cassandra.tcm.ownership.PlacementProvider;
+import org.apache.cassandra.tcm.ownership.PlacementTransitionPlan;
+import org.apache.cassandra.tcm.sequences.LockedRanges;
+import org.apache.cassandra.tcm.sequences.Move;
+import org.apache.cassandra.tcm.sequences.ProgressBarrier;
+import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
+import org.apache.cassandra.tcm.serialization.Version;
+
+public class PrepareMove implements Transformation
+{
+ public static final Serializer<PrepareMove> serializer = new Serializer<PrepareMove>();
+
+ protected final NodeId nodeId;
+ protected final Set<Token> tokens;
+ protected final PlacementProvider placementProvider;
+ protected final boolean streamData;
+
+ public PrepareMove(NodeId nodeId,
+ Set<Token> tokens,
+ PlacementProvider placementProvider,
+ boolean streamData)
+ {
+ this.nodeId = nodeId;
+ this.tokens = tokens;
+ this.placementProvider = placementProvider;
+ this.streamData = streamData;
+ }
+
+ public String toString()
+ {
+ return "PrepareMove{" +
+ "nodeId=" + nodeId +
+ ", tokens=" + tokens +
+ ", placementProvider=" + placementProvider +
+ ", streamData=" + streamData +
+ '}';
+ }
+
+ public Kind kind()
+ {
+ return Kind.PREPARE_MOVE;
+ }
+
+ public Result execute(ClusterMetadata prev)
+ {
+ PlacementTransitionPlan transitionPlan = placementProvider.planForMove(prev, nodeId, tokens, prev.schema.getKeyspaces());
+ LockedRanges.AffectedRanges rangesToLock = transitionPlan.affectedRanges();
+ LockedRanges.Key alreadyLockedBy = prev.lockedRanges.intersects(rangesToLock);
+
+ if (!alreadyLockedBy.equals(LockedRanges.NOT_LOCKED))
+ {
+ return new Rejected(String.format("Rejecting this plan as it interacts with a range locked by %s (locked: %s, new: %s)",
+ alreadyLockedBy, prev.lockedRanges, rangesToLock));
+ }
+
+ LockedRanges.Key lockKey = LockedRanges.keyFor(prev.nextEpoch());
+ StartMove startMove = new StartMove(nodeId, transitionPlan.addToWrites(), lockKey);
+ MidMove midMove = new MidMove(nodeId, transitionPlan.moveReads(), lockKey);
+ FinishMove finishMove = new FinishMove(nodeId, tokens, transitionPlan.removeFromWrites(), lockKey);
+
+ ProgressBarrier barrier = ProgressBarrier.immediate(rangesToLock.toPeers(prev.placements, prev.directory));
+ Move sequence = new Move(tokens,
+ barrier,
+ lockKey,
+ Kind.START_MOVE,
+ transitionPlan.toSplit,
+ startMove,
+ midMove,
+ finishMove,
+ false);
+
+ return success(prev.transformer()
+ .withNodeState(nodeId, NodeState.MOVING)
+ .proposeToken(nodeId, tokens)
+ .with(prev.lockedRanges.lock(lockKey, rangesToLock))
+ .with(transitionPlan.toSplit.apply(prev.placements))
+ .with(prev.inProgressSequences.with(nodeId, sequence)),
+ rangesToLock);
+ }
+
+ public static class Serializer<T extends PrepareMove> implements AsymmetricMetadataSerializer<Transformation, T>
+ {
+ public void serialize(Transformation t, DataOutputPlus out, Version version) throws IOException
+ {
+ T transformation = (T) t;
+ NodeId.serializer.serialize(transformation.nodeId, out, version);
+ out.writeInt(transformation.tokens.size());
+ for(Token token : transformation.tokens)
+ Token.metadataSerializer.serialize(token, out, version);
+ out.writeBoolean(transformation.streamData);
+ }
+
+ public T deserialize(DataInputPlus in, Version version) throws IOException
+ {
+ NodeId id = NodeId.serializer.deserialize(in, version);
+ int numTokens = in.readInt();
+ Set<Token> tokens = new HashSet<>(numTokens);
+ for (int i=0;i<numTokens;i++)
+ tokens.add(Token.metadataSerializer.deserialize(in, version));
+ boolean streamData = in.readBoolean();
+ return construct(id, tokens, ClusterMetadataService.instance().placementProvider(), streamData);
+ }
+
+ public long serializedSize(Transformation t, Version version)
+ {
+ T transformation = (T) t;
+ long size = NodeId.serializer.serializedSize(transformation.nodeId, version);
+ size += TypeSizes.INT_SIZE;
+ for (Token token : transformation.tokens)
+ size += Token.metadataSerializer.serializedSize(token, version);
+ size += TypeSizes.BOOL_SIZE;
+ return size;
+ }
+
+ public T construct(NodeId nodeId,
+ Set<Token> tokens,
+ PlacementProvider placementProvider,
+ boolean streamData)
+ {
+ return (T) new PrepareMove(nodeId, tokens, placementProvider, streamData);
+ }
+
+ }
+
+ public static class StartMove extends ApplyPlacementDeltas
+ {
+ public static final Serializer serializer = new Serializer();
+
+ StartMove(NodeId nodeId, PlacementDeltas delta, LockedRanges.Key lockKey)
+ {
+ super(nodeId, delta, lockKey, false);
+ }
+
+ @Override
+ public Kind kind()
+ {
+ return Kind.START_MOVE;
+ }
+
+ @Override
+ public ClusterMetadata.Transformer transform(ClusterMetadata prev, ClusterMetadata.Transformer transformer)
+ {
+ return transformer.with(prev.inProgressSequences.with(nodeId, (plan) -> plan.advance(prev.nextEpoch(), Kind.MID_MOVE)));
+ }
+
+ public static final class Serializer extends SerializerBase<PrepareMove.StartMove>
+ {
+ StartMove construct(NodeId nodeId, PlacementDeltas delta, LockedRanges.Key lockKey)
+ {
+ return new PrepareMove.StartMove(nodeId, delta, lockKey);
+ }
+ }
+ }
+
+ public static class MidMove extends ApplyPlacementDeltas
+ {
+ public static final Serializer serializer = new Serializer();
+
+ MidMove(NodeId nodeId, PlacementDeltas delta, LockedRanges.Key lockKey)
+ {
+ super(nodeId, delta, lockKey, false);
+ }
+
+ @Override
+ public Kind kind()
+ {
+ return Kind.MID_MOVE;
+ }
+
+ @Override
+ public ClusterMetadata.Transformer transform(ClusterMetadata prev, ClusterMetadata.Transformer transformer)
+ {
+ return transformer.with(prev.inProgressSequences.with(nodeId, (plan) -> plan.advance(prev.nextEpoch(), Kind.FINISH_MOVE)));
+ }
+
+ public static final class Serializer extends SerializerBase<PrepareMove.MidMove>
+ {
+ MidMove construct(NodeId nodeId, PlacementDeltas delta, LockedRanges.Key lockKey)
+ {
+ return new PrepareMove.MidMove(nodeId, delta, lockKey);
+ }
+ }
+ }
+
+ public static class FinishMove extends ApplyPlacementDeltas
+ {
+ public static final Serializer serializer = new Serializer();
+
+ public final Collection<Token> newTokens;
+ FinishMove(NodeId nodeId, Collection<Token> newTokens, PlacementDeltas delta, LockedRanges.Key lockKey)
+ {
+ super(nodeId, delta, lockKey, true);
+ this.newTokens = newTokens;
+ }
+
+ @Override
+ public Kind kind()
+ {
+ return Kind.FINISH_MOVE;
+ }
+
+ @Override
+ public ClusterMetadata.Transformer transform(ClusterMetadata prev, ClusterMetadata.Transformer transformer)
+ {
+ return transformer.unproposeTokens(nodeId)
+ .proposeToken(nodeId, newTokens)
+ .join(nodeId)
+ .with(prev.inProgressSequences.without(nodeId));
+ }
+
+ public static final class Serializer extends SerializerBase<PrepareMove.FinishMove>
+ {
+ @Override
+ public void serialize(Transformation t, DataOutputPlus out, Version version) throws IOException
+ {
+ super.serialize(t, out, version);
+ Collection<Token> tokens = ((FinishMove)t).newTokens;
+ out.writeUnsignedVInt32(tokens.size());
+ for (Token token : tokens)
+ Token.metadataSerializer.serialize(token, out, version);
+ }
+
+ public FinishMove deserialize(DataInputPlus in, Version version) throws IOException
+ {
+ NodeId nodeId = NodeId.serializer.deserialize(in, version);
+ PlacementDeltas delta = PlacementDeltas.serializer.deserialize(in, version);
+ LockedRanges.Key lockKey = LockedRanges.Key.serializer.deserialize(in, version);
+ int numTokens = in.readUnsignedVInt32();
+ List<Token> tokens = new ArrayList<>();
+ for (int i = 0; i < numTokens; i++)
+ tokens.add(Token.metadataSerializer.deserialize(in, version));
+
+ return new FinishMove(nodeId, tokens, delta, lockKey);
+ }
+
+ public long serializedSize(Transformation t, Version version)
+ {
+ long size = super.serializedSize(t, version);
+ Collection<Token> tokens = ((FinishMove)t).newTokens;
+ size += TypeSizes.sizeofUnsignedVInt(tokens.size());
+ for (Token token : tokens)
+ size += Token.metadataSerializer.serializedSize(token, version);
+ return size;
+ }
+
+ FinishMove construct(NodeId nodeId, PlacementDeltas delta, LockedRanges.Key lockKey)
+ {
+ throw new IllegalStateException();
+ }
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/tcm/transformations/PrepareReplace.java b/src/java/org/apache/cassandra/tcm/transformations/PrepareReplace.java
new file mode 100644
index 0000000000..fa6f5c769e
--- /dev/null
+++ b/src/java/org/apache/cassandra/tcm/transformations/PrepareReplace.java
@@ -0,0 +1,387 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tcm.transformations;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+
+import com.google.common.collect.ImmutableSet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.ClusterMetadataService;
+import org.apache.cassandra.tcm.Transformation;
+import org.apache.cassandra.tcm.membership.NodeId;
+import org.apache.cassandra.tcm.membership.NodeState;
+import org.apache.cassandra.tcm.ownership.PlacementDeltas;
+import org.apache.cassandra.tcm.ownership.PlacementProvider;
+import org.apache.cassandra.tcm.ownership.PlacementTransitionPlan;
+import org.apache.cassandra.tcm.sequences.BootstrapAndReplace;
+import org.apache.cassandra.tcm.sequences.LockedRanges;
+import org.apache.cassandra.tcm.sequences.ProgressBarrier;
+import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
+import org.apache.cassandra.tcm.serialization.Version;
+
+import static org.apache.cassandra.tcm.Transformation.Kind.START_REPLACE;
+
+public class PrepareReplace implements Transformation
+{
+ private static final Logger logger = LoggerFactory.getLogger(PrepareReplace.class);
+
+ public static final Serializer serializer = new Serializer();
+
+ private final NodeId replaced;
+ private final NodeId replacement;
+ private final PlacementProvider placementProvider;
+ private final boolean joinTokenRing;
+ private final boolean streamData;
+
+ public PrepareReplace(NodeId replaced,
+ NodeId replacement,
+ PlacementProvider placementProvider,
+ boolean joinTokenRing,
+ boolean streamData)
+ {
+ this.replaced = replaced;
+ this.replacement = replacement;
+ this.placementProvider = placementProvider;
+ this.joinTokenRing = joinTokenRing;
+ this.streamData = streamData;
+ }
+
+ @Override
+ public Kind kind()
+ {
+ return Kind.PREPARE_REPLACE;
+ }
+
+ public NodeId replacement()
+ {
+ return replacement;
+ }
+
+ @Override
+ public Result execute(ClusterMetadata prev)
+ {
+ LockedRanges.Key unlockKey = LockedRanges.keyFor(prev.nextEpoch());
+ LockedRanges lockedRanges = prev.lockedRanges;
+
+ PlacementTransitionPlan transitionPlan = placementProvider.planForReplacement(prev,
+ replaced,
+ replacement,
+ prev.schema.getKeyspaces());
+ PlacementDeltas.Builder addNewNodeToWrites = PlacementDeltas.builder();
+ PlacementDeltas.Builder addNewNodeToReads = PlacementDeltas.builder();
+ PlacementDeltas.Builder removeOldNodeFromWrites = PlacementDeltas.builder();
+
+ // Only addition of the new node to the write groups is done as a consequence of the first transformation. Adding the new
+ // node to the various read groups is deferred until the second transformation, after bootstrap. Also, track which ranges
+ // are going to be affected by this operation (i.e. which will be the "pending" ranges for the new node. If the
+ // plan is accepted those ranges will be locked to prevent other plans submitted later from interacting with the
+ // same ranges.
+ LockedRanges.AffectedRangesBuilder affectedRanges = LockedRanges.AffectedRanges.builder();
+ transitionPlan.toMaximal.forEach((replication, delta) -> {
+ delta.reads.additions.flattenValues().forEach(r -> affectedRanges.add(replication, r.range()));
+ addNewNodeToWrites.put(replication, delta.onlyWrites().onlyAdditions());
+ addNewNodeToReads.put(replication, delta.onlyReads());
+ });
+
+ transitionPlan.toFinal.forEach((replication, delta) -> {
+ delta.reads.additions.flattenValues().forEach(r -> affectedRanges.add(replication, r.range()));
+ addNewNodeToReads.put(replication, delta.onlyReads());
+ removeOldNodeFromWrites.put(replication, delta.onlyWrites().onlyRemovals());
+ });
+
+ LockedRanges.AffectedRanges rangesToLock = affectedRanges.build();
+ LockedRanges.Key alreadyLockedBy = lockedRanges.intersects(rangesToLock);
+
+ if (!alreadyLockedBy.equals(LockedRanges.NOT_LOCKED))
+ {
+ return new Rejected(String.format("Rejecting this plan as it interacts with a range locked by %s (locked: %s, new: %s)",
+ alreadyLockedBy, lockedRanges, rangesToLock));
+ }
+
+ StartReplace start = new StartReplace(replaced, replacement, addNewNodeToWrites.build(), unlockKey);
+ MidReplace mid = new MidReplace(replaced, replacement, addNewNodeToReads.build(), unlockKey);
+ FinishReplace finish = new FinishReplace(replaced, replacement, removeOldNodeFromWrites.build(), unlockKey);
+
+ Set<Token> tokens = new HashSet<>(prev.tokenMap.tokens(replaced));
+ ImmutableSet<NodeId> toWatermark = rangesToLock.toPeers(prev.placements, prev.directory);
+ ProgressBarrier barrier = new ProgressBarrier(prev.nextEpoch(), toWatermark, true);
+ BootstrapAndReplace plan = new BootstrapAndReplace(barrier,
+ unlockKey,
+ START_REPLACE,
+ tokens,
+ start, mid, finish,
+ joinTokenRing,
+ streamData);
+
+
+
+
+ LockedRanges newLockedRanges = lockedRanges.lock(unlockKey, rangesToLock);
+ ClusterMetadata.Transformer proposed = prev.transformer()
+ .with(newLockedRanges)
+ .with(prev.inProgressSequences.with(replacement(), plan));
+ logger.info("Node {} is replacing {}, tokens {}", prev.directory.endpoint(replacement), prev.directory.endpoint(replaced), prev.tokenMap.tokens(replaced));
+ return success(proposed, rangesToLock);
+ }
+
+ public String toString()
+ {
+ return "PrepareReplace{" +
+ "replaced=" + replaced+
+ ", replacement=" + replacement +
+ '}';
+ }
+
+
+ public static final class Serializer implements AsymmetricMetadataSerializer<Transformation, PrepareReplace>
+ {
+ public void serialize(Transformation t, DataOutputPlus out, Version version) throws IOException
+ {
+ assert t instanceof PrepareReplace;
+ PrepareReplace transformation = (PrepareReplace) t;
+ NodeId.serializer.serialize(transformation.replaced, out, version);
+ NodeId.serializer.serialize(transformation.replacement, out, version);
+ out.writeBoolean(transformation.joinTokenRing);
+ out.writeBoolean(transformation.streamData);
+ }
+
+ public PrepareReplace deserialize(DataInputPlus in, Version version) throws IOException
+ {
+ NodeId replaced = NodeId.serializer.deserialize(in, version);
+ NodeId replacement = NodeId.serializer.deserialize(in, version);
+ boolean joinTokenRing = in.readBoolean();
+ boolean streamData = in.readBoolean();
+ return new PrepareReplace(replaced,
+ replacement,
+ ClusterMetadataService.instance().placementProvider(),
+ joinTokenRing,
+ streamData);
+ }
+
+ public long serializedSize(Transformation t, Version version)
+ {
+ assert t instanceof PrepareReplace;
+ PrepareReplace transformation = (PrepareReplace) t;
+ return NodeId.serializer.serializedSize(transformation.replaced, version) +
+ NodeId.serializer.serializedSize(transformation.replacement, version) +
+ (TypeSizes.BOOL_SIZE * 2);
+ }
+ }
+
+ public static abstract class BaseReplaceTransformation extends ApplyPlacementDeltas
+ {
+ protected final NodeId replaced;
+
+ public BaseReplaceTransformation(NodeId replaced, NodeId replacement, PlacementDeltas delta, LockedRanges.Key unlockKey, boolean unlock)
+ {
+ super(replacement, delta, unlockKey, unlock);
+ this.replaced = replaced;
+ }
+
+ public NodeId replacement()
+ {
+ return nodeId;
+ }
+
+ public NodeId replaced()
+ {
+ return replaced;
+ }
+ }
+
+ // Analogous to ApplyPlacementDeltas.SerializerBase. The difference is that classes which extend
+ // BaseReplaceTransformation have 2 NodeIds, whereas the equivalents from PrepareJoin and PrepareLeave only have a single id
+ // TODO we can probably make the hierarchy of abstract/concrete transformations and associated serializers a bit less convoluted
+ public abstract static class SerializerBase<T extends BaseReplaceTransformation> implements AsymmetricMetadataSerializer<Transformation, T>
+ {
+ public void serialize(Transformation t, DataOutputPlus out, Version version) throws IOException
+ {
+ BaseReplaceTransformation change = (T) t;
+ NodeId.serializer.serialize(change.replaced(), out, version);
+ NodeId.serializer.serialize(change.replacement(), out, version);
+ PlacementDeltas.serializer.serialize(change.delta, out, version);
+ LockedRanges.Key.serializer.serialize(change.lockKey, out, version);
+ }
+
+ public T deserialize(DataInputPlus in, Version version) throws IOException
+ {
+ NodeId replaced = NodeId.serializer.deserialize(in, version);
+ NodeId replacement = NodeId.serializer.deserialize(in, version);
+ PlacementDeltas delta = PlacementDeltas.serializer.deserialize(in, version);
+ LockedRanges.Key lockKey = LockedRanges.Key.serializer.deserialize(in, version);
+ return construct(replaced, replacement, delta, lockKey);
+ }
+
+ public long serializedSize(Transformation t, Version version)
+ {
+ BaseReplaceTransformation change = (T) t;
+
+ return NodeId.serializer.serializedSize(change.replaced(), version) +
+ NodeId.serializer.serializedSize(change.replacement(), version) +
+ PlacementDeltas.serializer.serializedSize(change.delta, version) +
+ LockedRanges.Key.serializer.serializedSize(change.lockKey, version);
+ }
+
+ abstract T construct(NodeId replaced, NodeId replacement, PlacementDeltas delta, LockedRanges.Key lockKey);
+ }
+
+ // This is functionally identical to StartJoin. They only exist as distinct transformations for clarity.
+ public static class StartReplace extends BaseReplaceTransformation
+ {
+ public static final Serializer serializer = new Serializer();
+
+ public StartReplace(NodeId replaced, NodeId replacement, PlacementDeltas delta, LockedRanges.Key unlockKey)
+ {
+ super(replaced, replacement, delta, unlockKey, false);
+ }
+
+ @Override
+ public Kind kind()
+ {
+ return Kind.START_REPLACE;
+ }
+
+ @Override
+ public ClusterMetadata.Transformer transform(ClusterMetadata prev, ClusterMetadata.Transformer transformer)
+ {
+ return transformer.withNodeState(replacement(), NodeState.BOOTSTRAPPING)
+ .with(prev.inProgressSequences.with(nodeId, (plan) -> plan.advance(prev.nextEpoch(), Kind.MID_REPLACE)));
+ }
+
+ @Override
+ public String toString()
+ {
+ return "StartReplace{" +
+ ", delta=" + delta +
+ ", lockKey=" + lockKey +
+ ", unlock=" + unlock +
+ ", replaced=" + replaced +
+ ", replacement=" + replacement() +
+ '}';
+ }
+
+ public static final class Serializer extends PrepareReplace.SerializerBase<StartReplace>
+ {
+ @Override
+ public StartReplace construct(NodeId replaced, NodeId replacement, PlacementDeltas delta, LockedRanges.Key unlockKey)
+ {
+ return new StartReplace(replaced, replacement, delta, unlockKey);
+ }
+ }
+ }
+
+ public static class MidReplace extends BaseReplaceTransformation
+ {
+ public static final Serializer serializer = new Serializer();
+
+ public MidReplace(NodeId replaced, NodeId replacement, PlacementDeltas delta, LockedRanges.Key unlockKey)
+ {
+ super(replaced, replacement, delta, unlockKey, false);
+ }
+
+ @Override
+ public Kind kind()
+ {
+ return Kind.MID_REPLACE;
+ }
+
+ @Override
+ public ClusterMetadata.Transformer transform(ClusterMetadata prev, ClusterMetadata.Transformer transformer)
+ {
+ return transformer
+ .with(prev.inProgressSequences.with(nodeId, (plan) -> plan.advance(prev.nextEpoch(), Kind.FINISH_REPLACE)));
+ }
+
+ @Override
+ public String toString()
+ {
+ return "MidReplace{" +
+ ", delta=" + delta +
+ ", lockKey=" + lockKey +
+ ", unlock=" + unlock +
+ ", replaced=" + replaced +
+ ", replacement=" + replacement() +
+ '}';
+ }
+
+ public static final class Serializer extends PrepareReplace.SerializerBase<MidReplace>
+ {
+ @Override
+ MidReplace construct(NodeId replaced, NodeId replacement, PlacementDeltas delta, LockedRanges.Key lockKey)
+ {
+ return new MidReplace(replaced, replacement, delta, lockKey);
+ }
+ }
+ }
+
+ public static class FinishReplace extends BaseReplaceTransformation
+ {
+ public static final Serializer serializer = new Serializer();
+
+ public FinishReplace(NodeId replaced,
+ NodeId replacement,
+ PlacementDeltas delta,
+ LockedRanges.Key unlockKey)
+ {
+ super(replaced, replacement, delta, unlockKey, true);
+ }
+
+ @Override
+ public Kind kind()
+ {
+ return Kind.FINISH_REPLACE;
+ }
+
+ @Override
+ public ClusterMetadata.Transformer transform(ClusterMetadata prev, ClusterMetadata.Transformer transformer)
+ {
+ return transformer.replaced(replaced, replacement())
+ .with(prev.inProgressSequences.without(nodeId));
+ }
+
+ @Override
+ public String toString()
+ {
+ return "FinishReplace{" +
+ "replaced=" + replaced +
+ ", replacement=" + replacement() +
+ ", delta=" + delta +
+ ", unlockKey=" + lockKey +
+ '}';
+ }
+
+ public static final class Serializer extends PrepareReplace.SerializerBase<FinishReplace>
+ {
+ @Override
+ FinishReplace construct(NodeId replaced, NodeId replacement, PlacementDeltas delta, LockedRanges.Key lockKey)
+ {
+ return new FinishReplace(replaced, replacement, delta, lockKey);
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/tcm/transformations/Register.java b/src/java/org/apache/cassandra/tcm/transformations/Register.java
index fe1a5154c1..b3043f5df1 100644
--- a/src/java/org/apache/cassandra/tcm/transformations/Register.java
+++ b/src/java/org/apache/cassandra/tcm/transformations/Register.java
@@ -31,16 +31,17 @@ import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.locator.IEndpointSnitch;
+import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.ClusterMetadataService;
import org.apache.cassandra.tcm.Transformation;
-import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
-import org.apache.cassandra.tcm.serialization.Version;
-import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.membership.Directory;
import org.apache.cassandra.tcm.membership.Location;
import org.apache.cassandra.tcm.membership.NodeAddresses;
import org.apache.cassandra.tcm.membership.NodeId;
import org.apache.cassandra.tcm.membership.NodeVersion;
+import org.apache.cassandra.tcm.sequences.LockedRanges;
+import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
+import org.apache.cassandra.tcm.serialization.Version;
import org.apache.cassandra.utils.FBUtilities;
public class Register implements Transformation
@@ -77,7 +78,7 @@ public class Register implements Transformation
ClusterMetadata.Transformer next = prev.transformer()
.register(addresses, location, version);
- return success(next);
+ return success(next, LockedRanges.AffectedRanges.EMPTY);
}
public static NodeId maybeRegister()
diff --git a/src/java/org/apache/cassandra/tcm/transformations/SealPeriod.java b/src/java/org/apache/cassandra/tcm/transformations/SealPeriod.java
index 9333b12cf2..c824840bd4 100644
--- a/src/java/org/apache/cassandra/tcm/transformations/SealPeriod.java
+++ b/src/java/org/apache/cassandra/tcm/transformations/SealPeriod.java
@@ -20,13 +20,11 @@ package org.apache.cassandra.tcm.transformations;
import java.io.IOException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.Transformation;
+import org.apache.cassandra.tcm.sequences.LockedRanges;
import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
import org.apache.cassandra.tcm.serialization.Version;
@@ -37,8 +35,6 @@ import org.apache.cassandra.tcm.serialization.Version;
*/
public class SealPeriod implements Transformation
{
- private static final Logger logger = LoggerFactory.getLogger(SealPeriod.class);
-
public static final Serializer serializer = new Serializer();
public static SealPeriod instance = new SealPeriod();
@@ -57,7 +53,7 @@ public class SealPeriod implements Transformation
if (prev.lastInPeriod)
return new Rejected("Have just sealed this period");
- return success(prev.transformer(true));
+ return success(prev.transformer(true), LockedRanges.AffectedRanges.EMPTY);
}
static class Serializer implements AsymmetricMetadataSerializer<Transformation, SealPeriod>
diff --git a/src/java/org/apache/cassandra/tcm/transformations/UnsafeJoin.java b/src/java/org/apache/cassandra/tcm/transformations/UnsafeJoin.java
new file mode 100644
index 0000000000..802718ff93
--- /dev/null
+++ b/src/java/org/apache/cassandra/tcm/transformations/UnsafeJoin.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tcm.transformations;
+
+import java.util.Set;
+
+import com.google.common.collect.ImmutableSet;
+
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.ClusterMetadataService;
+import org.apache.cassandra.tcm.Epoch;
+import org.apache.cassandra.tcm.MetadataKey;
+import org.apache.cassandra.tcm.membership.NodeId;
+import org.apache.cassandra.tcm.ownership.PlacementProvider;
+import org.apache.cassandra.tcm.sequences.BootstrapAndJoin;
+import org.apache.cassandra.tcm.sequences.LockedRanges;
+
+// Unsafe Join is implemented as a two-step process to keep the replay logic consistent with other
+// transformations. In-progress sequence is going to get picked up, and executed.
+public class UnsafeJoin extends PrepareJoin
+{
+ public static final Serializer<UnsafeJoin> serializer = new Serializer<UnsafeJoin>()
+ {
+ public UnsafeJoin construct(NodeId nodeId, Set<Token> tokens, PlacementProvider placementProvider, boolean joinTokenRing, boolean streamData)
+ {
+ assert joinTokenRing;
+ assert !streamData;
+ return new UnsafeJoin(nodeId, tokens, placementProvider);
+ }
+ };
+
+ public UnsafeJoin(NodeId nodeId, Set<Token> tokens, PlacementProvider placementProvider)
+ {
+ super(nodeId, tokens, placementProvider, true, false);
+ }
+
+ public String toString()
+ {
+ return "UnsafeJoin{" +
+ "id=" + nodeId +
+ '}';
+ }
+
+ @Override
+ public Kind kind()
+ {
+ return Kind.UNSAFE_JOIN;
+ }
+
+ @Override
+ public Result execute(ClusterMetadata prev)
+ {
+ Result result = super.execute(prev);
+ if (result.isRejected())
+ return result;
+
+ ClusterMetadata metadata = result.success().metadata;
+ Epoch forceEpoch = metadata.epoch;
+
+ BootstrapAndJoin plan = (BootstrapAndJoin) metadata.inProgressSequences.get(nodeId());
+
+ Success success;
+ ImmutableSet.Builder<MetadataKey> modifiedKeys = ImmutableSet.builder();
+
+ success = plan.startJoin.execute(metadata).success();
+ metadata = success.metadata;
+ modifiedKeys.addAll(success.affectedMetadata);
+
+ success = plan.midJoin.execute(metadata).success();
+ metadata = success.metadata;
+ modifiedKeys.addAll(success.affectedMetadata);
+
+ success = plan.finishJoin.execute(metadata).success();
+ metadata = success.metadata;
+ modifiedKeys.addAll(success.affectedMetadata);
+
+ metadata = metadata.forceEpoch(forceEpoch);
+
+ return new Success(metadata, LockedRanges.AffectedRanges.EMPTY, modifiedKeys.build());
+ }
+
+ public static void unsafeJoin(NodeId nodeId, Set<Token> tokens)
+ {
+ UnsafeJoin join = new UnsafeJoin(nodeId, tokens, ClusterMetadataService.instance().placementProvider());
+ ClusterMetadataService.instance().commit(join);
+ }
+}
diff --git a/src/java/org/apache/cassandra/tcm/transformations/cms/EntireRange.java b/src/java/org/apache/cassandra/tcm/transformations/cms/EntireRange.java
index 82ba7117ab..7ec02248b5 100644
--- a/src/java/org/apache/cassandra/tcm/transformations/cms/EntireRange.java
+++ b/src/java/org/apache/cassandra/tcm/transformations/cms/EntireRange.java
@@ -24,14 +24,17 @@ import org.apache.cassandra.dht.Token;
import org.apache.cassandra.locator.EndpointsForRange;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.schema.ReplicationParams;
import org.apache.cassandra.tcm.ownership.DataPlacement;
import org.apache.cassandra.tcm.ownership.PlacementForRange;
+import org.apache.cassandra.tcm.sequences.LockedRanges;
import org.apache.cassandra.utils.FBUtilities;
public class EntireRange
{
public static final Range<Token> entireRange = new Range<>(DatabaseDescriptor.getPartitioner().getMinimumToken(),
DatabaseDescriptor.getPartitioner().getMinimumToken());
+ public static final LockedRanges.AffectedRanges affectedRanges = LockedRanges.AffectedRanges.singleton(ReplicationParams.meta(), entireRange);
public static Replica replica(InetAddressAndPort addr)
{
return new Replica(addr, entireRange, true);
diff --git a/src/java/org/apache/cassandra/tcm/transformations/cms/Initialize.java b/src/java/org/apache/cassandra/tcm/transformations/cms/Initialize.java
index ef50a72145..50e237a509 100644
--- a/src/java/org/apache/cassandra/tcm/transformations/cms/Initialize.java
+++ b/src/java/org/apache/cassandra/tcm/transformations/cms/Initialize.java
@@ -28,6 +28,8 @@ import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
import org.apache.cassandra.tcm.serialization.Version;
import org.apache.cassandra.tcm.transformations.ForceSnapshot;
+import static org.apache.cassandra.tcm.transformations.cms.EntireRange.affectedRanges;
+
public class Initialize extends ForceSnapshot
{
public static final AsymmetricMetadataSerializer<Transformation, Initialize> serializer = new AsymmetricMetadataSerializer<Transformation, Initialize>()
@@ -64,7 +66,7 @@ public class Initialize extends ForceSnapshot
{
ClusterMetadata next = baseState;
ClusterMetadata.Transformer transformer = next.transformer();
- return success(transformer);
+ return success(transformer, affectedRanges);
}
@Override
diff --git a/src/java/org/apache/cassandra/tcm/transformations/cms/PreInitialize.java b/src/java/org/apache/cassandra/tcm/transformations/cms/PreInitialize.java
index 6e71f7f4ea..08f00effdf 100644
--- a/src/java/org/apache/cassandra/tcm/transformations/cms/PreInitialize.java
+++ b/src/java/org/apache/cassandra/tcm/transformations/cms/PreInitialize.java
@@ -27,6 +27,7 @@ import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.Epoch;
import org.apache.cassandra.tcm.Transformation;
+import org.apache.cassandra.tcm.sequences.LockedRanges;
import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
import org.apache.cassandra.tcm.serialization.Version;
@@ -74,7 +75,7 @@ public class PreInitialize implements Transformation
metadata = transformed.metadata;
assert metadata.epoch.is(Epoch.FIRST) : metadata.epoch;
- return success(transformer);
+ return new Success(metadata, LockedRanges.AffectedRanges.EMPTY, transformed.modifiedKeys);
}
@Override
diff --git a/test/unit/org/apache/cassandra/tcm/StubClusterMetadataService.java b/test/unit/org/apache/cassandra/tcm/StubClusterMetadataService.java
index 03c0f7dae3..623c56cfaf 100644
--- a/test/unit/org/apache/cassandra/tcm/StubClusterMetadataService.java
+++ b/test/unit/org/apache/cassandra/tcm/StubClusterMetadataService.java
@@ -25,6 +25,7 @@ import org.apache.cassandra.tcm.log.Entry;
import org.apache.cassandra.tcm.log.LocalLog;
import org.apache.cassandra.tcm.log.LogStorage;
import org.apache.cassandra.tcm.log.Replication;
+import org.apache.cassandra.tcm.ownership.UniformRangePlacement;
public class StubClusterMetadataService extends ClusterMetadataService
{
@@ -44,7 +45,8 @@ public class StubClusterMetadataService extends ClusterMetadataService
Function<Processor, Processor> wrapProcessor,
Supplier<State> cmsStateSupplier)
{
- super(MetadataSnapshots.NO_OP,
+ super(new UniformRangePlacement(),
+ MetadataSnapshots.NO_OP,
LocalLog.asyncForTests(LogStorage.None, initial, false),
new StubProcessor(initial),
Commit.Replicator.NO_OP,
diff --git a/test/unit/org/apache/cassandra/tcm/log/LogListenerNotificationTest.java b/test/unit/org/apache/cassandra/tcm/log/LogListenerNotificationTest.java
index ef54f0fce8..3d851378b9 100644
--- a/test/unit/org/apache/cassandra/tcm/log/LogListenerNotificationTest.java
+++ b/test/unit/org/apache/cassandra/tcm/log/LogListenerNotificationTest.java
@@ -37,10 +37,14 @@ import org.apache.cassandra.tcm.MetadataKey;
import org.apache.cassandra.tcm.Transformation;
import org.apache.cassandra.tcm.extensions.IntValue;
import org.apache.cassandra.tcm.listeners.LogListener;
+import org.apache.cassandra.tcm.ownership.DataPlacements;
+import org.apache.cassandra.tcm.sequences.LockedRanges;
import org.apache.cassandra.tcm.transformations.CustomTransformation;
import org.apache.cassandra.tcm.transformations.cms.PreInitialize;
import static org.apache.cassandra.tcm.MetadataKeys.make;
+import static org.apache.cassandra.tcm.ownership.OwnershipUtils.randomPlacements;
+import static org.apache.cassandra.tcm.sequences.SequencesUtils.affectedRanges;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -58,6 +62,7 @@ public class LogListenerNotificationTest
{
long seed = System.nanoTime();
Random random = new Random(seed);
+ DataPlacements placements = randomPlacements(random);
List<Entry> input = new ArrayList<>();
Epoch epoch = Epoch.FIRST;
@@ -67,6 +72,7 @@ public class LogListenerNotificationTest
input.add(new Entry(new Entry.Id(i),
epoch,
new TestTransform(random.nextInt(),
+ affectedRanges(placements, random),
affectedMetadata(random))));
}
@@ -111,10 +117,12 @@ public class LogListenerNotificationTest
static class TestTransform extends CustomTransformation.PokeInt
{
+ private final LockedRanges.AffectedRanges ranges;
private final ImmutableSet<MetadataKey> keys;
- public TestTransform(int v, Set<MetadataKey> keys)
+ public TestTransform(int v, LockedRanges.AffectedRanges ranges, Set<MetadataKey> keys)
{
super(v);
+ this.ranges = ranges;
this.keys = ImmutableSet.copyOf(keys);
}
@@ -123,7 +131,7 @@ public class LogListenerNotificationTest
{
return new Success(prev.transformer()
.with(CustomTransformation.PokeInt.METADATA_KEY, IntValue.create(v))
- .build().metadata, keys);
+ .build().metadata, ranges, keys);
}
}
}
diff --git a/test/unit/org/apache/cassandra/tcm/sequences/SequencesUtils.java b/test/unit/org/apache/cassandra/tcm/sequences/SequencesUtils.java
index 1d7e08fd7c..39cd56bb16 100644
--- a/test/unit/org/apache/cassandra/tcm/sequences/SequencesUtils.java
+++ b/test/unit/org/apache/cassandra/tcm/sequences/SequencesUtils.java
@@ -21,9 +21,30 @@ package org.apache.cassandra.tcm.sequences;
import java.util.Random;
import org.apache.cassandra.tcm.Epoch;
+import org.apache.cassandra.tcm.ownership.DataPlacements;
public class SequencesUtils
{
+ public static LockedRanges lockedRanges(DataPlacements placements, Random random)
+ {
+ LockedRanges locked = LockedRanges.EMPTY;
+ for (int i = 0; i < random.nextInt(10) + 1; i++)
+ locked = locked.lock(LockedRanges.keyFor(epoch(random)), affectedRanges(placements, random));
+ return locked;
+ }
+
+ public static LockedRanges.AffectedRanges affectedRanges(DataPlacements placements, Random random)
+ {
+ LockedRanges.AffectedRangesBuilder affected = LockedRanges.AffectedRanges.builder();
+ placements.asMap().forEach((params, placement) -> {
+ placement.reads.replicaGroups().keySet().forEach((range) -> {
+ if (random.nextDouble() >= 0.6)
+ affected.add(params, range);
+ });
+ });
+ return affected.build();
+ }
+
public static Epoch epoch(Random random)
{
return Epoch.create(random.nextLong());
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org