You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ma...@apache.org on 2025/02/17 12:16:48 UTC
(cassandra) branch trunk updated: Make it possible to abort all kinds of multi step operations
This is an automated email from the ASF dual-hosted git repository.
marcuse pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/trunk by this push:
new ebbdd8bb81 Make it possible to abort all kinds of multi step operations
ebbdd8bb81 is described below
commit ebbdd8bb81cff1f120ae86a59a3934f26c30b6b7
Author: Marcus Eriksson <ma...@apache.org>
AuthorDate: Wed Jan 15 14:15:47 2025 +0100
Make it possible to abort all kinds of multi step operations
Patch by marcuse; reviewed by Sam Tunnicliffe for CASSANDRA-20217
---
CHANGES.txt | 1 +
.../apache/cassandra/service/StorageService.java | 14 +++++-
.../cassandra/service/StorageServiceMBean.java | 5 +-
.../tcm/sequences/SingleNodeSequences.java | 57 +++++++++++++++++-----
src/java/org/apache/cassandra/tools/NodeProbe.java | 14 +++++-
src/java/org/apache/cassandra/tools/NodeTool.java | 3 ++
.../cassandra/tools/nodetool/Decommission.java | 14 +++++-
.../org/apache/cassandra/tools/nodetool/Move.java | 24 ++++-----
.../cassandra/tools/nodetool/RemoveNode.java | 20 +++++---
.../distributed/test/FailingMoveTest.java | 44 ++++++++++++++++-
.../cassandra/distributed/test/RemoveNodeTest.java | 2 +-
.../distributed/test/ring/DecommissionTest.java | 56 +++++++++++++++++++++
12 files changed, 216 insertions(+), 38 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index c0b984e5ca..4202555d11 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
5.1
+ * Make it possible to abort all kinds of multi step operations (CASSANDRA-20217)
* Do not leak non-Java exceptions when calling snapshot operations via JMX (CASSANDRA-20335)
* Implement NOT_NULL constraint (CASSANDRA-20276)
* Improve error messages for constraints (CASSANDRA-20266)
diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java
index 5e16bae944..252f451d85 100644
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@ -3381,6 +3381,11 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
SingleNodeSequences.decommission(true, force);
}
+ public void abortDecommission(String nodeId)
+ {
+ SingleNodeSequences.abortDecommission(nodeId);
+ }
+
public void shutdownNetworking()
{
shutdownClientServers();
@@ -3461,9 +3466,9 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
}
@Override
- public void abortMove()
+ public void abortMove(String nodeId)
{
- SingleNodeSequences.abortMove();
+ SingleNodeSequences.abortMove(nodeId);
}
public String getRemovalStatus()
@@ -3533,6 +3538,11 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
SingleNodeSequences.removeNode(toRemove, force);
}
+ public void abortRemoveNode(String nodeId)
+ {
+ SingleNodeSequences.abortRemoveNode(nodeId);
+ }
+
public void assassinateEndpoint(String address)
{
try
diff --git a/src/java/org/apache/cassandra/service/StorageServiceMBean.java b/src/java/org/apache/cassandra/service/StorageServiceMBean.java
index f2b75195c1..57dfcea673 100644
--- a/src/java/org/apache/cassandra/service/StorageServiceMBean.java
+++ b/src/java/org/apache/cassandra/service/StorageServiceMBean.java
@@ -523,7 +523,7 @@ public interface StorageServiceMBean extends NotificationEmitter
* @param force Decommission even if this will reduce N to be less than RF.
*/
public void decommission(boolean force) throws InterruptedException;
-
+ public void abortDecommission(String nodeId);
/**
* Returns whether a node has failed to decommission.
*
@@ -547,7 +547,7 @@ public interface StorageServiceMBean extends NotificationEmitter
*/
public void move(String newToken) throws IOException;
public void resumeMove();
- public void abortMove();
+ public void abortMove(String nodeId);
/**
* removeToken removes token (and all data associated with
@@ -555,6 +555,7 @@ public interface StorageServiceMBean extends NotificationEmitter
*/
public void removeNode(String token);
public void removeNode(String token, boolean force);
+ public void abortRemoveNode(String nodeId);
public void assassinateEndpoint(String addr);
diff --git a/src/java/org/apache/cassandra/tcm/sequences/SingleNodeSequences.java b/src/java/org/apache/cassandra/tcm/sequences/SingleNodeSequences.java
index 7813fb1492..58c5f024f9 100644
--- a/src/java/org/apache/cassandra/tcm/sequences/SingleNodeSequences.java
+++ b/src/java/org/apache/cassandra/tcm/sequences/SingleNodeSequences.java
@@ -21,6 +21,8 @@ package org.apache.cassandra.tcm.sequences;
import java.util.Collections;
import java.util.EnumSet;
+import javax.annotation.Nullable;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -38,6 +40,7 @@ import org.apache.cassandra.tcm.transformations.PrepareLeave;
import org.apache.cassandra.tcm.transformations.PrepareMove;
import static org.apache.cassandra.service.StorageService.Mode.LEAVING;
+import static org.apache.cassandra.service.StorageService.Mode.MOVE_FAILED;
import static org.apache.cassandra.service.StorageService.Mode.NORMAL;
import static org.apache.cassandra.service.StorageService.Mode.DECOMMISSION_FAILED;
import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort;
@@ -96,6 +99,11 @@ public interface SingleNodeSequences
StorageService.instance.shutdownNetworking();
}
+ static void abortDecommission(String nodeId)
+ {
+ abortHelper(nodeId, MultiStepOperation.Kind.LEAVE, DECOMMISSION_FAILED);
+ }
+
/**
* Entrypoint to begin node removal process
*
@@ -134,6 +142,11 @@ public interface SingleNodeSequences
InProgressSequences.finishInProgressSequences(toRemove);
}
+ static void abortRemoveNode(String nodeId)
+ {
+ abortHelper(nodeId, MultiStepOperation.Kind.REMOVE, null);
+ }
+
/**
* move the node to new token or find a new token to boot to according to load
*
@@ -184,7 +197,7 @@ public interface SingleNodeSequences
logger.info(msg);
throw new IllegalStateException(msg);
}
- if (StorageService.instance.operationMode() != StorageService.Mode.MOVE_FAILED)
+ if (StorageService.instance.operationMode() != MOVE_FAILED)
{
String msg = "Can't resume a move operation unless it has failed";
logger.info(msg);
@@ -194,28 +207,48 @@ public interface SingleNodeSequences
InProgressSequences.finishInProgressSequences(self);
}
- static void abortMove()
+ static void abortMove(String nodeId)
+ {
+ abortHelper(nodeId, MultiStepOperation.Kind.MOVE, MOVE_FAILED);
+ }
+
+ /**
+ *
+ * @param nodeId node id to abort the MSO for, null for local node
+ * @param kind the expected kind of the multi step operation to abolt
+ * @param ssMode the legacy mode we want storage service to be in, null for any
+ */
+ private static void abortHelper(@Nullable String nodeId, MultiStepOperation.Kind kind, @Nullable StorageService.Mode ssMode)
{
if (ClusterMetadataService.instance().isMigrating() || ClusterMetadataService.state() == ClusterMetadataService.State.GOSSIP)
- throw new IllegalStateException("This cluster is migrating to cluster metadata, can't move until that is done.");
+ throw new IllegalStateException(String.format("This cluster is migrating to cluster metadata, can't abort %s until that is done.", kind));
ClusterMetadata metadata = ClusterMetadata.current();
- NodeId self = metadata.myNodeId();
- MultiStepOperation<?> sequence = metadata.inProgressSequences.get(self);
- if (sequence == null || sequence.kind() != MultiStepOperation.Kind.MOVE)
+ NodeId toAbort = nodeId == null ? metadata.myNodeId() : NodeId.fromString(nodeId);
+ MultiStepOperation<?> sequence = metadata.inProgressSequences.get(toAbort);
+ if (sequence == null || sequence.kind() != kind)
{
- String msg = "No move operation in progress, can't abort";
+ String msg = String.format("No %s operation in progress for %s, can't abort (%s)", kind, toAbort, sequence);
logger.info(msg);
throw new IllegalStateException(msg);
}
- if (StorageService.instance.operationMode() != StorageService.Mode.MOVE_FAILED)
+ if (toAbort.equals(metadata.myNodeId()))
+ {
+ if (ssMode != null && StorageService.instance.operationMode() != ssMode)
+ {
+ String msg = String.format("Can't abort a %s operation unless it has failed", kind);
+ logger.info(msg);
+ throw new IllegalStateException(msg);
+ }
+ StorageService.instance.clearTransientMode();
+ }
+ else if (Gossiper.instance.isAlive(metadata.directory.endpoint(toAbort)))
{
- String msg = "Can't abort a move operation unless it has failed";
+ String msg = String.format("Can't abort a %s operation for a node %s (%s) that is UP - run abortdecommission on that instance",
+ kind, toAbort, metadata.directory.endpoint(toAbort));
logger.info(msg);
throw new IllegalStateException(msg);
}
- StorageService.instance.clearTransientMode();
- ClusterMetadataService.instance().commit(new CancelInProgressSequence(self));
+ ClusterMetadataService.instance().commit(new CancelInProgressSequence(toAbort));
}
-
}
diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java
index f7ac9eff3b..5a6e8b3598 100644
--- a/src/java/org/apache/cassandra/tools/NodeProbe.java
+++ b/src/java/org/apache/cassandra/tools/NodeProbe.java
@@ -1026,6 +1026,11 @@ public class NodeProbe implements AutoCloseable
ssProxy.decommission(force);
}
+ public void abortDecommission(String nodeId)
+ {
+ ssProxy.abortDecommission(nodeId);
+ }
+
public void move(String newToken) throws IOException
{
ssProxy.move(newToken);
@@ -1036,9 +1041,9 @@ public class NodeProbe implements AutoCloseable
ssProxy.resumeMove();
}
- public void abortMove()
+ public void abortMove(String nodeId)
{
- ssProxy.abortMove();
+ ssProxy.abortMove(nodeId);
}
public void removeNode(String token)
@@ -1051,6 +1056,11 @@ public class NodeProbe implements AutoCloseable
ssProxy.removeNode(token, force);
}
+ public void abortRemoveNode(String nodeId)
+ {
+ ssProxy.abortRemoveNode(nodeId);
+ }
+
public String getRemovalStatus(boolean withPort)
{
return withPort ? ssProxy.getRemovalStatusWithPort() : ssProxy.getRemovalStatus();
diff --git a/src/java/org/apache/cassandra/tools/NodeTool.java b/src/java/org/apache/cassandra/tools/NodeTool.java
index bd1e302ba0..49be441b89 100644
--- a/src/java/org/apache/cassandra/tools/NodeTool.java
+++ b/src/java/org/apache/cassandra/tools/NodeTool.java
@@ -106,6 +106,7 @@ public class NodeTool
CompactionStats.class,
DataPaths.class,
Decommission.class,
+ Decommission.Abort.class,
DescribeCluster.class,
DescribeRing.class,
DisableAuditLog.class,
@@ -172,6 +173,7 @@ public class NodeTool
ListPendingHints.class,
ListSnapshots.class,
Move.class,
+ Move.Abort.class,
NetStats.class,
PauseHandoff.class,
ProfileLoad.class,
@@ -188,6 +190,7 @@ public class NodeTool
ReloadTriggers.class,
RelocateSSTables.class,
RemoveNode.class,
+ RemoveNode.Abort.class,
Repair.class,
ReplayBatchlog.class,
ResetFullQueryLog.class,
diff --git a/src/java/org/apache/cassandra/tools/nodetool/Decommission.java b/src/java/org/apache/cassandra/tools/nodetool/Decommission.java
index de70932c53..2c326c9b29 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/Decommission.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/Decommission.java
@@ -26,7 +26,6 @@ import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
@Command(name = "decommission", description = "Decommission the *node I am connecting to*")
public class Decommission extends NodeToolCmd
{
-
@Option(title = "force",
name = {"-f", "--force"},
description = "Force decommission of this node even when it reduces the number of replicas to below configured RF")
@@ -56,4 +55,17 @@ public class Decommission extends NodeToolCmd
throw new IllegalStateException("Unsupported operation: " + e.getMessage(), e);
}
}
+
+ @Command(name = "abortdecommission", description = "Abort an ongoing, failed decommission")
+ public static class Abort extends NodeToolCmd
+ {
+ @Option(title = "node id", name = "--node")
+ private String nodeId;
+
+ @Override
+ protected void execute(NodeProbe probe)
+ {
+ probe.abortDecommission(nodeId);
+ }
+ }
}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/Move.java b/src/java/org/apache/cassandra/tools/nodetool/Move.java
index 87c085b86b..ef05bf89b6 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/Move.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/Move.java
@@ -36,9 +36,6 @@ public class Move extends NodeToolCmd
@Option(title = "Resume an ongoing move operation", name = "--resume")
private boolean resume;
- @Option(title = "Abort an ongoing move operation", name = "--abort")
- private boolean abort;
-
@Override
public void execute(NodeProbe probe)
{
@@ -46,20 +43,12 @@ public class Move extends NodeToolCmd
{
if (!newToken.isEmpty())
{
- if (resume || abort)
- throw new IllegalArgumentException("Can't give both a token and --resume/--abort");
-
probe.move(newToken);
}
else
{
- if (abort && resume)
- throw new IllegalArgumentException("Can't both resume and abort");
-
if (resume)
probe.resumeMove();
- else if (abort)
- probe.abortMove();
else
throw new IllegalArgumentException("Need to give either a token for a new move operation, or --resume/--abort for an existing one");
}
@@ -68,4 +57,17 @@ public class Move extends NodeToolCmd
throw new RuntimeException("Error during moving node", e);
}
}
+
+ @Command(name = "abortmove", description = "Abort a failed move operation for this or a remote node")
+ public static class Abort extends NodeToolCmd
+ {
+ @Option(title = "node id", name = "--node")
+ private String nodeId;
+
+ @Override
+ public void execute(NodeProbe probe)
+ {
+ probe.abortMove(nodeId);
+ }
+ }
}
diff --git a/src/java/org/apache/cassandra/tools/nodetool/RemoveNode.java b/src/java/org/apache/cassandra/tools/nodetool/RemoveNode.java
index 89912ec640..94b3262058 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/RemoveNode.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/RemoveNode.java
@@ -22,13 +22,14 @@ import java.util.List;
import io.airlift.airline.Arguments;
import io.airlift.airline.Command;
+import io.airlift.airline.Option;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
@Command(name = "removenode", description = "Show status of current node removal, abort removal or remove provided ID")
public class RemoveNode extends NodeToolCmd
{
- @Arguments(title = "remove_operation", usage = "<status>|<abort> <ID>|<ID>|<ID> --force", description = "Show status of current node removal, abort removal, or remove provided ID", required = true)
+ @Arguments(title = "remove_operation", usage = "<status>|<ID>|<ID> --force", description = "Show status of current node removal, or remove provided ID", required = true)
private List<String> removeOperation = null;
@Override
@@ -41,15 +42,22 @@ public class RemoveNode extends NodeToolCmd
break;
case "force":
throw new IllegalArgumentException("Can't force a nodetool removenode. Instead abort the ongoing removenode and retry.");
- case "abort":
- if (removeOperation.size() < 2)
- probe.output().err.print("Abort requires the node id to abort the removal for.");
- probe.getCMSOperationsProxy().cancelInProgressSequences(removeOperation.get(1), "REMOVE");
- break;
default:
boolean force = removeOperation.size() > 1 && removeOperation.get(1).equals("--force");
probe.removeNode(removeOperation.get(0), force);
break;
}
}
+
+ @Command(name = "abortremovenode", description = "Abort a removenode command")
+ public static class Abort extends NodeToolCmd
+ {
+ @Option(title = "node id", name="--node", description = "The node being removed", required = true)
+ private String nodeId;
+
+ public void execute(NodeProbe probe)
+ {
+ probe.abortRemoveNode(nodeId);
+ }
+ }
}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/FailingMoveTest.java b/test/distributed/org/apache/cassandra/distributed/test/FailingMoveTest.java
index ac60f90cd3..65e0c28c3a 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/FailingMoveTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/FailingMoveTest.java
@@ -20,8 +20,11 @@ package org.apache.cassandra.distributed.test;
import java.io.IOException;
import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
+import com.google.common.util.concurrent.Uninterruptibles;
import org.junit.Test;
import net.bytebuddy.ByteBuddy;
@@ -32,6 +35,7 @@ import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.distributed.api.Feature;
import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.streaming.StreamPlan;
import org.apache.cassandra.streaming.StreamResultFuture;
@@ -41,6 +45,7 @@ import org.apache.cassandra.tcm.membership.NodeId;
import static net.bytebuddy.matcher.ElementMatchers.named;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
public class FailingMoveTest extends TestBaseImpl
{
@@ -94,12 +99,49 @@ public class FailingMoveTest extends TestBaseImpl
BB.shouldFail.set(false);
});
- cluster.get(3).nodetoolResult("move", "--abort").asserts().success();
+ cluster.get(3).nodetoolResult("abortmove").asserts().success();
cluster.get(3).runOnInstance(() -> assertEquals(StorageService.Mode.NORMAL, StorageService.instance.operationMode()));
assertNotEquals(moveToToken, getToken(cluster.get(3)));
}
}
+ @Test
+ public void testAbortMoveRemote() throws IOException, ExecutionException, InterruptedException
+ {
+ try (Cluster cluster = init(Cluster.build(3)
+ .withoutVNodes()
+ .withConfig(c -> c.with(Feature.GOSSIP, Feature.NETWORK))
+ .withInstanceInitializer(BB::install)
+ .start()))
+ {
+ cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl(id int primary key);"));
+ for (int i=0; i<30; i++)
+ cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.tbl (id) VALUES (?)"),
+ ConsistencyLevel.ALL, i);
+ String oldToken = getToken(cluster.get(3));
+ String moveToToken = "2305843009213693949";
+ assertNotEquals(oldToken, moveToToken);
+ cluster.get(3).nodetoolResult("move", moveToToken).asserts().failure();
+ int nodeId = cluster.get(3).callOnInstance(() -> {
+ assertEquals(StorageService.Mode.MOVE_FAILED, StorageService.instance.operationMode());
+ BB.shouldFail.set(false);
+ return ClusterMetadata.current().myNodeId().id();
+ });
+ cluster.get(3).shutdown().get();
+ cluster.get(2).runOnInstance(() -> {
+ while (Gossiper.instance.isAlive(ClusterMetadata.current().directory.endpoint(new NodeId(nodeId))))
+ Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
+ });
+ cluster.get(2).nodetoolResult("abortmove", "--node", String.valueOf(nodeId)).asserts().success();
+ cluster.get(3).startup();
+ assertNotEquals(moveToToken, getToken(cluster.get(3)));
+ cluster.get(3).runOnInstance(() -> {
+ assertEquals(StorageService.Mode.NORMAL, StorageService.instance.operationMode());
+ assertTrue(ClusterMetadata.current().inProgressSequences.isEmpty());
+ });
+ }
+ }
+
private String getToken(IInvokableInstance instance)
{
return instance.callsOnInstance(() -> {
diff --git a/test/distributed/org/apache/cassandra/distributed/test/RemoveNodeTest.java b/test/distributed/org/apache/cassandra/distributed/test/RemoveNodeTest.java
index 2710fdc832..cb1514b2c8 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/RemoveNodeTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/RemoveNodeTest.java
@@ -79,7 +79,7 @@ public class RemoveNodeTest extends TestBaseImpl
// Now abort the removal. This should succeed in committing a cancellation of the removal sequence before
// it can be completed, as non-CMS instance is still paused.
- cmsInstance.nodetoolResult("removenode", "abort", nodeId).asserts().success();
+ cmsInstance.nodetoolResult("abortremovenode", "--node", nodeId).asserts().success();
// Resume processing on the non-CMS instance. It will enact the MID_LEAVE step followed by the cancellation
// of the removal process.
diff --git a/test/distributed/org/apache/cassandra/distributed/test/ring/DecommissionTest.java b/test/distributed/org/apache/cassandra/distributed/test/ring/DecommissionTest.java
index 9826d013d5..6f67dac03b 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/ring/DecommissionTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/ring/DecommissionTest.java
@@ -22,9 +22,12 @@ import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import javax.annotation.Nullable;
+import com.google.common.util.concurrent.Uninterruptibles;
import org.junit.Test;
import net.bytebuddy.ByteBuddy;
@@ -37,9 +40,12 @@ import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.distributed.api.NodeToolResult;
import org.apache.cassandra.distributed.shared.ClusterUtils;
import org.apache.cassandra.distributed.test.TestBaseImpl;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.streaming.StreamSession;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.ClusterMetadataService;
+import org.apache.cassandra.tcm.membership.NodeId;
import org.apache.cassandra.tcm.membership.NodeVersion;
import org.apache.cassandra.tcm.serialization.Version;
import org.apache.cassandra.tcm.transformations.Startup;
@@ -94,6 +100,56 @@ public class DecommissionTest extends TestBaseImpl
}
}
+ @Test
+ public void testAbortDecom() throws IOException
+ {
+ try (Cluster cluster = builder().withNodes(3)
+ .withConfig(config -> config.with(NETWORK, GOSSIP))
+ .withInstanceInitializer(BB::install)
+ .start())
+ {
+ populate(cluster, 0, 100, 1, 2, ConsistencyLevel.QUORUM);
+ cluster.get(2).nodetoolResult("decommission", "--force").asserts().failure();
+ cluster.get(2).nodetoolResult("abortdecommission").asserts().success();
+ cluster.get(2).runOnInstance(() -> {
+ assertEquals(StorageService.Mode.NORMAL, StorageService.instance.operationMode());
+ assertTrue(ClusterMetadata.current().inProgressSequences.isEmpty());
+ });
+ cluster.get(2).nodetoolResult("decommission", "--force").asserts().success();
+ }
+ }
+
+ @Test
+ public void testAbortDecomRemote() throws IOException, ExecutionException, InterruptedException
+ {
+ try (Cluster cluster = builder().withNodes(3)
+ .withConfig(config -> config.with(NETWORK, GOSSIP))
+ .withInstanceInitializer(BB::install)
+ .start())
+ {
+ populate(cluster, 0, 100, 1, 2, ConsistencyLevel.QUORUM);
+ int nodeId = cluster.get(2).callOnInstance(() -> {
+ return ClusterMetadata.current().myNodeId().id();
+ });
+ cluster.get(2).nodetoolResult("decommission", "--force").asserts().failure();
+ cluster.get(2).shutdown().get();
+ cluster.get(3).runOnInstance(() -> {
+ while (Gossiper.instance.isAlive(ClusterMetadata.current().directory.endpoint(new NodeId(nodeId))))
+ Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
+ });
+ cluster.get(3).nodetoolResult("abortdecommission", "--node", String.valueOf(nodeId)).asserts().success();
+ cluster.get(2).startup();
+ cluster.get(2).runOnInstance(() -> {
+ assertEquals(StorageService.Mode.NORMAL, StorageService.instance.operationMode());
+ assertTrue(ClusterMetadata.current().inProgressSequences.isEmpty());
+ });
+ cluster.get(2).runOnInstance(() -> {
+ BB.first.set(true);
+ });
+ cluster.get(2).nodetoolResult("decommission", "--force").asserts().success();
+ }
+ }
+
@Test
public void testDecomDirectoryMinMaxVersions() throws IOException {
try (Cluster cluster = builder()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org