You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by br...@apache.org on 2021/04/29 18:21:21 UTC
[cassandra] branch trunk updated: Fix fat client removal race,
show seed list when unable to contact any seeds and warn when empty
This is an automated email from the ASF dual-hosted git repository.
brandonwilliams pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/trunk by this push:
new 3a61006 Fix fat client removal race, show seed list when unable to contact any seeds and warn when empty
3a61006 is described below
commit 3a610066eb3df5603d8a4d0fd491e0233c708eb5
Author: Brandon Williams <br...@apache.org>
AuthorDate: Thu Apr 29 13:19:38 2021 -0500
Fix fat client removal race, show seed list when unable to contact any
seeds and warn when empty
Patch by brandonwilliams; reviewed by David Capwell for CASSANDRA-16238
---
CHANGES.txt | 1 +
src/java/org/apache/cassandra/gms/Gossiper.java | 10 ++++++++++
src/java/org/apache/cassandra/service/StorageService.java | 2 +-
test/unit/org/apache/cassandra/gms/ShadowRoundTest.java | 4 ++--
4 files changed, 14 insertions(+), 3 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 21d3efe..c4f4d09 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
4.0-rc2
+ * Fix race in fat client removal (CASSANDRA-16238)
* Test org.apache.cassandra.net.AsyncPromiseTest FAILED (CASSANDRA-16596)
Merged from 3.11:
* Nodetool garbagecollect should retain SSTableLevel for LCS (CASSANDRA-16634)
diff --git a/src/java/org/apache/cassandra/gms/Gossiper.java b/src/java/org/apache/cassandra/gms/Gossiper.java
index e54fbc0..caacb65 100644
--- a/src/java/org/apache/cassandra/gms/Gossiper.java
+++ b/src/java/org/apache/cassandra/gms/Gossiper.java
@@ -577,6 +577,8 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
buildSeedsList();
seeds.remove(endpoint);
logger.info("removed {} from seeds, updated seeds list = {}", endpoint, seeds);
+ if (seeds.isEmpty())
+ logger.warn("Seeds list is now empty!");
}
liveEndpoints.remove(endpoint);
@@ -994,6 +996,14 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
{
logger.info("FatClient {} has been silent for {}ms, removing from gossip", endpoint, fatClientTimeout);
runInGossipStageBlocking(() -> {
+ if (!isGossipOnlyMember(endpoint))
+ {
+ // updating gossip and token metadata are not atomic, but rely on the single threaded gossip stage
+ // since status checks are done outside the gossip stage, need to confirm the state of the endpoint
+ // to make sure that the previous read data was correct
+ logger.info("Race condition marking {} as a FatClient; ignoring", endpoint);
+ return;
+ }
removeEndpoint(endpoint); // will put it in justRemovedEndpoints to respect quarantine delay
evictFromMembership(endpoint); // can get rid of the state immediately
});
diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java
index 367714d..313949d 100644
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@ -1746,7 +1746,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
SystemKeyspace.removeEndpoint(DatabaseDescriptor.getReplaceAddress());
}
if (!Gossiper.instance.seenAnySeed())
- throw new IllegalStateException("Unable to contact any seeds!");
+ throw new IllegalStateException("Unable to contact any seeds: " + Gossiper.instance.getSeeds());
if (Boolean.getBoolean("cassandra.reset_bootstrap_progress"))
{
diff --git a/test/unit/org/apache/cassandra/gms/ShadowRoundTest.java b/test/unit/org/apache/cassandra/gms/ShadowRoundTest.java
index 9037472..e4faa63 100644
--- a/test/unit/org/apache/cassandra/gms/ShadowRoundTest.java
+++ b/test/unit/org/apache/cassandra/gms/ShadowRoundTest.java
@@ -43,7 +43,6 @@ import org.apache.cassandra.locator.IEndpointSnitch;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.PropertyFileSnitch;
import org.apache.cassandra.net.Message;
-import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.net.MockMessagingService;
import org.apache.cassandra.net.MockMessagingSpy;
import org.apache.cassandra.net.Verb;
@@ -51,6 +50,7 @@ import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.FBUtilities;
import static org.apache.cassandra.net.MockMessagingService.verb;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -117,7 +117,7 @@ public class ShadowRoundTest
}
catch (Exception e)
{
- assertEquals("Unable to contact any seeds!", e.getMessage());
+ assertThat(e.getMessage()).startsWith("Unable to contact any seeds");
}
// we expect one SYN for each seed during shadow round + additional SYNs after gossiper has been enabled
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org