You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ja...@apache.org on 2018/01/31 00:33:07 UTC
cassandra git commit: Add nodetool getseeds and reloadseeds commands
Repository: cassandra
Updated Branches:
refs/heads/trunk 69db2359e -> bfecdf520
Add nodetool getseeds and reloadseeds commands
patch by Samuel Fink; reviewed by jasobrown for CASSANDRA-14190
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/bfecdf52
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/bfecdf52
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/bfecdf52
Branch: refs/heads/trunk
Commit: bfecdf52054a4da472af22b0c35c5db5f1132bbc
Parents: 69db235
Author: Samuel Fink <fs...@netapp.com>
Authored: Wed Jan 24 13:28:54 2018 -0500
Committer: Jason Brown <ja...@gmail.com>
Committed: Tue Jan 30 16:32:09 2018 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/config/DatabaseDescriptor.java | 10 ++
src/java/org/apache/cassandra/gms/Gossiper.java | 83 +++++++++--
.../org/apache/cassandra/gms/GossiperMBean.java | 5 +
.../org/apache/cassandra/tools/NodeProbe.java | 10 ++
.../org/apache/cassandra/tools/NodeTool.java | 2 +
.../cassandra/tools/nodetool/GetSeeds.java | 44 ++++++
.../cassandra/tools/nodetool/ReloadSeeds.java | 47 +++++++
.../org/apache/cassandra/gms/GossiperTest.java | 140 ++++++++++++++++++-
9 files changed, 332 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/bfecdf52/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index e28ffd9..a2e3654 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
4.0
+ * Non-disruptive seed node list reload (CASSANDRA-14190)
* Nodetool tablehistograms to print statics for all the tables (CASSANDRA-14185)
* Migrate dtests to use pytest and python3 (CASSANDRA-14134)
* Allow storage port to be configurable per node (CASSANDRA-7544)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/bfecdf52/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index 9012e3a..8e831cf 100644
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@ -1661,6 +1661,16 @@ public class DatabaseDescriptor
return ImmutableSet.<InetAddressAndPort>builder().addAll(seedProvider.getSeeds()).build();
}
+ public static SeedProvider getSeedProvider()
+ {
+ return seedProvider;
+ }
+
+ public static void setSeedProvider(SeedProvider newSeedProvider)
+ {
+ seedProvider = newSeedProvider;
+ }
+
public static InetAddress getListenAddress()
{
return listenAddress;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/bfecdf52/src/java/org/apache/cassandra/gms/Gossiper.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/gms/Gossiper.java b/src/java/org/apache/cassandra/gms/Gossiper.java
index eb6c500..a4e46f2 100644
--- a/src/java/org/apache/cassandra/gms/Gossiper.java
+++ b/src/java/org/apache/cassandra/gms/Gossiper.java
@@ -35,6 +35,7 @@ import com.google.common.collect.ImmutableMap;
import com.google.common.util.concurrent.Uninterruptibles;
import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.SeedProvider;
import org.apache.cassandra.utils.CassandraVersion;
import org.apache.cassandra.utils.Pair;
import org.slf4j.Logger;
@@ -89,7 +90,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
public final static int intervalInMillis = 1000;
public final static int QUARANTINE_DELAY = StorageService.RING_DELAY * 2;
private static final Logger logger = LoggerFactory.getLogger(Gossiper.class);
- public static final Gossiper instance = new Gossiper();
+ public static final Gossiper instance = new Gossiper(true);
// Timestamp to prevent processing any in-flight messages for we've not send any SYN yet, see CASSANDRA-12653.
volatile long firstSynSendAt = 0L;
@@ -199,7 +200,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
}
}
- private Gossiper()
+ Gossiper(boolean registerJmx)
{
// half of QUARATINE_DELAY, to ensure justRemovedEndpoints has enough leeway to prevent re-gossip
fatClientTimeout = (QUARANTINE_DELAY / 2);
@@ -207,14 +208,17 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
FailureDetector.instance.registerFailureDetectionEventListener(this);
// Register this instance with JMX
- try
- {
- MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
- mbs.registerMBean(this, new ObjectName(MBEAN_NAME));
- }
- catch (Exception e)
+ if (registerJmx)
{
- throw new RuntimeException(e);
+ try
+ {
+ MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+ mbs.registerMBean(this, new ObjectName(MBEAN_NAME));
+ }
+ catch (Exception e)
+ {
+ throw new RuntimeException(e);
+ }
}
}
@@ -1468,6 +1472,67 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
}
}
+ /**
+ * JMX interface for triggering an update of the seed node list.
+ */
+ public List<String> reloadSeeds()
+ {
+ logger.trace("Triggering reload of seed node list");
+
+ // Get the new set in the same that buildSeedsList does
+ Set<InetAddressAndPort> tmp = new HashSet<>();
+ try
+ {
+ for (InetAddressAndPort seed : DatabaseDescriptor.getSeeds())
+ {
+ if (seed.equals(FBUtilities.getBroadcastAddressAndPort()))
+ continue;
+ tmp.add(seed);
+ }
+ }
+ // If using the SimpleSeedProvider invalid yaml added to the config since startup could
+ // cause this to throw. Additionally, third party seed providers may throw exceptions.
+ // Handle the error and return a null to indicate that there was a problem.
+ catch (Throwable e)
+ {
+ JVMStabilityInspector.inspectThrowable(e);
+ logger.warn("Error while getting seed node list: {}", e.getLocalizedMessage());
+ return null;
+ }
+
+ if (tmp.size() == 0)
+ {
+ logger.trace("New seed node list is empty. Not updating seed list.");
+ return getSeeds();
+ }
+
+ if (tmp.equals(seeds))
+ {
+ logger.trace("New seed node list matches the existing list.");
+ return getSeeds();
+ }
+
+ // Add the new entries
+ seeds.addAll(tmp);
+ // Remove the old entries
+ seeds.retainAll(tmp);
+ logger.trace("New seed node list after reload {}", seeds);
+ return getSeeds();
+ }
+
+ /**
+ * JMX endpoint for getting the list of seeds from the node
+ */
+ public List<String> getSeeds()
+ {
+ List<String> seedList = new ArrayList<String>();
+ for (InetAddressAndPort seed : seeds)
+ {
+ seedList.add(seed.toString());
+ }
+ return seedList;
+ }
+
// initialize local HB state if needed, i.e., if gossiper has never been started before.
public void maybeInitializeLocalState(int generationNbr)
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/bfecdf52/src/java/org/apache/cassandra/gms/GossiperMBean.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/gms/GossiperMBean.java b/src/java/org/apache/cassandra/gms/GossiperMBean.java
index c4b244c..1b116e1 100644
--- a/src/java/org/apache/cassandra/gms/GossiperMBean.java
+++ b/src/java/org/apache/cassandra/gms/GossiperMBean.java
@@ -18,6 +18,7 @@
package org.apache.cassandra.gms;
import java.net.UnknownHostException;
+import java.util.List;
public interface GossiperMBean
{
@@ -29,4 +30,8 @@ public interface GossiperMBean
public void assassinateEndpoint(String address) throws UnknownHostException;
+ public List<String> reloadSeeds();
+
+ public List<String> getSeeds();
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/cassandra/blob/bfecdf52/src/java/org/apache/cassandra/tools/NodeProbe.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java
index d330ed4..ec8f7ba 100644
--- a/src/java/org/apache/cassandra/tools/NodeProbe.java
+++ b/src/java/org/apache/cassandra/tools/NodeProbe.java
@@ -711,6 +711,16 @@ public class NodeProbe implements AutoCloseable
gossProxy.assassinateEndpoint(address);
}
+ public List<String> reloadSeeds()
+ {
+ return gossProxy.reloadSeeds();
+ }
+
+ public List<String> getSeeds()
+ {
+ return gossProxy.getSeeds();
+ }
+
/**
* Set the compaction threshold
*
http://git-wip-us.apache.org/repos/asf/cassandra/blob/bfecdf52/src/java/org/apache/cassandra/tools/NodeTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/NodeTool.java b/src/java/org/apache/cassandra/tools/NodeTool.java
index d707499..f7b7f76 100644
--- a/src/java/org/apache/cassandra/tools/NodeTool.java
+++ b/src/java/org/apache/cassandra/tools/NodeTool.java
@@ -87,6 +87,7 @@ public class NodeTool
GetTraceProbability.class,
GetInterDCStreamThroughput.class,
GetEndpoints.class,
+ GetSeeds.class,
GetSSTables.class,
GetMaxHintWindow.class,
GossipInfo.class,
@@ -102,6 +103,7 @@ public class NodeTool
Refresh.class,
RemoveNode.class,
Assassinate.class,
+ ReloadSeeds.class,
ResetFullQueryLog.class,
Repair.class,
RepairAdmin.class,
http://git-wip-us.apache.org/repos/asf/cassandra/blob/bfecdf52/src/java/org/apache/cassandra/tools/nodetool/GetSeeds.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/nodetool/GetSeeds.java b/src/java/org/apache/cassandra/tools/nodetool/GetSeeds.java
new file mode 100644
index 0000000..207363c
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/GetSeeds.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.tools.nodetool;
+
+import java.util.List;
+
+import io.airlift.airline.Command;
+
+import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
+
+@Command(name = "getseeds", description = "Get the currently in use seed node IP list excluding the node IP")
+public class GetSeeds extends NodeToolCmd
+{
+ @Override
+ public void execute(NodeProbe probe)
+ {
+ List<String> seedList = probe.getSeeds();
+ if (seedList.isEmpty())
+ {
+ System.out.println("Seed node list does not contain any remote node IPs");
+ }
+ else
+ {
+ System.out.println("Current list of seed node IPs, excluding the current node's IP: " + String.join(" ", seedList));
+ }
+
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/cassandra/blob/bfecdf52/src/java/org/apache/cassandra/tools/nodetool/ReloadSeeds.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/nodetool/ReloadSeeds.java b/src/java/org/apache/cassandra/tools/nodetool/ReloadSeeds.java
new file mode 100644
index 0000000..b9682cf
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/ReloadSeeds.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.tools.nodetool;
+
+import java.util.List;
+
+import io.airlift.airline.Command;
+
+import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
+
+@Command(name = "reloadseeds", description = "Reload the seed node list from the seed node provider")
+public class ReloadSeeds extends NodeToolCmd
+{
+ @Override
+ public void execute(NodeProbe probe)
+ {
+ List<String> seedList = probe.reloadSeeds();
+ if (seedList == null)
+ {
+ System.out.println("Failed to reload the seed node list.");
+ }
+ else if (seedList.isEmpty())
+ {
+ System.out.println("Seed node list does not contain any remote node IPs");
+ }
+ else
+ {
+ System.out.println("Updated seed node IP list, excluding the current node's IP: " + String.join(" ", seedList));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/bfecdf52/test/unit/org/apache/cassandra/gms/GossiperTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/gms/GossiperTest.java b/test/unit/org/apache/cassandra/gms/GossiperTest.java
index 8c65cb4..b856983 100644
--- a/test/unit/org/apache/cassandra/gms/GossiperTest.java
+++ b/test/unit/org/apache/cassandra/gms/GossiperTest.java
@@ -18,12 +18,16 @@
package org.apache.cassandra.gms;
+import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import com.google.common.collect.ImmutableMap;
+import com.google.common.net.InetAddresses;
+import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -33,6 +37,7 @@ import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.RandomPartitioner;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.SeedProvider;
import org.apache.cassandra.locator.TokenMetadata;
import org.apache.cassandra.service.StorageService;
@@ -44,6 +49,7 @@ public class GossiperTest
{
DatabaseDescriptor.daemonInitialization();
}
+
static final IPartitioner partitioner = new RandomPartitioner();
StorageService ss = StorageService.instance;
TokenMetadata tmd = StorageService.instance.getTokenMetadata();
@@ -52,11 +58,20 @@ public class GossiperTest
List<InetAddressAndPort> hosts = new ArrayList<>();
List<UUID> hostIds = new ArrayList<>();
+ private SeedProvider originalSeedProvider;
+
@Before
public void setup()
{
tmd.clearUnsafe();
- };
+ originalSeedProvider = DatabaseDescriptor.getSeedProvider();
+ }
+
+ @After
+ public void tearDown()
+ {
+ DatabaseDescriptor.setSeedProvider(originalSeedProvider);
+ }
@Test
public void testLargeGenerationJump() throws UnknownHostException, InterruptedException
@@ -90,4 +105,127 @@ public class GossiperTest
//The generation should not have been updated because it is over Gossiper.MAX_GENERATION_DIFFERENCE in the future
assertEquals(proposedRemoteHeartBeat.getGeneration(), actualRemoteHeartBeat.getGeneration());
}
+
+ // Note: This test might fail if for some reason the node broadcast address is in 127.99.0.0/16
+ @Test
+ public void testReloadSeeds() throws UnknownHostException
+ {
+ Gossiper gossiper = new Gossiper(false);
+ List<String> loadedList;
+
+ // Initialize the seed list directly to a known set to start with
+ gossiper.seeds.clear();
+ InetAddressAndPort addr = InetAddressAndPort.getByAddress(InetAddress.getByName("127.99.1.1"));
+ int nextSize = 4;
+ List<InetAddressAndPort> nextSeeds = new ArrayList<>(nextSize);
+ for (int i = 0; i < nextSize; i ++)
+ {
+ gossiper.seeds.add(addr);
+ nextSeeds.add(addr);
+ addr = InetAddressAndPort.getByAddress(InetAddresses.increment(addr.address));
+ }
+ Assert.assertEquals(nextSize, gossiper.seeds.size());
+
+ // Add another unique address to the list
+ addr = InetAddressAndPort.getByAddress(InetAddresses.increment(addr.address));
+ nextSeeds.add(addr);
+ nextSize++;
+ DatabaseDescriptor.setSeedProvider(new TestSeedProvider(nextSeeds));
+ loadedList = gossiper.reloadSeeds();
+
+ // Check that the new entry was added
+ Assert.assertEquals(nextSize, loadedList.size());
+ for (InetAddressAndPort a : nextSeeds)
+ Assert.assertTrue(loadedList.contains(a.toString()));
+
+ // Check that the return value of the reloadSeeds matches the content of the getSeeds call
+ // and that they both match the internal contents of the Gossiper seeds list
+ Assert.assertEquals(loadedList.size(), gossiper.getSeeds().size());
+ for (InetAddressAndPort a : gossiper.seeds)
+ {
+ Assert.assertTrue(loadedList.contains(a.toString()));
+ Assert.assertTrue(gossiper.getSeeds().contains(a.toString()));
+ }
+
+ // Add a duplicate of the last address to the seed provider list
+ int uniqueSize = nextSize;
+ nextSeeds.add(addr);
+ nextSize++;
+ DatabaseDescriptor.setSeedProvider(new TestSeedProvider(nextSeeds));
+ loadedList = gossiper.reloadSeeds();
+
+ // Check that the number of seed nodes reported hasn't increased
+ Assert.assertEquals(uniqueSize, loadedList.size());
+ for (InetAddressAndPort a : nextSeeds)
+ Assert.assertTrue(loadedList.contains(a.toString()));
+
+ // Create a new list that has no overlaps with the previous list
+ addr = InetAddressAndPort.getByAddress(InetAddress.getByName("127.99.2.1"));
+ int disjointSize = 3;
+ List<InetAddressAndPort> disjointSeeds = new ArrayList<>(disjointSize);
+ for (int i = 0; i < disjointSize; i ++)
+ {
+ disjointSeeds.add(addr);
+ addr = InetAddressAndPort.getByAddress(InetAddresses.increment(addr.address));
+ }
+ DatabaseDescriptor.setSeedProvider(new TestSeedProvider(disjointSeeds));
+ loadedList = gossiper.reloadSeeds();
+
+ // Check that the list now contains exactly the new other list.
+ Assert.assertEquals(disjointSize, gossiper.getSeeds().size());
+ Assert.assertEquals(disjointSize, loadedList.size());
+ for (InetAddressAndPort a : disjointSeeds)
+ {
+ Assert.assertTrue(gossiper.getSeeds().contains(a.toString()));
+ Assert.assertTrue(loadedList.contains(a.toString()));
+ }
+
+ // Set the seed node provider to return an empty list
+ DatabaseDescriptor.setSeedProvider(new TestSeedProvider(new ArrayList<InetAddressAndPort>()));
+ loadedList = gossiper.reloadSeeds();
+
+ // Check that the in memory seed node list was not modified
+ Assert.assertEquals(disjointSize, loadedList.size());
+ for (InetAddressAndPort a : disjointSeeds)
+ Assert.assertTrue(loadedList.contains(a.toString()));
+
+ // Change the seed provider to one that throws an unchecked exception
+ DatabaseDescriptor.setSeedProvider(new ErrorSeedProvider());
+ loadedList = gossiper.reloadSeeds();
+
+ // Check for the expected null response from a reload error
+ Assert.assertNull(loadedList);
+
+ // Check that the in memory seed node list was not modified and the exception was caught
+ Assert.assertEquals(disjointSize, gossiper.getSeeds().size());
+ for (InetAddressAndPort a : disjointSeeds)
+ Assert.assertTrue(gossiper.getSeeds().contains(a.toString()));
+ }
+
+ static class TestSeedProvider implements SeedProvider
+ {
+ private List<InetAddressAndPort> seeds;
+
+ TestSeedProvider(List<InetAddressAndPort> seeds)
+ {
+ this.seeds = seeds;
+ }
+
+ @Override
+ public List<InetAddressAndPort> getSeeds()
+ {
+ return seeds;
+ }
+ }
+
+ // A seed provider for testing which throws assertion errors when queried
+ static class ErrorSeedProvider implements SeedProvider
+ {
+ @Override
+ public List<InetAddressAndPort> getSeeds()
+ {
+ assert(false);
+ return new ArrayList<>();
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org