You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jj...@apache.org on 2016/09/28 01:27:30 UTC
[1/3] cassandra git commit: Reject invalid DC names as option while
creating or altering NetworkTopologyStrategy
Repository: cassandra
Updated Branches:
refs/heads/cassandra-3.0 21d8a7d3b -> f2c5ad743
refs/heads/trunk b80ef9b25 -> d45f323eb
Reject invalid DC names as option while creating or altering NetworkTopologyStrategy
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/f2c5ad74
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/f2c5ad74
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/f2c5ad74
Branch: refs/heads/cassandra-3.0
Commit: f2c5ad743933498e60e7eef55e8daaa6ce338a03
Parents: 21d8a7d
Author: Nachiket Patil <na...@apple.com>
Authored: Fri Aug 5 16:05:34 2016 -0700
Committer: Jeff Jirsa <je...@crowdstrike.com>
Committed: Tue Sep 27 18:16:26 2016 -0700
----------------------------------------------------------------------
CHANGES.txt | 1 +
NEWS.txt | 11 ++++
.../locator/AbstractReplicationStrategy.java | 2 +-
.../locator/NetworkTopologyStrategy.java | 39 ++++++++++++-
.../org/apache/cassandra/cql3/CQLTester.java | 11 ++++
.../validation/entities/SecondaryIndexTest.java | 10 ----
.../cql3/validation/operations/AlterTest.java | 47 +++++++++++++++-
.../cql3/validation/operations/CreateTest.java | 59 ++++++++++++++++++++
.../apache/cassandra/dht/BootStrapperTest.java | 10 +++-
.../org/apache/cassandra/service/MoveTest.java | 9 ++-
10 files changed, 181 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f2c5ad74/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 4280abd..6edc491 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -6,6 +6,7 @@
* Extend ColumnIdentifier.internedInstances key to include the type that generated the byte buffer (CASSANDRA-12516)
* Backport CASSANDRA-10756 (race condition in NativeTransportService shutdown) (CASSANDRA-12472)
* If CF has no clustering columns, any row cache is full partition cache (CASSANDRA-12499)
+ * Reject invalid replication settings when creating or altering a keyspace (CASSANDRA-12681)
Merged from 2.2:
* Fix exceptions when enabling gossip on nodes that haven't joined the ring (CASSANDRA-12253)
* Fix authentication problem when invoking clqsh copy from a SOURCE command (CASSANDRA-12642)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f2c5ad74/NEWS.txt
----------------------------------------------------------------------
diff --git a/NEWS.txt b/NEWS.txt
index 0bd3920..b97a420 100644
--- a/NEWS.txt
+++ b/NEWS.txt
@@ -13,6 +13,17 @@ restore snapshots created with the previous major version using the
'sstableloader' tool. You can upgrade the file format of your snapshots
using the provided 'sstableupgrade' tool.
+3.0.10
+=====
+
+Upgrading
+---------
+ - To protect against accidental data loss, cassandra no longer allows
+ users to set arbitrary datacenter names for NetworkTopologyStrategy.
+ Cassandra will allow users to continue using existing keyspaces
+ with invalid datacenter names, but will validat DC names on CREATE and
+ ALTER
+
3.0.9
=====
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f2c5ad74/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java b/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java
index c90c6a1..d72c0c2 100644
--- a/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java
+++ b/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java
@@ -319,7 +319,7 @@ public abstract class AbstractReplicationStrategy
}
}
- private void validateExpectedOptions() throws ConfigurationException
+ protected void validateExpectedOptions() throws ConfigurationException
{
Collection expectedOptions = recognizedOptions();
if (expectedOptions == null)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f2c5ad74/src/java/org/apache/cassandra/locator/NetworkTopologyStrategy.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/locator/NetworkTopologyStrategy.java b/src/java/org/apache/cassandra/locator/NetworkTopologyStrategy.java
index 7c8d95e..78f5b06 100644
--- a/src/java/org/apache/cassandra/locator/NetworkTopologyStrategy.java
+++ b/src/java/org/apache/cassandra/locator/NetworkTopologyStrategy.java
@@ -24,9 +24,11 @@ import java.util.Map.Entry;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.locator.TokenMetadata.Topology;
+import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.FBUtilities;
import com.google.common.collect.Multimap;
@@ -193,10 +195,43 @@ public class NetworkTopologyStrategy extends AbstractReplicationStrategy
}
}
+ /*
+ * (non-javadoc) Method to generate list of valid data center names to be used to validate the replication parameters during CREATE / ALTER keyspace operations.
+ * All peers of current node are fetched from {@link TokenMetadata} and then a set is build by fetching DC name of each peer.
+ * @return a set of valid DC names
+ */
+ private static Set<String> buildValidDataCentersSet()
+ {
+ final Set<String> validDataCenters = new HashSet<>();
+ final IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
+
+ // Add data center of localhost.
+ validDataCenters.add(snitch.getDatacenter(FBUtilities.getBroadcastAddress()));
+ // Fetch and add DCs of all peers.
+ for (final InetAddress peer : StorageService.instance.getTokenMetadata().getAllEndpoints())
+ {
+ validDataCenters.add(snitch.getDatacenter(peer));
+ }
+
+ return validDataCenters;
+ }
+
public Collection<String> recognizedOptions()
{
- // We explicitely allow all options
- return null;
+ // only valid options are valid DC names.
+ return buildValidDataCentersSet();
+ }
+
+ protected void validateExpectedOptions() throws ConfigurationException
+ {
+ // Do not accept query with no data centers specified.
+ if (this.configOptions.isEmpty())
+ {
+ throw new ConfigurationException("Configuration for at least one datacenter must be present");
+ }
+
+ // Validate the data center names
+ super.validateExpectedOptions();
}
@Override
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f2c5ad74/test/unit/org/apache/cassandra/cql3/CQLTester.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/cql3/CQLTester.java b/test/unit/org/apache/cassandra/cql3/CQLTester.java
index 7f5eb02..69a0b79 100644
--- a/test/unit/org/apache/cassandra/cql3/CQLTester.java
+++ b/test/unit/org/apache/cassandra/cql3/CQLTester.java
@@ -56,6 +56,8 @@ import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.SyntaxException;
import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.locator.AbstractEndpointSnitch;
+import org.apache.cassandra.locator.IEndpointSnitch;
import org.apache.cassandra.serializers.TypeSerializer;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.service.QueryState;
@@ -82,6 +84,8 @@ public abstract class CQLTester
protected static final long ROW_CACHE_SIZE_IN_MB = Integer.valueOf(System.getProperty("cassandra.test.row_cache_size_in_mb", "0"));
private static final AtomicInteger seqNumber = new AtomicInteger();
protected static final ByteBuffer TOO_BIG = ByteBuffer.allocate(FBUtilities.MAX_UNSIGNED_SHORT + 1024);
+ public static final String DATA_CENTER = "datacenter1";
+ public static final String RACK1 = "rack1";
private static org.apache.cassandra.transport.Server server;
protected static final int nativePort;
@@ -127,6 +131,13 @@ public abstract class CQLTester
{
throw new RuntimeException(e);
}
+ // Register an EndpointSnitch which returns fixed values for test.
+ DatabaseDescriptor.setEndpointSnitch(new AbstractEndpointSnitch()
+ {
+ @Override public String getRack(InetAddress endpoint) { return RACK1; }
+ @Override public String getDatacenter(InetAddress endpoint) { return DATA_CENTER; }
+ @Override public int compareEndpoints(InetAddress target, InetAddress a1, InetAddress a2) { return 0; }
+ });
}
public static ResultMessage lastSchemaChangeResult;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f2c5ad74/test/unit/org/apache/cassandra/cql3/validation/entities/SecondaryIndexTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/SecondaryIndexTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/SecondaryIndexTest.java
index 0cf13bd..2b31481 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/SecondaryIndexTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/SecondaryIndexTest.java
@@ -239,16 +239,6 @@ public class SecondaryIndexTest extends CQLTester
}
/**
- * Check one can use arbitrary name for datacenter when creating keyspace (#4278),
- * migrated from cql_tests.py:TestCQL.keyspace_creation_options_test()
- */
- @Test
- public void testDataCenterName() throws Throwable
- {
- execute("CREATE KEYSPACE Foo WITH replication = { 'class' : 'NetworkTopologyStrategy', 'us-east' : 1, 'us-west' : 1 };");
- }
-
- /**
* Migrated from cql_tests.py:TestCQL.indexes_composite_test()
*/
@Test
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f2c5ad74/test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java
index bcd6587..48108cd 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java
@@ -212,13 +212,13 @@ public class AlterTest extends CQLTester
row(ks1, true),
row(ks2, false));
- schemaChange("ALTER KEYSPACE " + ks1 + " WITH replication = { 'class' : 'NetworkTopologyStrategy', 'dc1' : 1 } AND durable_writes=False");
+ schemaChange("ALTER KEYSPACE " + ks1 + " WITH replication = { 'class' : 'NetworkTopologyStrategy', '" + DATA_CENTER + "' : 1 } AND durable_writes=False");
schemaChange("ALTER KEYSPACE " + ks2 + " WITH durable_writes=true");
assertRowsIgnoringOrderAndExtra(execute("SELECT keyspace_name, durable_writes, replication FROM system_schema.keyspaces"),
row(KEYSPACE, true, map("class", "org.apache.cassandra.locator.SimpleStrategy", "replication_factor", "1")),
row(KEYSPACE_PER_TEST, true, map("class", "org.apache.cassandra.locator.SimpleStrategy", "replication_factor", "1")),
- row(ks1, false, map("class", "org.apache.cassandra.locator.NetworkTopologyStrategy", "dc1", "1")),
+ row(ks1, false, map("class", "org.apache.cassandra.locator.NetworkTopologyStrategy", DATA_CENTER , "1")),
row(ks2, true, map("class", "org.apache.cassandra.locator.SimpleStrategy", "replication_factor", "1")));
execute("USE " + ks1);
@@ -233,6 +233,49 @@ public class AlterTest extends CQLTester
}
/**
+ * Test {@link ConfigurationException} thrown on alter keyspace to no DC option in replication configuration.
+ */
+ @Test
+ public void testAlterKeyspaceWithNoOptionThrowsConfigurationException() throws Throwable
+ {
+ // Create keyspaces
+ execute("CREATE KEYSPACE testABC WITH replication={ 'class' : 'NetworkTopologyStrategy', '" + DATA_CENTER + "' : 3 }");
+ execute("CREATE KEYSPACE testXYZ WITH replication={ 'class' : 'SimpleStrategy', 'replication_factor' : 3 }");
+
+ // Try to alter the created keyspace without any option
+ assertInvalidThrow(ConfigurationException.class, "ALTER KEYSPACE testABC WITH replication={ 'class' : 'NetworkTopologyStrategy' }");
+ assertInvalidThrow(ConfigurationException.class, "ALTER KEYSPACE testXYZ WITH replication={ 'class' : 'SimpleStrategy' }");
+
+ // Make sure that the alter works as expected
+ execute("ALTER KEYSPACE testABC WITH replication={ 'class' : 'NetworkTopologyStrategy', '" + DATA_CENTER + "' : 2 }");
+ execute("ALTER KEYSPACE testXYZ WITH replication={ 'class' : 'SimpleStrategy', 'replication_factor' : 2 }");
+
+ // clean up
+ execute("DROP KEYSPACE IF EXISTS testABC");
+ execute("DROP KEYSPACE IF EXISTS testXYZ");
+ }
+
+ /**
+ * Test {@link ConfigurationException} thrown when altering a keyspace to invalid DC option in replication configuration.
+ */
+ @Test
+ public void testAlterKeyspaceWithNTSOnlyAcceptsConfiguredDataCenterNames() throws Throwable
+ {
+ // Create a keyspace with expected DC name.
+ execute("CREATE KEYSPACE testABC WITH replication = {'class' : 'NetworkTopologyStrategy', '" + DATA_CENTER + "' : 2 }");
+
+ // try modifying the keyspace
+ assertInvalidThrow(ConfigurationException.class, "ALTER KEYSPACE testABC WITH replication = { 'class' : 'NetworkTopologyStrategy', 'INVALID_DC' : 2 }");
+ execute("ALTER KEYSPACE testABC WITH replication = {'class' : 'NetworkTopologyStrategy', '" + DATA_CENTER + "' : 3 }");
+
+ // Mix valid and invalid, should throw an exception
+ assertInvalidThrow(ConfigurationException.class, "ALTER KEYSPACE testABC WITH replication={ 'class' : 'NetworkTopologyStrategy', '" + DATA_CENTER + "' : 2 , 'INVALID_DC': 1}");
+
+ // clean-up
+ execute("DROP KEYSPACE IF EXISTS testABC");
+ }
+
+ /**
* Test for bug of 5232,
* migrated from cql_tests.py:TestCQL.alter_bug_test()
*/
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f2c5ad74/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java
index 33a41d8..0781169 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.cassandra.cql3.validation.operations;
+import java.net.InetAddress;
import java.util.Collection;
import java.util.Collections;
import java.util.UUID;
@@ -25,12 +26,15 @@ import org.junit.Test;
import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.Schema;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.partitions.Partition;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.SyntaxException;
+import org.apache.cassandra.locator.AbstractEndpointSnitch;
+import org.apache.cassandra.locator.IEndpointSnitch;
import org.apache.cassandra.schema.SchemaKeyspace;
import org.apache.cassandra.triggers.ITrigger;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -345,6 +349,33 @@ public class CreateTest extends CQLTester
}
/**
+ * Test {@link ConfigurationException} is thrown on create keyspace with invalid DC option in replication configuration .
+ */
+ @Test
+ public void testCreateKeyspaceWithNTSOnlyAcceptsConfiguredDataCenterNames() throws Throwable
+ {
+ assertInvalidThrow(ConfigurationException.class, "CREATE KEYSPACE testABC WITH replication = { 'class' : 'NetworkTopologyStrategy', 'INVALID_DC' : 2 }");
+ execute("CREATE KEYSPACE testABC WITH replication = {'class' : 'NetworkTopologyStrategy', '" + DATA_CENTER + "' : 2 }");
+
+ // Mix valid and invalid, should throw an exception
+ assertInvalidThrow(ConfigurationException.class, "CREATE KEYSPACE testXYZ WITH replication={ 'class' : 'NetworkTopologyStrategy', '" + DATA_CENTER + "' : 2 , 'INVALID_DC': 1}");
+
+ // clean-up
+ execute("DROP KEYSPACE IF EXISTS testABC");
+ execute("DROP KEYSPACE IF EXISTS testXYZ");
+ }
+
+ /**
+ * Test {@link ConfigurationException} is thrown on create keyspace without any options.
+ */
+ @Test
+ public void testConfigurationExceptionThrownWhenCreateKeyspaceWithNoOptions() throws Throwable
+ {
+ assertInvalidThrow(ConfigurationException.class, "CREATE KEYSPACE testXYZ with replication = { 'class': 'NetworkTopologyStrategy' }");
+ assertInvalidThrow(ConfigurationException.class, "CREATE KEYSPACE testXYZ WITH replication = { 'class' : 'SimpleStrategy' }");
+ }
+
+ /**
* Test create and drop table
* migrated from cql_tests.py:TestCQL.table_test()
*/
@@ -495,6 +526,34 @@ public class CreateTest extends CQLTester
}
@Test
+ // tests CASSANDRA-4278
+ public void testHyphenDatacenters() throws Throwable
+ {
+ IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
+
+ // Register an EndpointSnitch which returns fixed values for test.
+ DatabaseDescriptor.setEndpointSnitch(new AbstractEndpointSnitch()
+ {
+ @Override
+ public String getRack(InetAddress endpoint) { return RACK1; }
+
+ @Override
+ public String getDatacenter(InetAddress endpoint) { return "us-east-1"; }
+
+ @Override
+ public int compareEndpoints(InetAddress target, InetAddress a1, InetAddress a2) { return 0; }
+ });
+
+ execute("CREATE KEYSPACE Foo WITH replication = { 'class' : 'NetworkTopologyStrategy', 'us-east-1' : 1 };");
+
+ // Restore the previous EndpointSnitch
+ DatabaseDescriptor.setEndpointSnitch(snitch);
+
+ // Clean up
+ execute("DROP KEYSPACE IF EXISTS Foo");
+ }
+
+ @Test
// tests CASSANDRA-9565
public void testDoubleWith() throws Throwable
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f2c5ad74/test/unit/org/apache/cassandra/dht/BootStrapperTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/dht/BootStrapperTest.java b/test/unit/org/apache/cassandra/dht/BootStrapperTest.java
index 8974791..cd34aea 100644
--- a/test/unit/org/apache/cassandra/dht/BootStrapperTest.java
+++ b/test/unit/org/apache/cassandra/dht/BootStrapperTest.java
@@ -27,6 +27,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.UUID;
import com.google.common.collect.Lists;
@@ -59,7 +60,7 @@ public class BootStrapperTest
static IPartitioner oldPartitioner;
@BeforeClass
- public static void setup() throws ConfigurationException
+ public static void setup() throws Exception
{
oldPartitioner = StorageService.instance.setPartitionerUnsafe(Murmur3Partitioner.instance);
SchemaLoader.startGossiper();
@@ -177,6 +178,13 @@ public class BootStrapperTest
int vn = 16;
String ks = "BootStrapperTestNTSKeyspace" + rackCount + replicas;
String dc = "1";
+
+ // Register peers with expected DC for NetworkTopologyStrategy.
+ TokenMetadata metadata = StorageService.instance.getTokenMetadata();
+ metadata.clearUnsafe();
+ metadata.updateHostId(UUID.randomUUID(), InetAddress.getByName("127.1.0.99"));
+ metadata.updateHostId(UUID.randomUUID(), InetAddress.getByName("127.15.0.99"));
+
SchemaLoader.createKeyspace(ks, KeyspaceParams.nts(dc, replicas, "15", 15), SchemaLoader.standardCFMD(ks, "Standard1"));
TokenMetadata tm = new TokenMetadata();
tm.clearUnsafe();
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f2c5ad74/test/unit/org/apache/cassandra/service/MoveTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/service/MoveTest.java b/test/unit/org/apache/cassandra/service/MoveTest.java
index 53365aa..6c07a47 100644
--- a/test/unit/org/apache/cassandra/service/MoveTest.java
+++ b/test/unit/org/apache/cassandra/service/MoveTest.java
@@ -82,7 +82,7 @@ public class MoveTest
* So instead of extending SchemaLoader, we call it's method below.
*/
@BeforeClass
- public static void setup() throws ConfigurationException
+ public static void setup() throws Exception
{
oldPartitioner = StorageService.instance.setPartitionerUnsafe(partitioner);
SchemaLoader.loadSchema();
@@ -105,7 +105,7 @@ public class MoveTest
StorageService.instance.getTokenMetadata().clearUnsafe();
}
- private static void addNetworkTopologyKeyspace(String keyspaceName, Integer... replicas) throws ConfigurationException
+ private static void addNetworkTopologyKeyspace(String keyspaceName, Integer... replicas) throws Exception
{
DatabaseDescriptor.setEndpointSnitch(new AbstractNetworkTopologySnitch()
@@ -139,6 +139,11 @@ public class MoveTest
}
});
+ final TokenMetadata tmd = StorageService.instance.getTokenMetadata();
+ tmd.clearUnsafe();
+ tmd.updateHostId(UUID.randomUUID(), InetAddress.getByName("127.0.0.1"));
+ tmd.updateHostId(UUID.randomUUID(), InetAddress.getByName("127.0.0.2"));
+
KeyspaceMetadata keyspace = KeyspaceMetadata.create(keyspaceName,
KeyspaceParams.nts(configOptions(replicas)),
Tables.of(CFMetaData.Builder.create(keyspaceName, "CF1")
[3/3] cassandra git commit: Merge branch 'cassandra-3.0' into trunk
Posted by jj...@apache.org.
Merge branch 'cassandra-3.0' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/d45f323e
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/d45f323e
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/d45f323e
Branch: refs/heads/trunk
Commit: d45f323eb972c6fec146e5cfa84fdc47eb8aa5eb
Parents: b80ef9b f2c5ad7
Author: Jeff Jirsa <je...@crowdstrike.com>
Authored: Tue Sep 27 18:16:55 2016 -0700
Committer: Jeff Jirsa <je...@crowdstrike.com>
Committed: Tue Sep 27 18:26:17 2016 -0700
----------------------------------------------------------------------
CHANGES.txt | 1 +
NEWS.txt | 3 +
.../locator/AbstractReplicationStrategy.java | 2 +-
.../locator/NetworkTopologyStrategy.java | 41 ++++++++++++++
.../org/apache/cassandra/cql3/CQLTester.java | 11 ++++
.../validation/entities/SecondaryIndexTest.java | 10 ----
.../cql3/validation/operations/AlterTest.java | 47 +++++++++++++++-
.../cql3/validation/operations/CreateTest.java | 59 ++++++++++++++++++++
.../apache/cassandra/dht/BootStrapperTest.java | 8 +++
.../org/apache/cassandra/service/MoveTest.java | 9 ++-
10 files changed, 176 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d45f323e/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 75e7d2a,6edc491..7a5d73a
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -88,58 -33,12 +88,59 @@@ Merged from 3.0
* Disk failure policy should not be invoked on out of space (CASSANDRA-12385)
* Calculate last compacted key on startup (CASSANDRA-6216)
* Add schema to snapshot manifest, add USING TIMESTAMP clause to ALTER TABLE statements (CASSANDRA-7190)
+ * If CF has no clustering columns, any row cache is full partition cache (CASSANDRA-12499)
++ * Reject invalid replication settings when creating or altering a keyspace (CASSANDRA-12681)
+Merged from 2.2:
+ * Make Collections deserialization more robust (CASSANDRA-12618)
+ * Fix exceptions when enabling gossip on nodes that haven't joined the ring (CASSANDRA-12253)
+ * Fix authentication problem when invoking clqsh copy from a SOURCE command (CASSANDRA-12642)
+ * Decrement pending range calculator jobs counter in finally block
+ * cqlshlib tests: increase default execute timeout (CASSANDRA-12481)
+ * Forward writes to replacement node when replace_address != broadcast_address (CASSANDRA-8523)
+ * Fail repair on non-existing table (CASSANDRA-12279)
+ * Enable repair -pr and -local together (fix regression of CASSANDRA-7450) (CASSANDRA-12522)
+
+
+3.8, 3.9
+ * Fix value skipping with counter columns (CASSANDRA-11726)
+ * Fix nodetool tablestats miss SSTable count (CASSANDRA-12205)
+ * Fixed flacky SSTablesIteratedTest (CASSANDRA-12282)
+ * Fixed flacky SSTableRewriterTest: check file counts before calling validateCFS (CASSANDRA-12348)
+ * cqlsh: Fix handling of $$-escaped strings (CASSANDRA-12189)
+ * Fix SSL JMX requiring truststore containing server cert (CASSANDRA-12109)
+ * RTE from new CDC column breaks in flight queries (CASSANDRA-12236)
+ * Fix hdr logging for single operation workloads (CASSANDRA-12145)
+ * Fix SASI PREFIX search in CONTAINS mode with partial terms (CASSANDRA-12073)
+ * Increase size of flushExecutor thread pool (CASSANDRA-12071)
+ * Partial revert of CASSANDRA-11971, cannot recycle buffer in SP.sendMessagesToNonlocalDC (CASSANDRA-11950)
+ * Upgrade netty to 4.0.39 (CASSANDRA-12032, CASSANDRA-12034)
+ * Improve details in compaction log message (CASSANDRA-12080)
+ * Allow unset values in CQLSSTableWriter (CASSANDRA-11911)
+ * Chunk cache to request compressor-compatible buffers if pool space is exhausted (CASSANDRA-11993)
+ * Remove DatabaseDescriptor dependencies from SequentialWriter (CASSANDRA-11579)
+ * Move skip_stop_words filter before stemming (CASSANDRA-12078)
+ * Support seek() in EncryptedFileSegmentInputStream (CASSANDRA-11957)
+ * SSTable tools mishandling LocalPartitioner (CASSANDRA-12002)
+ * When SEPWorker assigned work, set thread name to match pool (CASSANDRA-11966)
+ * Add cross-DC latency metrics (CASSANDRA-11596)
+ * Allow terms in selection clause (CASSANDRA-10783)
+ * Add bind variables to trace (CASSANDRA-11719)
+ * Switch counter shards' clock to timestamps (CASSANDRA-9811)
+ * Introduce HdrHistogram and response/service/wait separation to stress tool (CASSANDRA-11853)
+ * entry-weighers in QueryProcessor should respect partitionKeyBindIndexes field (CASSANDRA-11718)
+ * Support older ant versions (CASSANDRA-11807)
+ * Estimate compressed on disk size when deciding if sstable size limit reached (CASSANDRA-11623)
+ * cassandra-stress profiles should support case sensitive schemas (CASSANDRA-11546)
+ * Remove DatabaseDescriptor dependency from FileUtils (CASSANDRA-11578)
+ * Faster streaming (CASSANDRA-9766)
+ * Add prepared query parameter to trace for "Execute CQL3 prepared query" session (CASSANDRA-11425)
+ * Add repaired percentage metric (CASSANDRA-11503)
+ * Add Change-Data-Capture (CASSANDRA-8844)
+Merged from 3.0:
+ * Fix paging for 2.x to 3.x upgrades (CASSANDRA-11195)
* Fix clean interval not sent to commit log for empty memtable flush (CASSANDRA-12436)
* Fix potential resource leak in RMIServerSocketFactoryImpl (CASSANDRA-12331)
- * Backport CASSANDRA-12002 (CASSANDRA-12177)
* Make sure compaction stats are updated when compaction is interrupted (CASSANDRA-12100)
- * Fix potential bad messaging service message for paged range reads
- within mixed-version 3.x clusters (CASSANDRA-12249)
* Change commitlog and sstables to track dirty and clean intervals (CASSANDRA-11828)
* NullPointerException during compaction on table with static columns (CASSANDRA-12336)
* Fixed ConcurrentModificationException when reading metrics in GraphiteReporter (CASSANDRA-11823)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d45f323e/NEWS.txt
----------------------------------------------------------------------
diff --cc NEWS.txt
index ad0f2be,b97a420..9ab7c26
--- a/NEWS.txt
+++ b/NEWS.txt
@@@ -13,120 -13,19 +13,123 @@@ restore snapshots created with the prev
'sstableloader' tool. You can upgrade the file format of your snapshots
using the provided 'sstableupgrade' tool.
-3.0.10
-=====
+3.10
+====
-Upgrading
----------
- - To protect against accidental data loss, cassandra no longer allows
- users to set arbitrary datacenter names for NetworkTopologyStrategy.
- Cassandra will allow users to continue using existing keyspaces
- with invalid datacenter names, but will validat DC names on CREATE and
- ALTER
+New features
+------------
+ - Runtime modification of concurrent_compactors is now available via nodetool
+ - Support for the assignment operators +=/-= has been added for update queries.
+ - An Index implementation may now provide a task which runs prior to joining
+ the ring. See CASSANDRA-12039
+ - Filtering on partition key columns is now also supported for queries without
+ secondary indexes.
+ - A slow query log has been added: slow queries will be logged at DEBUG level.
+ For more details refer to CASSANDRA-12403 and slow_query_log_timeout_in_ms
+ in cassandra.yaml.
+ - Support for GROUP BY queries has been added.
+ - A new compaction-stress tool has been added to test the throughput of compaction
+ for any cassandra-stress user schema. see compaction-stress help for how to use.
+ - Compaction can now take into account overlapping tables that don't take part
+ in the compaction to look for deleted or overwritten data in the compacted tables.
+ Then such data is found, it can be safely discarded, which in turn should enable
+ the removal of tombstones over that data.
+
+ The behavior can be engaged in two ways:
+ - as a "nodetool garbagecollect -g CELL/ROW" operation, which applies
+ single-table compaction on all sstables to discard deleted data in one step.
+ - as a "provide_overlapping_tombstones:CELL/ROW/NONE" compaction strategy flag,
+ which uses overlapping tables as a source of deletions/overwrites during all
+ compactions.
+ The argument specifies the granularity at which deleted data is to be found:
+ - If ROW is specified, only whole deleted rows (or sets of rows) will be
+ discarded.
+ - If CELL is specified, any columns whose value is overwritten or deleted
+ will also be discarded.
+ - NONE (default) specifies the old behavior, overlapping tables are not used to
+ decide when to discard data.
+ Which option to use depends on your workload, both ROW and CELL increase the
+ disk load on compaction (especially with the size-tiered compaction strategy),
+ with CELL being more resource-intensive. Both should lead to better read
+ performance if deleting rows (resp. overwriting or deleting cells) is common.
+ - Prepared statements are now persisted in the table prepared_statements in
+ the system keyspace. Upon startup, this table is used to preload all
+ previously prepared statements - i.e. in many cases clients do not need to
+ re-prepare statements against restarted nodes.
+ - cqlsh can now connect to older Cassandra versions by downgrading the native
+ protocol version. Please note that this is currently not part of our release
+ testing and, as a consequence, it is not guaranteed to work in all cases.
+ See CASSANDRA-12150 for more details.
+ - Snapshots that are automatically taken before a table is dropped or truncated
+ will have a "dropped" or "truncated" prefix on their snapshot tag name.
+ - Metrics are exposed for successful and failed authentication attempts.
+ These can be located using the object names org.apache.cassandra.metrics:type=Client,name=AuthSuccess
+ and org.apache.cassandra.metrics:type=Client,name=AuthFailure respectively.
+ - Add support to "unset" JSON fields in prepared statements by specifying DEFAULT UNSET.
+ See CASSANDRA-11424 for details
+ - Allow TTL with null value on insert and update. It will be treated as equivalent to inserting a 0.
+
+Upgrading
+---------
+ - Request timeouts in cassandra.yaml (read_request_timeout_in_ms, etc) now apply to the
+ "full" request time on the coordinator. Previously, they only covered the time from
+ when the coordinator sent a message to a replica until the time that the replica
+ responded. Additionally, the previous behavior was to reset the timeout when performing
+ a read repair, making a second read to fix a short read, and when subranges were read
+ as part of a range scan or secondary index query. In 3.10 and higher, the timeout
+ is no longer reset for these "subqueries". The entire request must complete within
+ the specified timeout. As a consequence, your timeouts may need to be adjusted
+ to account for this. See CASSANDRA-12256 for more details.
+ - Logs written to stdout are now consistent with logs written to files.
+ Time is now local (it was UTC on the console and local in files). Date, thread, file
+ and line info where added to stdout. (see CASSANDRA-12004)
+ - The 'clientutil' jar, which has been somewhat broken on the 3.x branch, is not longer provided.
+ The features provided by that jar are provided by any good java driver and we advise relying on drivers rather on
+ that jar, but if you need that jar for backward compatiblity until you do so, you should use the version provided
+ on previous Cassandra branch, like the 3.0 branch (by design, the functionality provided by that jar are stable
+ accross versions so using the 3.0 jar for a client connecting to 3.x should work without issues).
+ - (Tools development) DatabaseDescriptor no longer implicitly startups components/services like
+ commit log replay. This may break existing 3rd party tools and clients. In order to startup
+ a standalone tool or client application, use the DatabaseDescriptor.toolInitialization() or
+ DatabaseDescriptor.clientInitialization() methods. Tool initialization sets up partitioner,
+ snitch, encryption context. Client initialization just applies the configuration but does not
+ setup anything. Instead of using Config.setClientMode() or Config.isClientMode(), which are
+ deprecated now, use one of the appropiate new methods in DatabaseDescriptor.
+ - Application layer keep-alives were added to the streaming protocol to prevent idle incoming connections from
+ timing out and failing the stream session (CASSANDRA-11839). This effectively deprecates the streaming_socket_timeout_in_ms
+ property in favor of streaming_keep_alive_period_in_secs. See cassandra.yaml for more details about this property.
++ - Cassandra will no longer allow invalid keyspace replication options, such as invalid datacenter names for
++ NetworkTopologyStrategy. Existing keyspaces will continue to operate, but CREATE and ALTER will validate that
++ all datacenters specified exist in the cluster.
+
+3.8
+===
-3.0.9
-=====
+New features
+------------
+ - Shared pool threads are now named according to the stage they are executing
+ tasks for. Thread names mentioned in traced queries change accordingly.
+ - A new option has been added to cassandra-stress "-rate fixed={number}/s"
+ that forces a scheduled rate of operations/sec over time. Using this, stress can
+ accurately account for coordinated ommission from the stress process.
+ - The cassandra-stress "-rate limit=" option has been renamed to "-rate throttle="
+ - hdr histograms have been added to stress runs, it's output can be saved to disk using:
+ "-log hdrfile=" option. This histogram includes response/service/wait times when used with the
+ fixed or throttle rate options. The histogram file can be plotted on
+ http://hdrhistogram.github.io/HdrHistogram/plotFiles.html
+ - TimeWindowCompactionStrategy has been added. This has proven to be a better approach
+ to time series compaction and new tables should use this instead of DTCS. See
+ CASSANDRA-9666 for details.
+ - Change-Data-Capture is now available. See cassandra.yaml and for cdc-specific flags and
+ a brief explanation of on-disk locations for archived data in CommitLog form. This can
+ be enabled via ALTER TABLE ... WITH cdc=true.
+ Upon flush, CommitLogSegments containing data for CDC-enabled tables are moved to
+ the data/cdc_raw directory until removed by the user and writes to CDC-enabled tables
+ will be rejected with a WriteTimeoutException once cdc_total_space_in_mb is reached
+ between unflushed CommitLogSegments and cdc_raw.
+ NOTE: CDC is disabled by default in the .yaml file. Do not enable CDC on a mixed-version
+ cluster as it will lead to exceptions which can interrupt traffic. Once all nodes
+ have been upgraded to 3.8 it is safe to enable this feature and restart the cluster.
Upgrading
---------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d45f323e/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d45f323e/src/java/org/apache/cassandra/locator/NetworkTopologyStrategy.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/locator/NetworkTopologyStrategy.java
index 756b689,78f5b06..442e6cf
--- a/src/java/org/apache/cassandra/locator/NetworkTopologyStrategy.java
+++ b/src/java/org/apache/cassandra/locator/NetworkTopologyStrategy.java
@@@ -27,8 -28,8 +28,9 @@@ import org.apache.cassandra.config.Data
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.locator.TokenMetadata.Topology;
+ import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.Pair;
import com.google.common.collect.Multimap;
@@@ -214,16 -185,55 +216,55 @@@ public class NetworkTopologyStrategy ex
return datacenters.keySet();
}
- public void validateOptions() throws ConfigurationException
- {
- for (Entry<String, String> e : this.configOptions.entrySet())
- {
- if (e.getKey().equalsIgnoreCase("replication_factor"))
- throw new ConfigurationException("replication_factor is an option for SimpleStrategy, not NetworkTopologyStrategy");
- validateReplicationFactor(e.getValue());
- }
- }
-
+ /*
+ * (non-javadoc) Method to generate list of valid data center names to be used to validate the replication parameters during CREATE / ALTER keyspace operations.
+ * All peers of current node are fetched from {@link TokenMetadata} and then a set is build by fetching DC name of each peer.
+ * @return a set of valid DC names
+ */
+ private static Set<String> buildValidDataCentersSet()
+ {
+ final Set<String> validDataCenters = new HashSet<>();
+ final IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
+
+ // Add data center of localhost.
+ validDataCenters.add(snitch.getDatacenter(FBUtilities.getBroadcastAddress()));
+ // Fetch and add DCs of all peers.
+ for (final InetAddress peer : StorageService.instance.getTokenMetadata().getAllEndpoints())
+ {
+ validDataCenters.add(snitch.getDatacenter(peer));
+ }
+
+ return validDataCenters;
+ }
+
+ public Collection<String> recognizedOptions()
+ {
+ // only valid options are valid DC names.
+ return buildValidDataCentersSet();
+ }
+
+ protected void validateExpectedOptions() throws ConfigurationException
+ {
+ // Do not accept query with no data centers specified.
+ if (this.configOptions.isEmpty())
+ {
+ throw new ConfigurationException("Configuration for at least one datacenter must be present");
+ }
+
+ // Validate the data center names
+ super.validateExpectedOptions();
+ }
+
+ public void validateOptions() throws ConfigurationException
+ {
+ for (Entry<String, String> e : this.configOptions.entrySet())
+ {
+ if (e.getKey().equalsIgnoreCase("replication_factor"))
+ throw new ConfigurationException("replication_factor is an option for SimpleStrategy, not NetworkTopologyStrategy");
+ validateReplicationFactor(e.getValue());
+ }
+ }
+
@Override
public boolean hasSameSettings(AbstractReplicationStrategy other)
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d45f323e/test/unit/org/apache/cassandra/cql3/CQLTester.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/cql3/CQLTester.java
index 3bb753f,69a0b79..29d51a8
--- a/test/unit/org/apache/cassandra/cql3/CQLTester.java
+++ b/test/unit/org/apache/cassandra/cql3/CQLTester.java
@@@ -57,6 -56,8 +57,7 @@@ import org.apache.cassandra.dht.Murmur3
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.SyntaxException;
import org.apache.cassandra.io.util.FileUtils;
+ import org.apache.cassandra.locator.AbstractEndpointSnitch;
-import org.apache.cassandra.locator.IEndpointSnitch;
import org.apache.cassandra.serializers.TypeSerializer;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.service.QueryState;
@@@ -113,8 -114,11 +116,16 @@@ public abstract class CQLTeste
}
PROTOCOL_VERSIONS = builder.build();
- // Once per-JVM is enough
- prepareServer();
-
nativeAddr = InetAddress.getLoopbackAddress();
++ // Register an EndpointSnitch which returns fixed values for test.
++ DatabaseDescriptor.setEndpointSnitch(new AbstractEndpointSnitch()
++ {
++ @Override public String getRack(InetAddress endpoint) { return RACK1; }
++ @Override public String getDatacenter(InetAddress endpoint) { return DATA_CENTER; }
++ @Override public int compareEndpoints(InetAddress target, InetAddress a1, InetAddress a2) { return 0; }
++ });
++
try
{
try (ServerSocket serverSocket = new ServerSocket(0))
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d45f323e/test/unit/org/apache/cassandra/cql3/validation/entities/SecondaryIndexTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d45f323e/test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java
index c9be678,48108cd..672b6eb
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java
@@@ -218,7 -218,7 +218,7 @@@ public class AlterTest extends CQLTeste
assertRowsIgnoringOrderAndExtra(execute("SELECT keyspace_name, durable_writes, replication FROM system_schema.keyspaces"),
row(KEYSPACE, true, map("class", "org.apache.cassandra.locator.SimpleStrategy", "replication_factor", "1")),
row(KEYSPACE_PER_TEST, true, map("class", "org.apache.cassandra.locator.SimpleStrategy", "replication_factor", "1")),
- row(ks1, false, map("class", "org.apache.cassandra.locator.NetworkTopologyStrategy", "dc1", "1")),
- row(ks1, false, map("class", "org.apache.cassandra.locator.NetworkTopologyStrategy", DATA_CENTER , "1")),
++ row(ks1, false, map("class", "org.apache.cassandra.locator.NetworkTopologyStrategy", DATA_CENTER, "1")),
row(ks2, true, map("class", "org.apache.cassandra.locator.SimpleStrategy", "replication_factor", "1")));
execute("USE " + ks1);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d45f323e/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java
index da0824f,0781169..4957b18
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java
@@@ -23,9 -24,10 +24,10 @@@ import java.util.UUID
import org.junit.Test;
-
import org.apache.cassandra.config.CFMetaData;
+ import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.config.SchemaConstants;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.partitions.Partition;
@@@ -495,6 -526,34 +526,34 @@@ public class CreateTest extends CQLTest
}
@Test
+ // tests CASSANDRA-4278
+ public void testHyphenDatacenters() throws Throwable
+ {
+ IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
+
+ // Register an EndpointSnitch which returns fixed values for test.
+ DatabaseDescriptor.setEndpointSnitch(new AbstractEndpointSnitch()
+ {
+ @Override
+ public String getRack(InetAddress endpoint) { return RACK1; }
+
+ @Override
+ public String getDatacenter(InetAddress endpoint) { return "us-east-1"; }
+
+ @Override
+ public int compareEndpoints(InetAddress target, InetAddress a1, InetAddress a2) { return 0; }
+ });
+
+ execute("CREATE KEYSPACE Foo WITH replication = { 'class' : 'NetworkTopologyStrategy', 'us-east-1' : 1 };");
+
+ // Restore the previous EndpointSnitch
+ DatabaseDescriptor.setEndpointSnitch(snitch);
+
- // Clean up
++ // clean up
+ execute("DROP KEYSPACE IF EXISTS Foo");
+ }
+
+ @Test
// tests CASSANDRA-9565
public void testDoubleWith() throws Throwable
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d45f323e/test/unit/org/apache/cassandra/dht/BootStrapperTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d45f323e/test/unit/org/apache/cassandra/service/MoveTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/service/MoveTest.java
index 05757c0,6c07a47..e8d5ccd
--- a/test/unit/org/apache/cassandra/service/MoveTest.java
+++ b/test/unit/org/apache/cassandra/service/MoveTest.java
@@@ -82,9 -82,8 +82,9 @@@ public class MoveTes
* So instead of extending SchemaLoader, we call it's method below.
*/
@BeforeClass
- public static void setup() throws ConfigurationException
+ public static void setup() throws Exception
{
+ DatabaseDescriptor.daemonInitialization();
oldPartitioner = StorageService.instance.setPartitionerUnsafe(partitioner);
SchemaLoader.loadSchema();
SchemaLoader.schemaDefinition("MoveTest");
@@@ -140,6 -139,11 +140,11 @@@
}
});
+ final TokenMetadata tmd = StorageService.instance.getTokenMetadata();
- tmd.clearUnsafe();
- tmd.updateHostId(UUID.randomUUID(), InetAddress.getByName("127.0.0.1"));
- tmd.updateHostId(UUID.randomUUID(), InetAddress.getByName("127.0.0.2"));
++ tmd.clearUnsafe();
++ tmd.updateHostId(UUID.randomUUID(), InetAddress.getByName("127.0.0.1"));
++ tmd.updateHostId(UUID.randomUUID(), InetAddress.getByName("127.0.0.2"));
+
KeyspaceMetadata keyspace = KeyspaceMetadata.create(keyspaceName,
KeyspaceParams.nts(configOptions(replicas)),
Tables.of(CFMetaData.Builder.create(keyspaceName, "CF1")
[2/3] cassandra git commit: Reject invalid DC names as option while
creating or altering NetworkTopologyStrategy
Posted by jj...@apache.org.
Reject invalid DC names as option while creating or altering NetworkTopologyStrategy
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/f2c5ad74
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/f2c5ad74
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/f2c5ad74
Branch: refs/heads/trunk
Commit: f2c5ad743933498e60e7eef55e8daaa6ce338a03
Parents: 21d8a7d
Author: Nachiket Patil <na...@apple.com>
Authored: Fri Aug 5 16:05:34 2016 -0700
Committer: Jeff Jirsa <je...@crowdstrike.com>
Committed: Tue Sep 27 18:16:26 2016 -0700
----------------------------------------------------------------------
CHANGES.txt | 1 +
NEWS.txt | 11 ++++
.../locator/AbstractReplicationStrategy.java | 2 +-
.../locator/NetworkTopologyStrategy.java | 39 ++++++++++++-
.../org/apache/cassandra/cql3/CQLTester.java | 11 ++++
.../validation/entities/SecondaryIndexTest.java | 10 ----
.../cql3/validation/operations/AlterTest.java | 47 +++++++++++++++-
.../cql3/validation/operations/CreateTest.java | 59 ++++++++++++++++++++
.../apache/cassandra/dht/BootStrapperTest.java | 10 +++-
.../org/apache/cassandra/service/MoveTest.java | 9 ++-
10 files changed, 181 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f2c5ad74/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 4280abd..6edc491 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -6,6 +6,7 @@
* Extend ColumnIdentifier.internedInstances key to include the type that generated the byte buffer (CASSANDRA-12516)
* Backport CASSANDRA-10756 (race condition in NativeTransportService shutdown) (CASSANDRA-12472)
* If CF has no clustering columns, any row cache is full partition cache (CASSANDRA-12499)
+ * Reject invalid replication settings when creating or altering a keyspace (CASSANDRA-12681)
Merged from 2.2:
* Fix exceptions when enabling gossip on nodes that haven't joined the ring (CASSANDRA-12253)
* Fix authentication problem when invoking clqsh copy from a SOURCE command (CASSANDRA-12642)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f2c5ad74/NEWS.txt
----------------------------------------------------------------------
diff --git a/NEWS.txt b/NEWS.txt
index 0bd3920..b97a420 100644
--- a/NEWS.txt
+++ b/NEWS.txt
@@ -13,6 +13,17 @@ restore snapshots created with the previous major version using the
'sstableloader' tool. You can upgrade the file format of your snapshots
using the provided 'sstableupgrade' tool.
+3.0.10
+=====
+
+Upgrading
+---------
+ - To protect against accidental data loss, cassandra no longer allows
+ users to set arbitrary datacenter names for NetworkTopologyStrategy.
+ Cassandra will allow users to continue using existing keyspaces
+ with invalid datacenter names, but will validat DC names on CREATE and
+ ALTER
+
3.0.9
=====
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f2c5ad74/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java b/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java
index c90c6a1..d72c0c2 100644
--- a/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java
+++ b/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java
@@ -319,7 +319,7 @@ public abstract class AbstractReplicationStrategy
}
}
- private void validateExpectedOptions() throws ConfigurationException
+ protected void validateExpectedOptions() throws ConfigurationException
{
Collection expectedOptions = recognizedOptions();
if (expectedOptions == null)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f2c5ad74/src/java/org/apache/cassandra/locator/NetworkTopologyStrategy.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/locator/NetworkTopologyStrategy.java b/src/java/org/apache/cassandra/locator/NetworkTopologyStrategy.java
index 7c8d95e..78f5b06 100644
--- a/src/java/org/apache/cassandra/locator/NetworkTopologyStrategy.java
+++ b/src/java/org/apache/cassandra/locator/NetworkTopologyStrategy.java
@@ -24,9 +24,11 @@ import java.util.Map.Entry;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.locator.TokenMetadata.Topology;
+import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.FBUtilities;
import com.google.common.collect.Multimap;
@@ -193,10 +195,43 @@ public class NetworkTopologyStrategy extends AbstractReplicationStrategy
}
}
+ /*
+ * (non-javadoc) Method to generate list of valid data center names to be used to validate the replication parameters during CREATE / ALTER keyspace operations.
+ * All peers of current node are fetched from {@link TokenMetadata} and then a set is build by fetching DC name of each peer.
+ * @return a set of valid DC names
+ */
+ private static Set<String> buildValidDataCentersSet()
+ {
+ final Set<String> validDataCenters = new HashSet<>();
+ final IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
+
+ // Add data center of localhost.
+ validDataCenters.add(snitch.getDatacenter(FBUtilities.getBroadcastAddress()));
+ // Fetch and add DCs of all peers.
+ for (final InetAddress peer : StorageService.instance.getTokenMetadata().getAllEndpoints())
+ {
+ validDataCenters.add(snitch.getDatacenter(peer));
+ }
+
+ return validDataCenters;
+ }
+
public Collection<String> recognizedOptions()
{
- // We explicitely allow all options
- return null;
+ // only valid options are valid DC names.
+ return buildValidDataCentersSet();
+ }
+
+ protected void validateExpectedOptions() throws ConfigurationException
+ {
+ // Do not accept query with no data centers specified.
+ if (this.configOptions.isEmpty())
+ {
+ throw new ConfigurationException("Configuration for at least one datacenter must be present");
+ }
+
+ // Validate the data center names
+ super.validateExpectedOptions();
}
@Override
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f2c5ad74/test/unit/org/apache/cassandra/cql3/CQLTester.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/cql3/CQLTester.java b/test/unit/org/apache/cassandra/cql3/CQLTester.java
index 7f5eb02..69a0b79 100644
--- a/test/unit/org/apache/cassandra/cql3/CQLTester.java
+++ b/test/unit/org/apache/cassandra/cql3/CQLTester.java
@@ -56,6 +56,8 @@ import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.SyntaxException;
import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.locator.AbstractEndpointSnitch;
+import org.apache.cassandra.locator.IEndpointSnitch;
import org.apache.cassandra.serializers.TypeSerializer;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.service.QueryState;
@@ -82,6 +84,8 @@ public abstract class CQLTester
protected static final long ROW_CACHE_SIZE_IN_MB = Integer.valueOf(System.getProperty("cassandra.test.row_cache_size_in_mb", "0"));
private static final AtomicInteger seqNumber = new AtomicInteger();
protected static final ByteBuffer TOO_BIG = ByteBuffer.allocate(FBUtilities.MAX_UNSIGNED_SHORT + 1024);
+ public static final String DATA_CENTER = "datacenter1";
+ public static final String RACK1 = "rack1";
private static org.apache.cassandra.transport.Server server;
protected static final int nativePort;
@@ -127,6 +131,13 @@ public abstract class CQLTester
{
throw new RuntimeException(e);
}
+ // Register an EndpointSnitch which returns fixed values for test.
+ DatabaseDescriptor.setEndpointSnitch(new AbstractEndpointSnitch()
+ {
+ @Override public String getRack(InetAddress endpoint) { return RACK1; }
+ @Override public String getDatacenter(InetAddress endpoint) { return DATA_CENTER; }
+ @Override public int compareEndpoints(InetAddress target, InetAddress a1, InetAddress a2) { return 0; }
+ });
}
public static ResultMessage lastSchemaChangeResult;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f2c5ad74/test/unit/org/apache/cassandra/cql3/validation/entities/SecondaryIndexTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/SecondaryIndexTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/SecondaryIndexTest.java
index 0cf13bd..2b31481 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/SecondaryIndexTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/SecondaryIndexTest.java
@@ -239,16 +239,6 @@ public class SecondaryIndexTest extends CQLTester
}
/**
- * Check one can use arbitrary name for datacenter when creating keyspace (#4278),
- * migrated from cql_tests.py:TestCQL.keyspace_creation_options_test()
- */
- @Test
- public void testDataCenterName() throws Throwable
- {
- execute("CREATE KEYSPACE Foo WITH replication = { 'class' : 'NetworkTopologyStrategy', 'us-east' : 1, 'us-west' : 1 };");
- }
-
- /**
* Migrated from cql_tests.py:TestCQL.indexes_composite_test()
*/
@Test
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f2c5ad74/test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java
index bcd6587..48108cd 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java
@@ -212,13 +212,13 @@ public class AlterTest extends CQLTester
row(ks1, true),
row(ks2, false));
- schemaChange("ALTER KEYSPACE " + ks1 + " WITH replication = { 'class' : 'NetworkTopologyStrategy', 'dc1' : 1 } AND durable_writes=False");
+ schemaChange("ALTER KEYSPACE " + ks1 + " WITH replication = { 'class' : 'NetworkTopologyStrategy', '" + DATA_CENTER + "' : 1 } AND durable_writes=False");
schemaChange("ALTER KEYSPACE " + ks2 + " WITH durable_writes=true");
assertRowsIgnoringOrderAndExtra(execute("SELECT keyspace_name, durable_writes, replication FROM system_schema.keyspaces"),
row(KEYSPACE, true, map("class", "org.apache.cassandra.locator.SimpleStrategy", "replication_factor", "1")),
row(KEYSPACE_PER_TEST, true, map("class", "org.apache.cassandra.locator.SimpleStrategy", "replication_factor", "1")),
- row(ks1, false, map("class", "org.apache.cassandra.locator.NetworkTopologyStrategy", "dc1", "1")),
+ row(ks1, false, map("class", "org.apache.cassandra.locator.NetworkTopologyStrategy", DATA_CENTER , "1")),
row(ks2, true, map("class", "org.apache.cassandra.locator.SimpleStrategy", "replication_factor", "1")));
execute("USE " + ks1);
@@ -233,6 +233,49 @@ public class AlterTest extends CQLTester
}
/**
+ * Test {@link ConfigurationException} thrown on alter keyspace to no DC option in replication configuration.
+ */
+ @Test
+ public void testAlterKeyspaceWithNoOptionThrowsConfigurationException() throws Throwable
+ {
+ // Create keyspaces
+ execute("CREATE KEYSPACE testABC WITH replication={ 'class' : 'NetworkTopologyStrategy', '" + DATA_CENTER + "' : 3 }");
+ execute("CREATE KEYSPACE testXYZ WITH replication={ 'class' : 'SimpleStrategy', 'replication_factor' : 3 }");
+
+ // Try to alter the created keyspace without any option
+ assertInvalidThrow(ConfigurationException.class, "ALTER KEYSPACE testABC WITH replication={ 'class' : 'NetworkTopologyStrategy' }");
+ assertInvalidThrow(ConfigurationException.class, "ALTER KEYSPACE testXYZ WITH replication={ 'class' : 'SimpleStrategy' }");
+
+ // Make sure that the alter works as expected
+ execute("ALTER KEYSPACE testABC WITH replication={ 'class' : 'NetworkTopologyStrategy', '" + DATA_CENTER + "' : 2 }");
+ execute("ALTER KEYSPACE testXYZ WITH replication={ 'class' : 'SimpleStrategy', 'replication_factor' : 2 }");
+
+ // clean up
+ execute("DROP KEYSPACE IF EXISTS testABC");
+ execute("DROP KEYSPACE IF EXISTS testXYZ");
+ }
+
+ /**
+ * Test {@link ConfigurationException} thrown when altering a keyspace to invalid DC option in replication configuration.
+ */
+ @Test
+ public void testAlterKeyspaceWithNTSOnlyAcceptsConfiguredDataCenterNames() throws Throwable
+ {
+ // Create a keyspace with expected DC name.
+ execute("CREATE KEYSPACE testABC WITH replication = {'class' : 'NetworkTopologyStrategy', '" + DATA_CENTER + "' : 2 }");
+
+ // try modifying the keyspace
+ assertInvalidThrow(ConfigurationException.class, "ALTER KEYSPACE testABC WITH replication = { 'class' : 'NetworkTopologyStrategy', 'INVALID_DC' : 2 }");
+ execute("ALTER KEYSPACE testABC WITH replication = {'class' : 'NetworkTopologyStrategy', '" + DATA_CENTER + "' : 3 }");
+
+ // Mix valid and invalid, should throw an exception
+ assertInvalidThrow(ConfigurationException.class, "ALTER KEYSPACE testABC WITH replication={ 'class' : 'NetworkTopologyStrategy', '" + DATA_CENTER + "' : 2 , 'INVALID_DC': 1}");
+
+ // clean-up
+ execute("DROP KEYSPACE IF EXISTS testABC");
+ }
+
+ /**
* Test for bug of 5232,
* migrated from cql_tests.py:TestCQL.alter_bug_test()
*/
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f2c5ad74/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java
index 33a41d8..0781169 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.cassandra.cql3.validation.operations;
+import java.net.InetAddress;
import java.util.Collection;
import java.util.Collections;
import java.util.UUID;
@@ -25,12 +26,15 @@ import org.junit.Test;
import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.Schema;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.partitions.Partition;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.SyntaxException;
+import org.apache.cassandra.locator.AbstractEndpointSnitch;
+import org.apache.cassandra.locator.IEndpointSnitch;
import org.apache.cassandra.schema.SchemaKeyspace;
import org.apache.cassandra.triggers.ITrigger;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -345,6 +349,33 @@ public class CreateTest extends CQLTester
}
/**
+ * Test {@link ConfigurationException} is thrown on create keyspace with invalid DC option in replication configuration .
+ */
+ @Test
+ public void testCreateKeyspaceWithNTSOnlyAcceptsConfiguredDataCenterNames() throws Throwable
+ {
+ assertInvalidThrow(ConfigurationException.class, "CREATE KEYSPACE testABC WITH replication = { 'class' : 'NetworkTopologyStrategy', 'INVALID_DC' : 2 }");
+ execute("CREATE KEYSPACE testABC WITH replication = {'class' : 'NetworkTopologyStrategy', '" + DATA_CENTER + "' : 2 }");
+
+ // Mix valid and invalid, should throw an exception
+ assertInvalidThrow(ConfigurationException.class, "CREATE KEYSPACE testXYZ WITH replication={ 'class' : 'NetworkTopologyStrategy', '" + DATA_CENTER + "' : 2 , 'INVALID_DC': 1}");
+
+ // clean-up
+ execute("DROP KEYSPACE IF EXISTS testABC");
+ execute("DROP KEYSPACE IF EXISTS testXYZ");
+ }
+
+ /**
+ * Test {@link ConfigurationException} is thrown on create keyspace without any options.
+ */
+ @Test
+ public void testConfigurationExceptionThrownWhenCreateKeyspaceWithNoOptions() throws Throwable
+ {
+ assertInvalidThrow(ConfigurationException.class, "CREATE KEYSPACE testXYZ with replication = { 'class': 'NetworkTopologyStrategy' }");
+ assertInvalidThrow(ConfigurationException.class, "CREATE KEYSPACE testXYZ WITH replication = { 'class' : 'SimpleStrategy' }");
+ }
+
+ /**
* Test create and drop table
* migrated from cql_tests.py:TestCQL.table_test()
*/
@@ -495,6 +526,34 @@ public class CreateTest extends CQLTester
}
@Test
+ // tests CASSANDRA-4278
+ public void testHyphenDatacenters() throws Throwable
+ {
+ IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
+
+ // Register an EndpointSnitch which returns fixed values for test.
+ DatabaseDescriptor.setEndpointSnitch(new AbstractEndpointSnitch()
+ {
+ @Override
+ public String getRack(InetAddress endpoint) { return RACK1; }
+
+ @Override
+ public String getDatacenter(InetAddress endpoint) { return "us-east-1"; }
+
+ @Override
+ public int compareEndpoints(InetAddress target, InetAddress a1, InetAddress a2) { return 0; }
+ });
+
+ execute("CREATE KEYSPACE Foo WITH replication = { 'class' : 'NetworkTopologyStrategy', 'us-east-1' : 1 };");
+
+ // Restore the previous EndpointSnitch
+ DatabaseDescriptor.setEndpointSnitch(snitch);
+
+ // Clean up
+ execute("DROP KEYSPACE IF EXISTS Foo");
+ }
+
+ @Test
// tests CASSANDRA-9565
public void testDoubleWith() throws Throwable
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f2c5ad74/test/unit/org/apache/cassandra/dht/BootStrapperTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/dht/BootStrapperTest.java b/test/unit/org/apache/cassandra/dht/BootStrapperTest.java
index 8974791..cd34aea 100644
--- a/test/unit/org/apache/cassandra/dht/BootStrapperTest.java
+++ b/test/unit/org/apache/cassandra/dht/BootStrapperTest.java
@@ -27,6 +27,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.UUID;
import com.google.common.collect.Lists;
@@ -59,7 +60,7 @@ public class BootStrapperTest
static IPartitioner oldPartitioner;
@BeforeClass
- public static void setup() throws ConfigurationException
+ public static void setup() throws Exception
{
oldPartitioner = StorageService.instance.setPartitionerUnsafe(Murmur3Partitioner.instance);
SchemaLoader.startGossiper();
@@ -177,6 +178,13 @@ public class BootStrapperTest
int vn = 16;
String ks = "BootStrapperTestNTSKeyspace" + rackCount + replicas;
String dc = "1";
+
+ // Register peers with expected DC for NetworkTopologyStrategy.
+ TokenMetadata metadata = StorageService.instance.getTokenMetadata();
+ metadata.clearUnsafe();
+ metadata.updateHostId(UUID.randomUUID(), InetAddress.getByName("127.1.0.99"));
+ metadata.updateHostId(UUID.randomUUID(), InetAddress.getByName("127.15.0.99"));
+
SchemaLoader.createKeyspace(ks, KeyspaceParams.nts(dc, replicas, "15", 15), SchemaLoader.standardCFMD(ks, "Standard1"));
TokenMetadata tm = new TokenMetadata();
tm.clearUnsafe();
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f2c5ad74/test/unit/org/apache/cassandra/service/MoveTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/service/MoveTest.java b/test/unit/org/apache/cassandra/service/MoveTest.java
index 53365aa..6c07a47 100644
--- a/test/unit/org/apache/cassandra/service/MoveTest.java
+++ b/test/unit/org/apache/cassandra/service/MoveTest.java
@@ -82,7 +82,7 @@ public class MoveTest
* So instead of extending SchemaLoader, we call it's method below.
*/
@BeforeClass
- public static void setup() throws ConfigurationException
+ public static void setup() throws Exception
{
oldPartitioner = StorageService.instance.setPartitionerUnsafe(partitioner);
SchemaLoader.loadSchema();
@@ -105,7 +105,7 @@ public class MoveTest
StorageService.instance.getTokenMetadata().clearUnsafe();
}
- private static void addNetworkTopologyKeyspace(String keyspaceName, Integer... replicas) throws ConfigurationException
+ private static void addNetworkTopologyKeyspace(String keyspaceName, Integer... replicas) throws Exception
{
DatabaseDescriptor.setEndpointSnitch(new AbstractNetworkTopologySnitch()
@@ -139,6 +139,11 @@ public class MoveTest
}
});
+ final TokenMetadata tmd = StorageService.instance.getTokenMetadata();
+ tmd.clearUnsafe();
+ tmd.updateHostId(UUID.randomUUID(), InetAddress.getByName("127.0.0.1"));
+ tmd.updateHostId(UUID.randomUUID(), InetAddress.getByName("127.0.0.2"));
+
KeyspaceMetadata keyspace = KeyspaceMetadata.create(keyspaceName,
KeyspaceParams.nts(configOptions(replicas)),
Tables.of(CFMetaData.Builder.create(keyspaceName, "CF1")