You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by sl...@apache.org on 2014/06/05 08:40:41 UTC
[1/2] git commit: Reduce run time for CQL tests
Repository: cassandra
Updated Branches:
refs/heads/trunk 2306fb877 -> 99ba549d0
Reduce run time for CQL tests
patch by slebresne; reviewed by thobbs for CASSANDRA-7327
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/78e91c4c
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/78e91c4c
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/78e91c4c
Branch: refs/heads/trunk
Commit: 78e91c4cacb7128a994a7a8ca16f5a9e20b576ce
Parents: 285c369
Author: Sylvain Lebresne <sy...@datastax.com>
Authored: Thu May 29 16:16:53 2014 +0200
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Thu Jun 5 08:39:48 2014 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
build.xml | 9 ++-
src/java/org/apache/cassandra/auth/Auth.java | 2 +-
.../cql3/statements/AlterKeyspaceStatement.java | 4 +-
.../cql3/statements/AlterTableStatement.java | 4 +-
.../cql3/statements/AlterTypeStatement.java | 8 +-
.../cql3/statements/CreateIndexStatement.java | 4 +-
.../statements/CreateKeyspaceStatement.java | 4 +-
.../cql3/statements/CreateTableStatement.java | 4 +-
.../cql3/statements/CreateTriggerStatement.java | 4 +-
.../cql3/statements/CreateTypeStatement.java | 4 +-
.../cql3/statements/DropIndexStatement.java | 4 +-
.../cql3/statements/DropKeyspaceStatement.java | 4 +-
.../cql3/statements/DropTableStatement.java | 4 +-
.../cql3/statements/DropTriggerStatement.java | 4 +-
.../cql3/statements/DropTypeStatement.java | 4 +-
.../statements/SchemaAlteringStatement.java | 16 +++-
.../org/apache/cassandra/db/DefsTables.java | 10 ++-
src/java/org/apache/cassandra/db/Memtable.java | 7 +-
.../cassandra/service/MigrationManager.java | 85 +++++++++++++++++---
.../cassandra/service/StorageService.java | 2 +-
.../unit/org/apache/cassandra/SchemaLoader.java | 7 +-
.../org/apache/cassandra/cql3/CQLTester.java | 74 +++++++++++++++--
23 files changed, 202 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 23f36c8..41ee6e5 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -3,6 +3,7 @@
* Make sure we clear out repair sessions from netstats (CASSANDRA-7329)
* Don't fail streams on failure detector downs (CASSANDRA-3569)
* Add optional keyspace to DROP INDEX statement (CASSANDRA-7314)
+ * Reduce run time for CQL tests (CASSANDRA-7327)
Merged from 2.0:
* Add per-CF range read request latency metrics (CASSANDRA-7338)
* Fix NPE in StreamTransferTask.createMessageForRetry() (CASSANDRA-7323)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/build.xml
----------------------------------------------------------------------
diff --git a/build.xml b/build.xml
index 9f373c4..7e34904 100644
--- a/build.xml
+++ b/build.xml
@@ -1078,6 +1078,8 @@
<jvmarg value="-javaagent:${basedir}/lib/jamm-0.2.6.jar" />
<jvmarg value="-ea"/>
<jvmarg value="-Xss256k"/>
+ <jvmarg value="-Dcassandra.memtable_row_overhead_computation_step=100"/>
+ <jvmarg value="-Dcassandra.test.use_prepared=${cassandra.test.use_prepared}"/>
<optjvmargs/>
<classpath>
<path refid="cassandra.classpath" />
@@ -1206,8 +1208,8 @@
<target name="cql-test" depends="build-test" description="Execute CQL tests">
<testmacro suitename="cql" inputdir="${test.unit.src}"
- timeout="${test.timeout}" filter="**/cql3/*Test.java">
- <jvmarg value="-Dcassandra.test.use_prepared=${cassandra.test.use_prepared}"/>
+ timeout="${test.timeout}" filter="**/cql3/*Test.java"
+ forkmode="once">
</testmacro>
</target>
@@ -1216,7 +1218,7 @@
<echo message="running ${test.methods} tests from ${test.name}"/>
<mkdir dir="${build.test.dir}/cassandra"/>
<mkdir dir="${build.test.dir}/output"/>
- <junit fork="on" failureproperty="testfailed" maxmemory="1024m" timeout="${test.timeout}">
+ <junit fork="on" forkmode="once" failureproperty="testfailed" maxmemory="1024m" timeout="${test.timeout}">
<sysproperty key="net.sourceforge.cobertura.datafile" file="${cobertura.datafile}"/>
<formatter type="brief" usefile="false"/>
<jvmarg value="-Dstorage-config=${test.conf}"/>
@@ -1225,6 +1227,7 @@
<jvmarg value="-ea"/>
<jvmarg value="-Xss256k"/>
<jvmarg value="-Dcassandra.test.use_prepared=${cassandra.test.use_prepared}"/>
+ <jvmarg value="-Dcassandra.memtable_row_overhead_computation_step=100"/>
<classpath>
<path refid="cassandra.classpath" />
<pathelement location="${test.classes}"/>
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/src/java/org/apache/cassandra/auth/Auth.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/auth/Auth.java b/src/java/org/apache/cassandra/auth/Auth.java
index 587fc66..7c532b0 100644
--- a/src/java/org/apache/cassandra/auth/Auth.java
+++ b/src/java/org/apache/cassandra/auth/Auth.java
@@ -181,7 +181,7 @@ public class Auth
try
{
KSMetaData ksm = KSMetaData.newKeyspace(AUTH_KS, SimpleStrategy.class.getName(), ImmutableMap.of("replication_factor", "1"), true);
- MigrationManager.announceNewKeyspace(ksm, 0);
+ MigrationManager.announceNewKeyspace(ksm, 0, false);
}
catch (Exception e)
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/src/java/org/apache/cassandra/cql3/statements/AlterKeyspaceStatement.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cql3/statements/AlterKeyspaceStatement.java b/src/java/org/apache/cassandra/cql3/statements/AlterKeyspaceStatement.java
index 39d1cde..a940f9e 100644
--- a/src/java/org/apache/cassandra/cql3/statements/AlterKeyspaceStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/AlterKeyspaceStatement.java
@@ -79,14 +79,14 @@ public class AlterKeyspaceStatement extends SchemaAlteringStatement
}
}
- public void announceMigration() throws RequestValidationException
+ public void announceMigration(boolean isLocalOnly) throws RequestValidationException
{
KSMetaData ksm = Schema.instance.getKSMetaData(name);
// In the (very) unlikely case the keyspace was dropped since validate()
if (ksm == null)
throw new InvalidRequestException("Unknown keyspace " + name);
- MigrationManager.announceKeyspaceUpdate(attrs.asKSMetadataUpdate(ksm));
+ MigrationManager.announceKeyspaceUpdate(attrs.asKSMetadataUpdate(ksm), isLocalOnly);
}
public ResultMessage.SchemaChange.Change changeType()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/src/java/org/apache/cassandra/cql3/statements/AlterTableStatement.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cql3/statements/AlterTableStatement.java b/src/java/org/apache/cassandra/cql3/statements/AlterTableStatement.java
index 27acb3c..cff0e8f 100644
--- a/src/java/org/apache/cassandra/cql3/statements/AlterTableStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/AlterTableStatement.java
@@ -74,7 +74,7 @@ public class AlterTableStatement extends SchemaAlteringStatement
// validated in announceMigration()
}
- public void announceMigration() throws RequestValidationException
+ public void announceMigration(boolean isLocalOnly) throws RequestValidationException
{
CFMetaData meta = validateColumnFamily(keyspace(), columnFamily());
CFMetaData cfm = meta.copy();
@@ -237,7 +237,7 @@ public class AlterTableStatement extends SchemaAlteringStatement
break;
}
- MigrationManager.announceColumnFamilyUpdate(cfm, false);
+ MigrationManager.announceColumnFamilyUpdate(cfm, false, isLocalOnly);
}
public String toString()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/src/java/org/apache/cassandra/cql3/statements/AlterTypeStatement.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cql3/statements/AlterTypeStatement.java b/src/java/org/apache/cassandra/cql3/statements/AlterTypeStatement.java
index 1996457..3cac7cc 100644
--- a/src/java/org/apache/cassandra/cql3/statements/AlterTypeStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/AlterTypeStatement.java
@@ -89,7 +89,7 @@ public abstract class AlterTypeStatement extends SchemaAlteringStatement
return name.getKeyspace();
}
- public void announceMigration() throws InvalidRequestException, ConfigurationException
+ public void announceMigration(boolean isLocalOnly) throws InvalidRequestException, ConfigurationException
{
KSMetaData ksm = Schema.instance.getKSMetaData(name.getKeyspace());
if (ksm == null)
@@ -104,7 +104,7 @@ public abstract class AlterTypeStatement extends SchemaAlteringStatement
// Now, we need to announce the type update to basically change it for new tables using this type,
// but we also need to find all existing user types and CF using it and change them.
- MigrationManager.announceTypeUpdate(updated);
+ MigrationManager.announceTypeUpdate(updated, isLocalOnly);
for (KSMetaData ksm2 : Schema.instance.getKeyspaceDefinitions())
{
@@ -115,7 +115,7 @@ public abstract class AlterTypeStatement extends SchemaAlteringStatement
for (ColumnDefinition def : copy.allColumns())
modified |= updateDefinition(copy, def, toUpdate.keyspace, toUpdate.name, updated);
if (modified)
- MigrationManager.announceColumnFamilyUpdate(copy, false);
+ MigrationManager.announceColumnFamilyUpdate(copy, false, isLocalOnly);
}
// Other user types potentially using the updated type
@@ -131,7 +131,7 @@ public abstract class AlterTypeStatement extends SchemaAlteringStatement
}
AbstractType<?> upd = updateWith(ut, toUpdate.keyspace, toUpdate.name, updated);
if (upd != null)
- MigrationManager.announceTypeUpdate((UserType)upd);
+ MigrationManager.announceTypeUpdate((UserType)upd, isLocalOnly);
}
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/src/java/org/apache/cassandra/cql3/statements/CreateIndexStatement.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cql3/statements/CreateIndexStatement.java b/src/java/org/apache/cassandra/cql3/statements/CreateIndexStatement.java
index ea1f933..b2e8d59 100644
--- a/src/java/org/apache/cassandra/cql3/statements/CreateIndexStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/CreateIndexStatement.java
@@ -117,7 +117,7 @@ public class CreateIndexStatement extends SchemaAlteringStatement
throw new InvalidRequestException(String.format("Cannot add secondary index to already primarily indexed column %s", target.column));
}
- public void announceMigration() throws RequestValidationException
+ public void announceMigration(boolean isLocalOnly) throws RequestValidationException
{
logger.debug("Updating column {} definition for index {}", target.column, indexName);
CFMetaData cfm = Schema.instance.getCFMetaData(keyspace(), columnFamily()).copy();
@@ -147,7 +147,7 @@ public class CreateIndexStatement extends SchemaAlteringStatement
cd.setIndexName(indexName);
cfm.addDefaultIndexNames();
- MigrationManager.announceColumnFamilyUpdate(cfm, false);
+ MigrationManager.announceColumnFamilyUpdate(cfm, false, isLocalOnly);
}
public ResultMessage.SchemaChange.Change changeType()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/src/java/org/apache/cassandra/cql3/statements/CreateKeyspaceStatement.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cql3/statements/CreateKeyspaceStatement.java b/src/java/org/apache/cassandra/cql3/statements/CreateKeyspaceStatement.java
index 2ed1d91..c528e16 100644
--- a/src/java/org/apache/cassandra/cql3/statements/CreateKeyspaceStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/CreateKeyspaceStatement.java
@@ -97,11 +97,11 @@ public class CreateKeyspaceStatement extends SchemaAlteringStatement
attrs.getReplicationOptions());
}
- public void announceMigration() throws RequestValidationException
+ public void announceMigration(boolean isLocalOnly) throws RequestValidationException
{
try
{
- MigrationManager.announceNewKeyspace(attrs.asKSMetadata(name));
+ MigrationManager.announceNewKeyspace(attrs.asKSMetadata(name), isLocalOnly);
}
catch (AlreadyExistsException e)
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/src/java/org/apache/cassandra/cql3/statements/CreateTableStatement.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cql3/statements/CreateTableStatement.java b/src/java/org/apache/cassandra/cql3/statements/CreateTableStatement.java
index 7cef999..d764845 100644
--- a/src/java/org/apache/cassandra/cql3/statements/CreateTableStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/CreateTableStatement.java
@@ -105,11 +105,11 @@ public class CreateTableStatement extends SchemaAlteringStatement
return columnDefs;
}
- public void announceMigration() throws RequestValidationException
+ public void announceMigration(boolean isLocalOnly) throws RequestValidationException
{
try
{
- MigrationManager.announceNewColumnFamily(getCFMetaData());
+ MigrationManager.announceNewColumnFamily(getCFMetaData(), isLocalOnly);
}
catch (AlreadyExistsException e)
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/src/java/org/apache/cassandra/cql3/statements/CreateTriggerStatement.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cql3/statements/CreateTriggerStatement.java b/src/java/org/apache/cassandra/cql3/statements/CreateTriggerStatement.java
index 0e14d2e..e99da8a 100644
--- a/src/java/org/apache/cassandra/cql3/statements/CreateTriggerStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/CreateTriggerStatement.java
@@ -65,12 +65,12 @@ public class CreateTriggerStatement extends SchemaAlteringStatement
}
}
- public void announceMigration() throws ConfigurationException
+ public void announceMigration(boolean isLocalOnly) throws ConfigurationException
{
CFMetaData cfm = Schema.instance.getCFMetaData(keyspace(), columnFamily()).copy();
cfm.addTriggerDefinition(TriggerDefinition.create(triggerName, triggerClass));
logger.info("Adding trigger with name {} and class {}", triggerName, triggerClass);
- MigrationManager.announceColumnFamilyUpdate(cfm, false);
+ MigrationManager.announceColumnFamilyUpdate(cfm, false, isLocalOnly);
}
public ResultMessage.SchemaChange.Change changeType()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/src/java/org/apache/cassandra/cql3/statements/CreateTypeStatement.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cql3/statements/CreateTypeStatement.java b/src/java/org/apache/cassandra/cql3/statements/CreateTypeStatement.java
index cc5447e..41116f5 100644
--- a/src/java/org/apache/cassandra/cql3/statements/CreateTypeStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/CreateTypeStatement.java
@@ -111,7 +111,7 @@ public class CreateTypeStatement extends SchemaAlteringStatement
return new UserType(name.getKeyspace(), name.getUserTypeName(), names, types);
}
- public void announceMigration() throws InvalidRequestException, ConfigurationException
+ public void announceMigration(boolean isLocalOnly) throws InvalidRequestException, ConfigurationException
{
KSMetaData ksm = Schema.instance.getKSMetaData(name.getKeyspace());
assert ksm != null; // should haven't validate otherwise
@@ -122,6 +122,6 @@ public class CreateTypeStatement extends SchemaAlteringStatement
UserType type = createType();
checkForDuplicateNames(type);
- MigrationManager.announceNewType(type);
+ MigrationManager.announceNewType(type, isLocalOnly);
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/src/java/org/apache/cassandra/cql3/statements/DropIndexStatement.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cql3/statements/DropIndexStatement.java b/src/java/org/apache/cassandra/cql3/statements/DropIndexStatement.java
index 44704a9..4321886 100644
--- a/src/java/org/apache/cassandra/cql3/statements/DropIndexStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/DropIndexStatement.java
@@ -60,14 +60,14 @@ public class DropIndexStatement extends SchemaAlteringStatement
return ResultMessage.SchemaChange.Change.UPDATED;
}
- public void announceMigration() throws InvalidRequestException, ConfigurationException
+ public void announceMigration(boolean isLocalOnly) throws InvalidRequestException, ConfigurationException
{
CFMetaData cfm = findIndexedCF();
if (cfm == null)
return;
CFMetaData updatedCfm = updateCFMetadata(cfm);
- MigrationManager.announceColumnFamilyUpdate(updatedCfm, false);
+ MigrationManager.announceColumnFamilyUpdate(updatedCfm, false, isLocalOnly);
}
private CFMetaData updateCFMetadata(CFMetaData cfm)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/src/java/org/apache/cassandra/cql3/statements/DropKeyspaceStatement.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cql3/statements/DropKeyspaceStatement.java b/src/java/org/apache/cassandra/cql3/statements/DropKeyspaceStatement.java
index 30fd964..6c99bdd 100644
--- a/src/java/org/apache/cassandra/cql3/statements/DropKeyspaceStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/DropKeyspaceStatement.java
@@ -55,11 +55,11 @@ public class DropKeyspaceStatement extends SchemaAlteringStatement
return keyspace;
}
- public void announceMigration() throws ConfigurationException
+ public void announceMigration(boolean isLocalOnly) throws ConfigurationException
{
try
{
- MigrationManager.announceKeyspaceDrop(keyspace);
+ MigrationManager.announceKeyspaceDrop(keyspace, isLocalOnly);
}
catch(ConfigurationException e)
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/src/java/org/apache/cassandra/cql3/statements/DropTableStatement.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cql3/statements/DropTableStatement.java b/src/java/org/apache/cassandra/cql3/statements/DropTableStatement.java
index d27261c..443c6d3 100644
--- a/src/java/org/apache/cassandra/cql3/statements/DropTableStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/DropTableStatement.java
@@ -54,11 +54,11 @@ public class DropTableStatement extends SchemaAlteringStatement
// validated in announceMigration()
}
- public void announceMigration() throws ConfigurationException
+ public void announceMigration(boolean isLocalOnly) throws ConfigurationException
{
try
{
- MigrationManager.announceColumnFamilyDrop(keyspace(), columnFamily());
+ MigrationManager.announceColumnFamilyDrop(keyspace(), columnFamily(), isLocalOnly);
}
catch (ConfigurationException e)
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/src/java/org/apache/cassandra/cql3/statements/DropTriggerStatement.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cql3/statements/DropTriggerStatement.java b/src/java/org/apache/cassandra/cql3/statements/DropTriggerStatement.java
index c9b33a1..ff7948f 100644
--- a/src/java/org/apache/cassandra/cql3/statements/DropTriggerStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/DropTriggerStatement.java
@@ -53,13 +53,13 @@ public class DropTriggerStatement extends SchemaAlteringStatement
ThriftValidation.validateColumnFamily(keyspace(), columnFamily());
}
- public void announceMigration() throws ConfigurationException
+ public void announceMigration(boolean isLocalOnly) throws ConfigurationException
{
CFMetaData cfm = Schema.instance.getCFMetaData(keyspace(), columnFamily()).copy();
if (!cfm.removeTrigger(triggerName))
throw new ConfigurationException(String.format("Trigger %s was not found", triggerName));
logger.info("Dropping trigger with name {}", triggerName);
- MigrationManager.announceColumnFamilyUpdate(cfm, false);
+ MigrationManager.announceColumnFamilyUpdate(cfm, false, isLocalOnly);
}
public ResultMessage.SchemaChange.Change changeType()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/src/java/org/apache/cassandra/cql3/statements/DropTypeStatement.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cql3/statements/DropTypeStatement.java b/src/java/org/apache/cassandra/cql3/statements/DropTypeStatement.java
index 1e1ded5..4036d33 100644
--- a/src/java/org/apache/cassandra/cql3/statements/DropTypeStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/DropTypeStatement.java
@@ -137,7 +137,7 @@ public class DropTypeStatement extends SchemaAlteringStatement
return name.getKeyspace();
}
- public void announceMigration() throws InvalidRequestException, ConfigurationException
+ public void announceMigration(boolean isLocalOnly) throws InvalidRequestException, ConfigurationException
{
KSMetaData ksm = Schema.instance.getKSMetaData(name.getKeyspace());
assert ksm != null;
@@ -145,6 +145,6 @@ public class DropTypeStatement extends SchemaAlteringStatement
UserType toDrop = ksm.userTypes.getType(name.getUserTypeName());
// Can be null with ifExists
if (toDrop != null)
- MigrationManager.announceTypeDrop(toDrop);
+ MigrationManager.announceTypeDrop(toDrop, isLocalOnly);
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/src/java/org/apache/cassandra/cql3/statements/SchemaAlteringStatement.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cql3/statements/SchemaAlteringStatement.java b/src/java/org/apache/cassandra/cql3/statements/SchemaAlteringStatement.java
index 94df854..7d454a5 100644
--- a/src/java/org/apache/cassandra/cql3/statements/SchemaAlteringStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/SchemaAlteringStatement.java
@@ -64,18 +64,26 @@ public abstract class SchemaAlteringStatement extends CFStatement implements CQL
public abstract ResultMessage.SchemaChange.Change changeType();
- public abstract void announceMigration() throws RequestValidationException;
+ public abstract void announceMigration(boolean isLocalOnly) throws RequestValidationException;
public ResultMessage execute(QueryState state, QueryOptions options) throws RequestValidationException
{
- announceMigration();
+ announceMigration(false);
String tableName = cfName == null || columnFamily() == null ? "" : columnFamily();
return new ResultMessage.SchemaChange(changeType(), keyspace(), tableName);
}
public ResultMessage executeInternal(QueryState state, QueryOptions options)
{
- // executeInternal is for local query only, thus altering schema is not supported
- throw new UnsupportedOperationException();
+ try
+ {
+ announceMigration(true);
+ String tableName = cfName == null || columnFamily() == null ? "" : columnFamily();
+ return new ResultMessage.SchemaChange(changeType(), keyspace(), tableName);
+ }
+ catch (RequestValidationException e)
+ {
+ throw new RuntimeException(e);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/src/java/org/apache/cassandra/db/DefsTables.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/DefsTables.java b/src/java/org/apache/cassandra/db/DefsTables.java
index 3902e05..ede3ebd 100644
--- a/src/java/org/apache/cassandra/db/DefsTables.java
+++ b/src/java/org/apache/cassandra/db/DefsTables.java
@@ -161,6 +161,12 @@ public class DefsTables
*/
public static synchronized void mergeSchema(Collection<Mutation> mutations) throws ConfigurationException, IOException
{
+ mergeSchemaInternal(mutations, true);
+ Schema.instance.updateVersionAndAnnounce();
+ }
+
+ public static synchronized void mergeSchemaInternal(Collection<Mutation> mutations, boolean doFlush) throws ConfigurationException, IOException
+ {
// current state of the schema
Map<DecoratedKey, ColumnFamily> oldKeyspaces = SystemKeyspace.getSchema(SystemKeyspace.SCHEMA_KEYSPACES_CF);
Map<DecoratedKey, ColumnFamily> oldColumnFamilies = SystemKeyspace.getSchema(SystemKeyspace.SCHEMA_COLUMNFAMILIES_CF);
@@ -169,7 +175,7 @@ public class DefsTables
for (Mutation mutation : mutations)
mutation.apply();
- if (!StorageService.instance.isClientMode())
+ if (doFlush && !StorageService.instance.isClientMode())
flushSchemaCFs();
// with new data applied
@@ -184,8 +190,6 @@ public class DefsTables
// it is safe to drop a keyspace only when all nested ColumnFamilies where deleted
for (String keyspaceToDrop : keyspacesToDrop)
dropKeyspace(keyspaceToDrop);
-
- Schema.instance.updateVersionAndAnnounce();
}
private static Set<String> mergeKeyspaces(Map<DecoratedKey, ColumnFamily> old, Map<DecoratedKey, ColumnFamily> updated)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/src/java/org/apache/cassandra/db/Memtable.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/Memtable.java b/src/java/org/apache/cassandra/db/Memtable.java
index 417b4a7..3c02678 100644
--- a/src/java/org/apache/cassandra/db/Memtable.java
+++ b/src/java/org/apache/cassandra/db/Memtable.java
@@ -53,7 +53,7 @@ public class Memtable
private static final Logger logger = LoggerFactory.getLogger(Memtable.class);
static final MemtablePool MEMORY_POOL = DatabaseDescriptor.getMemtableAllocatorPool();
- private static final int ROW_OVERHEAD_HEAP_SIZE;
+ private static final int ROW_OVERHEAD_HEAP_SIZE = estimateRowOverhead(Integer.valueOf(System.getProperty("cassandra.memtable_row_overhead_computation_step", "100000")));
private final MemtableAllocator allocator;
private final AtomicLong liveDataSize = new AtomicLong(0);
@@ -389,14 +389,13 @@ public class Memtable
}
}
- static
+ private static int estimateRowOverhead(final int count)
{
// calculate row overhead
final OpOrder.Group group = new OpOrder().start();
int rowOverhead;
MemtableAllocator allocator = MEMORY_POOL.newAllocator();
ConcurrentNavigableMap<RowPosition, Object> rows = new ConcurrentSkipListMap<>();
- final int count = 100000;
final Object val = new Object();
for (int i = 0 ; i < count ; i++)
rows.put(allocator.clone(new BufferDecoratedKey(new LongToken((long) i), ByteBufferUtil.EMPTY_BYTE_BUFFER), group), val);
@@ -406,6 +405,6 @@ public class Memtable
rowOverhead += AtomicBTreeColumns.EMPTY_SIZE;
allocator.setDiscarding();
allocator.setDiscarded();
- ROW_OVERHEAD_HEAP_SIZE = rowOverhead;
+ return rowOverhead;
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/src/java/org/apache/cassandra/service/MigrationManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/MigrationManager.java b/src/java/org/apache/cassandra/service/MigrationManager.java
index ec46d3f..a6408ec 100644
--- a/src/java/org/apache/cassandra/service/MigrationManager.java
+++ b/src/java/org/apache/cassandra/service/MigrationManager.java
@@ -211,10 +211,15 @@ public class MigrationManager
public static void announceNewKeyspace(KSMetaData ksm) throws ConfigurationException
{
- announceNewKeyspace(ksm, FBUtilities.timestampMicros());
+ announceNewKeyspace(ksm, false);
}
- public static void announceNewKeyspace(KSMetaData ksm, long timestamp) throws ConfigurationException
+ public static void announceNewKeyspace(KSMetaData ksm, boolean announceLocally) throws ConfigurationException
+ {
+ announceNewKeyspace(ksm, FBUtilities.timestampMicros(), announceLocally);
+ }
+
+ public static void announceNewKeyspace(KSMetaData ksm, long timestamp, boolean announceLocally) throws ConfigurationException
{
ksm.validate();
@@ -222,11 +227,16 @@ public class MigrationManager
throw new AlreadyExistsException(ksm.name);
logger.info(String.format("Create new Keyspace: %s", ksm));
- announce(ksm.toSchema(timestamp));
+ announce(ksm.toSchema(timestamp), announceLocally);
}
public static void announceNewColumnFamily(CFMetaData cfm) throws ConfigurationException
{
+ announceNewColumnFamily(cfm, false);
+ }
+
+ public static void announceNewColumnFamily(CFMetaData cfm, boolean announceLocally) throws ConfigurationException
+ {
cfm.validate();
KSMetaData ksm = Schema.instance.getKSMetaData(cfm.ksName);
@@ -236,16 +246,26 @@ public class MigrationManager
throw new AlreadyExistsException(cfm.ksName, cfm.cfName);
logger.info(String.format("Create new ColumnFamily: %s", cfm));
- announce(addSerializedKeyspace(cfm.toSchema(FBUtilities.timestampMicros()), cfm.ksName));
+ announce(addSerializedKeyspace(cfm.toSchema(FBUtilities.timestampMicros()), cfm.ksName), announceLocally);
}
public static void announceNewType(UserType newType)
{
- announce(UTMetaData.toSchema(newType, FBUtilities.timestampMicros()));
+ announceNewType(newType, false);
+ }
+
+ public static void announceNewType(UserType newType, boolean announceLocally)
+ {
+ announce(UTMetaData.toSchema(newType, FBUtilities.timestampMicros()), announceLocally);
}
public static void announceKeyspaceUpdate(KSMetaData ksm) throws ConfigurationException
{
+ announceKeyspaceUpdate(ksm, false);
+ }
+
+ public static void announceKeyspaceUpdate(KSMetaData ksm, boolean announceLocally) throws ConfigurationException
+ {
ksm.validate();
KSMetaData oldKsm = Schema.instance.getKSMetaData(ksm.name);
@@ -253,11 +273,16 @@ public class MigrationManager
throw new ConfigurationException(String.format("Cannot update non existing keyspace '%s'.", ksm.name));
logger.info(String.format("Update Keyspace '%s' From %s To %s", ksm.name, oldKsm, ksm));
- announce(oldKsm.toSchemaUpdate(ksm, FBUtilities.timestampMicros()));
+ announce(oldKsm.toSchemaUpdate(ksm, FBUtilities.timestampMicros()), announceLocally);
}
public static void announceColumnFamilyUpdate(CFMetaData cfm, boolean fromThrift) throws ConfigurationException
{
+ announceColumnFamilyUpdate(cfm, fromThrift, false);
+ }
+
+ public static void announceColumnFamilyUpdate(CFMetaData cfm, boolean fromThrift, boolean announceLocally) throws ConfigurationException
+ {
cfm.validate();
CFMetaData oldCfm = Schema.instance.getCFMetaData(cfm.ksName, cfm.cfName);
@@ -267,32 +292,47 @@ public class MigrationManager
oldCfm.validateCompatility(cfm);
logger.info(String.format("Update ColumnFamily '%s/%s' From %s To %s", cfm.ksName, cfm.cfName, oldCfm, cfm));
- announce(addSerializedKeyspace(oldCfm.toSchemaUpdate(cfm, FBUtilities.timestampMicros(), fromThrift), cfm.ksName));
+ announce(addSerializedKeyspace(oldCfm.toSchemaUpdate(cfm, FBUtilities.timestampMicros(), fromThrift), cfm.ksName), announceLocally);
}
public static void announceTypeUpdate(UserType updatedType)
{
- announceNewType(updatedType);
+ announceTypeUpdate(updatedType, false);
+ }
+
+ public static void announceTypeUpdate(UserType updatedType, boolean announceLocally)
+ {
+ announceNewType(updatedType, announceLocally);
}
public static void announceKeyspaceDrop(String ksName) throws ConfigurationException
{
+ announceKeyspaceDrop(ksName, false);
+ }
+
+ public static void announceKeyspaceDrop(String ksName, boolean announceLocally) throws ConfigurationException
+ {
KSMetaData oldKsm = Schema.instance.getKSMetaData(ksName);
if (oldKsm == null)
throw new ConfigurationException(String.format("Cannot drop non existing keyspace '%s'.", ksName));
logger.info(String.format("Drop Keyspace '%s'", oldKsm.name));
- announce(oldKsm.dropFromSchema(FBUtilities.timestampMicros()));
+ announce(oldKsm.dropFromSchema(FBUtilities.timestampMicros()), announceLocally);
}
public static void announceColumnFamilyDrop(String ksName, String cfName) throws ConfigurationException
{
+ announceColumnFamilyDrop(ksName, cfName, false);
+ }
+
+ public static void announceColumnFamilyDrop(String ksName, String cfName, boolean announceLocally) throws ConfigurationException
+ {
CFMetaData oldCfm = Schema.instance.getCFMetaData(ksName, cfName);
if (oldCfm == null)
throw new ConfigurationException(String.format("Cannot drop non existing column family '%s' in keyspace '%s'.", cfName, ksName));
logger.info(String.format("Drop ColumnFamily '%s/%s'", oldCfm.ksName, oldCfm.cfName));
- announce(addSerializedKeyspace(oldCfm.dropFromSchema(FBUtilities.timestampMicros()), ksName));
+ announce(addSerializedKeyspace(oldCfm.dropFromSchema(FBUtilities.timestampMicros()), ksName), announceLocally);
}
// Include the serialized keyspace for when a target node missed the CREATE KEYSPACE migration (see #5631).
@@ -304,16 +344,35 @@ public class MigrationManager
public static void announceTypeDrop(UserType droppedType)
{
- announce(UTMetaData.dropFromSchema(droppedType, FBUtilities.timestampMicros()));
+ announceTypeDrop(droppedType, false);
+ }
+
+ public static void announceTypeDrop(UserType droppedType, boolean announceLocally)
+ {
+ announce(UTMetaData.dropFromSchema(droppedType, FBUtilities.timestampMicros()), announceLocally);
}
/**
* actively announce a new version to active hosts via rpc
* @param schema The schema mutation to be applied
*/
- private static void announce(Mutation schema)
+ private static void announce(Mutation schema, boolean announceLocally)
{
- FBUtilities.waitOnFuture(announce(Collections.singletonList(schema)));
+ if (announceLocally)
+ {
+ try
+ {
+ DefsTables.mergeSchemaInternal(Collections.singletonList(schema), false);
+ }
+ catch (ConfigurationException | IOException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+ else
+ {
+ FBUtilities.waitOnFuture(announce(Collections.singletonList(schema)));
+ }
}
private static void pushSchemaMutation(InetAddress endpoint, Collection<Mutation> schema)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/src/java/org/apache/cassandra/service/StorageService.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java
index a363279..90f43ac 100644
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@ -890,7 +890,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
if (Schema.instance.getKSMetaData(Tracing.TRACE_KS) == null)
{
KSMetaData tracingKeyspace = KSMetaData.traceKeyspace();
- MigrationManager.announceNewKeyspace(tracingKeyspace, 0);
+ MigrationManager.announceNewKeyspace(tracingKeyspace, 0, false);
}
if (!isSurveyMode)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/test/unit/org/apache/cassandra/SchemaLoader.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/SchemaLoader.java b/test/unit/org/apache/cassandra/SchemaLoader.java
index 7f637a4..a9d69fd 100644
--- a/test/unit/org/apache/cassandra/SchemaLoader.java
+++ b/test/unit/org/apache/cassandra/SchemaLoader.java
@@ -54,6 +54,10 @@ public class SchemaLoader
{
prepareServer();
+ // Migrations aren't happy if gossiper is not started. Even if we don't use migrations though,
+ // some tests now expect us to start gossip for them.
+ startGossiper();
+
// if you're messing with low-level sstable stuff, it can be useful to inject the schema directly
// Schema.instance.load(schemaDefinition());
for (KSMetaData ksm : schemaDefinition())
@@ -76,9 +80,6 @@ public class SchemaLoader
});
Keyspace.setInitialized();
- // Migrations aren't happy if gossiper is not started. Even if we don't use migrations though,
- // some tests now expect us to start gossip for them.
- startGossiper();
}
public static void startGossiper()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78e91c4c/test/unit/org/apache/cassandra/cql3/CQLTester.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/cql3/CQLTester.java b/test/unit/org/apache/cassandra/cql3/CQLTester.java
index 06119ed..b8912e7 100644
--- a/test/unit/org/apache/cassandra/cql3/CQLTester.java
+++ b/test/unit/org/apache/cassandra/cql3/CQLTester.java
@@ -17,12 +17,16 @@
*/
package org.apache.cassandra.cql3;
+import java.io.File;
import java.nio.ByteBuffer;
import java.util.*;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import com.google.common.base.Objects;
import org.junit.AfterClass;
+import org.junit.After;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.slf4j.Logger;
@@ -30,8 +34,11 @@ import org.slf4j.LoggerFactory;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.marshal.*;
import org.apache.cassandra.exceptions.*;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.ByteBufferUtil;
/**
@@ -42,26 +49,79 @@ public abstract class CQLTester
protected static final Logger logger = LoggerFactory.getLogger(CQLTester.class);
private static final String KEYSPACE = "cql_test_keyspace";
-
private static final boolean USE_PREPARED_VALUES = Boolean.valueOf(System.getProperty("cassandra.test.use_prepared", "true"));
-
private static final AtomicInteger seqNumber = new AtomicInteger();
+ static
+ {
+ // Once per-JVM is enough
+ SchemaLoader.prepareServer();
+ }
+
private String currentTable;
@BeforeClass
public static void setUpClass() throws Throwable
{
- // This start gossiper for the sake of schema migrations. We might be able to get rid of that with some work.
- SchemaLoader.prepareServer();
-
schemaChange(String.format("CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}", KEYSPACE));
}
@AfterClass
public static void tearDownClass()
{
- SchemaLoader.stopGossiper();
+ }
+
+ @After
+ public void afterTest() throws Throwable
+ {
+ if (currentTable == null)
+ return;
+
+ final String toDrop = currentTable;
+ currentTable = null;
+
+ // We want to clean up after the test, but dropping a table is rather long so just do that asynchronously
+ StorageService.tasks.execute(new Runnable()
+ {
+ public void run()
+ {
+ try
+ {
+ schemaChange(String.format("DROP TABLE %s.%s", KEYSPACE, toDrop));
+
+ // Dropping doesn't delete the sstables. It's not a huge deal but it's cleaner to cleanup after us
+ // Thas said, we shouldn't delete blindly before the SSTableDeletingTask for the table we drop
+ // have run or they will be unhappy. Since those taks are scheduled on StorageService.tasks and that's
+ // mono-threaded, just push a task on the queue to find when it's empty. No perfect but good enough.
+
+ final CountDownLatch latch = new CountDownLatch(1);
+ StorageService.tasks.execute(new Runnable()
+ {
+ public void run()
+ {
+ latch.countDown();
+ }
+ });
+ latch.await(2, TimeUnit.SECONDS);
+
+ removeAllSSTables(KEYSPACE, toDrop);
+ }
+ catch (Exception e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+ });
+ }
+
+ private static void removeAllSSTables(String ks, String table)
+ {
+ // clean up data directory which are stored as data directory/keyspace/data files
+ for (File d : Directories.getKSChildDirectories(ks))
+ {
+ if (d.exists() && d.getName().contains(table))
+ FileUtils.deleteRecursive(d);
+ }
}
protected void createTable(String query)
@@ -77,7 +137,7 @@ public abstract class CQLTester
try
{
// executeOnceInternal don't work for schema changes
- QueryProcessor.process(query, ConsistencyLevel.ONE);
+ QueryProcessor.executeOnceInternal(query);
}
catch (Exception e)
{
[2/2] git commit: Merge branch 'cassandra-2.1' into trunk
Posted by sl...@apache.org.
Merge branch 'cassandra-2.1' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/99ba549d
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/99ba549d
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/99ba549d
Branch: refs/heads/trunk
Commit: 99ba549d0dc378295292c870ed1b68cb757bafc9
Parents: 2306fb8 78e91c4
Author: Sylvain Lebresne <sy...@datastax.com>
Authored: Thu Jun 5 08:40:27 2014 +0200
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Thu Jun 5 08:40:27 2014 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
build.xml | 9 ++-
src/java/org/apache/cassandra/auth/Auth.java | 2 +-
.../cql3/statements/AlterKeyspaceStatement.java | 4 +-
.../cql3/statements/AlterTableStatement.java | 4 +-
.../cql3/statements/AlterTypeStatement.java | 8 +-
.../cql3/statements/CreateIndexStatement.java | 4 +-
.../statements/CreateKeyspaceStatement.java | 4 +-
.../cql3/statements/CreateTableStatement.java | 4 +-
.../cql3/statements/CreateTriggerStatement.java | 4 +-
.../cql3/statements/CreateTypeStatement.java | 4 +-
.../cql3/statements/DropIndexStatement.java | 4 +-
.../cql3/statements/DropKeyspaceStatement.java | 4 +-
.../cql3/statements/DropTableStatement.java | 4 +-
.../cql3/statements/DropTriggerStatement.java | 4 +-
.../cql3/statements/DropTypeStatement.java | 4 +-
.../statements/SchemaAlteringStatement.java | 16 +++-
.../org/apache/cassandra/db/DefsTables.java | 10 ++-
src/java/org/apache/cassandra/db/Memtable.java | 7 +-
.../cassandra/service/MigrationManager.java | 85 +++++++++++++++++---
.../cassandra/service/StorageService.java | 2 +-
.../unit/org/apache/cassandra/SchemaLoader.java | 7 +-
.../org/apache/cassandra/cql3/CQLTester.java | 74 +++++++++++++++--
23 files changed, 202 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/99ba549d/CHANGES.txt
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/99ba549d/build.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/99ba549d/src/java/org/apache/cassandra/service/StorageService.java
----------------------------------------------------------------------