You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by sl...@apache.org on 2015/11/06 14:52:48 UTC
[1/3] cassandra git commit: Always update system keyspace to the most
up-to-date version
Repository: cassandra
Updated Branches:
refs/heads/trunk 5c97de91b -> a177502b8
Always update system keyspace to the most up-to-date version
patch by slebresne; reviewed by carlyeks for CASSANDRA-10652
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/6367987f
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/6367987f
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/6367987f
Branch: refs/heads/trunk
Commit: 6367987f49a25512dbf715df9b6d564d53f27235
Parents: 801c50e
Author: Sylvain Lebresne <sy...@datastax.com>
Authored: Thu Nov 5 15:30:54 2015 +0100
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Fri Nov 6 14:42:31 2015 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/service/MigrationManager.java | 22 +++++++-
.../cassandra/service/StorageService.java | 56 ++++++++++++--------
3 files changed, 55 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6367987f/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 538d5e4..7e969a5 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.2.4
+ * Use most up-to-date version of schema for system tables (CASSANDRA-10652)
* Deprecate memory_allocator in cassandra.yaml (CASSANDRA-10581,10628)
* Expose phi values from failure detector via JMX and tweak debug
and trace logging (CASSANDRA-9526)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6367987f/src/java/org/apache/cassandra/service/MigrationManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/MigrationManager.java b/src/java/org/apache/cassandra/service/MigrationManager.java
index 9087672..ec7448a 100644
--- a/src/java/org/apache/cassandra/service/MigrationManager.java
+++ b/src/java/org/apache/cassandra/service/MigrationManager.java
@@ -277,12 +277,32 @@ public class MigrationManager
public static void announceNewColumnFamily(CFMetaData cfm, boolean announceLocally) throws ConfigurationException
{
+ announceNewColumnFamily(cfm, announceLocally, true);
+ }
+
+ /**
+ * Announces the table even if the definition is already know locally.
+ * This should generally be avoided but is used internally when we want to force the most up to date version of
+ * a system table schema (Note that we don't know if the schema we force _is_ the most recent version or not, we
+ * just rely on idempotency to basically ignore that announce if it's not. That's why we can't use announceUpdateColumnFamily,
+ * it would for instance delete new columns if this is not called with the most up-to-date version)
+ *
+ * Note that this is only safe for system tables where we know the cfId is fixed and will be the same whatever version
+ * of the definition is used.
+ */
+ public static void forceAnnounceNewColumnFamily(CFMetaData cfm) throws ConfigurationException
+ {
+ announceNewColumnFamily(cfm, false, false);
+ }
+
+ private static void announceNewColumnFamily(CFMetaData cfm, boolean announceLocally, boolean throwOnDuplicate) throws ConfigurationException
+ {
cfm.validate();
KSMetaData ksm = Schema.instance.getKSMetaData(cfm.ksName);
if (ksm == null)
throw new ConfigurationException(String.format("Cannot add table '%s' to non existing keyspace '%s'.", cfm.cfName, cfm.ksName));
- else if (ksm.cfMetaData().containsKey(cfm.cfName))
+ else if (throwOnDuplicate && ksm.cfMetaData().containsKey(cfm.cfName))
throw new AlreadyExistsException(cfm.ksName, cfm.cfName);
logger.info(String.format("Create new table: %s", cfm));
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6367987f/src/java/org/apache/cassandra/service/StorageService.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java
index 77483f2..ad209fc 100644
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@ -950,11 +950,8 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
}
// if we don't have system_traces keyspace at this point, then create it manually
- if (Schema.instance.getKSMetaData(TraceKeyspace.NAME) == null)
- maybeAddKeyspace(TraceKeyspace.definition());
-
- if (Schema.instance.getKSMetaData(SystemDistributedKeyspace.NAME) == null)
- MigrationManager.announceNewKeyspace(SystemDistributedKeyspace.definition(), 0, false);
+ maybeAddOrUpdateKeyspace(TraceKeyspace.definition());
+ maybeAddOrUpdateKeyspace(SystemDistributedKeyspace.definition());
if (!isSurveyMode)
{
@@ -1020,24 +1017,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
private void doAuthSetup()
{
- try
- {
- // if we don't have system_auth keyspace at this point, then create it
- if (Schema.instance.getKSMetaData(AuthKeyspace.NAME) == null)
- maybeAddKeyspace(AuthKeyspace.definition());
- }
- catch (Exception e)
- {
- throw new AssertionError(e); // shouldn't ever happen.
- }
-
- // create any necessary tables as we may be upgrading in which case
- // the ks exists with the only the legacy tables defined.
- // Also, the addKeyspace above can be racy if multiple nodes are started
- // concurrently - see CASSANDRA-9201
- for (Map.Entry<String, CFMetaData> table : AuthKeyspace.definition().cfMetaData().entrySet())
- if (Schema.instance.getCFMetaData(AuthKeyspace.NAME, table.getKey()) == null)
- maybeAddTable(table.getValue());
+ maybeAddOrUpdateKeyspace(AuthKeyspace.definition());
DatabaseDescriptor.getRoleManager().setup();
DatabaseDescriptor.getAuthenticator().setup();
@@ -1069,6 +1049,36 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
}
}
+ /**
+ * Ensure the schema of a pseudo-system keyspace (a distributed system keyspace: traces, auth and the so-called distributedKeyspace),
+ * is up to date with what we expected (creating it if it doesn't exist and updating tables that may have been upgraded).
+ */
+ private void maybeAddOrUpdateKeyspace(KSMetaData expected)
+ {
+ // Note that want to deal with the keyspace and its table a bit differently: for the keyspace definition
+ // itself, we want to create it if it doesn't exist yet, but if it does exist, we don't want to modify it,
+ // because user can modify the definition to change the replication factor (#6016) and we don't want to
+ // override it. For the tables however, we have to deal with the fact that new version can add new columns
+ // (#8162 being an example), so even if the table definition exists, we still need to force the "current"
+ // version of the schema, the one the node will be expecting.
+
+ KSMetaData defined = Schema.instance.getKSMetaData(expected.name);
+ if (defined == null)
+ {
+ // The keyspace doesn't exist, create it
+ maybeAddKeyspace(expected);
+ return;
+ }
+
+ // While the keyspace exists, it might miss table or have outdated one
+ for (CFMetaData expectedTable : expected.cfMetaData().values())
+ {
+ CFMetaData definedTable = defined.cfMetaData().get(expectedTable.cfName);
+ if (definedTable == null || !definedTable.equals(expectedTable))
+ MigrationManager.forceAnnounceNewColumnFamily(expectedTable);
+ }
+ }
+
public boolean isJoined()
{
return joined && !isSurveyMode;
[3/3] cassandra git commit: Merge branch 'cassandra-3.0' into trunk
Posted by sl...@apache.org.
Merge branch 'cassandra-3.0' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/a177502b
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/a177502b
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/a177502b
Branch: refs/heads/trunk
Commit: a177502b8f9e5ebbf19b8bbfcbe12f4497f0e9bb
Parents: 5c97de9 57d56bd
Author: Sylvain Lebresne <sy...@datastax.com>
Authored: Fri Nov 6 14:52:40 2015 +0100
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Fri Nov 6 14:52:40 2015 +0100
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../cassandra/service/MigrationManager.java | 22 +++++++-
.../cassandra/service/StorageService.java | 56 ++++++++++++--------
3 files changed, 56 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/a177502b/CHANGES.txt
----------------------------------------------------------------------
[2/3] cassandra git commit: Merge branch 'cassandra-2.2' into
cassandra-3.0
Posted by sl...@apache.org.
Merge branch 'cassandra-2.2' into cassandra-3.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/57d56bd1
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/57d56bd1
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/57d56bd1
Branch: refs/heads/trunk
Commit: 57d56bd1c2ce28722667221b553a8f505d9eee77
Parents: 962aa2b 6367987
Author: Sylvain Lebresne <sy...@datastax.com>
Authored: Fri Nov 6 14:52:23 2015 +0100
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Fri Nov 6 14:52:23 2015 +0100
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../cassandra/service/MigrationManager.java | 22 +++++++-
.../cassandra/service/StorageService.java | 56 ++++++++++++--------
3 files changed, 56 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/57d56bd1/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 6d26afa,7e969a5..a72cab2
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,37 -1,11 +1,39 @@@
-2.2.4
+3.0
+ * Fix distinct queries in mixed version cluster (CASSANDRA-10573)
+ * Skip sstable on clustering in names query (CASSANDRA-10571)
+ * Remove value skipping as it breaks read-repair (CASSANDRA-10655)
+ * Fix bootstrapping with MVs (CASSANDRA-10621)
+ * Make sure EACH_QUORUM reads are using NTS (CASSANDRA-10584)
+ * Fix MV replica filtering for non-NetworkTopologyStrategy (CASSANDRA-10634)
+ * (Hadoop) fix CIF describeSplits() not handling 0 size estimates (CASSANDRA-10600)
+ * Fix reading of legacy sstables (CASSANDRA-10590)
+ * Use CQL type names in schema metadata tables (CASSANDRA-10365)
+ * Guard batchlog replay against integer division by zero (CASSANDRA-9223)
+ * Fix bug when adding a column to thrift with the same name than a primary key (CASSANDRA-10608)
+ * Add client address argument to IAuthenticator::newSaslNegotiator (CASSANDRA-8068)
+ * Fix implementation of LegacyLayout.LegacyBoundComparator (CASSANDRA-10602)
+ * Don't use 'names query' read path for counters (CASSANDRA-10572)
+ * Fix backward compatibility for counters (CASSANDRA-10470)
+ * Remove memory_allocator paramter from cassandra.yaml (CASSANDRA-10581,10628)
+ * Execute the metadata reload task of all registered indexes on CFS::reload (CASSANDRA-10604)
+ * Fix thrift cas operations with defined columns (CASSANDRA-10576)
+ * Fix PartitionUpdate.operationCount()for updates with static column operations (CASSANDRA-10606)
+ * Fix thrift get() queries with defined columns (CASSANDRA-10586)
+ * Fix marking of indexes as built and removed (CASSANDRA-10601)
+ * Skip initialization of non-registered 2i instances, remove Index::getIndexName (CASSANDRA-10595)
+ * Fix batches on multiple tables (CASSANDRA-10554)
+ * Ensure compaction options are validated when updating KeyspaceMetadata (CASSANDRA-10569)
+ * Flatten Iterator Transformation Hierarchy (CASSANDRA-9975)
+ * Remove token generator (CASSANDRA-5261)
+ * RolesCache should not be created for any authenticator that does not requireAuthentication (CASSANDRA-10562)
+ * Fix LogTransaction checking only a single directory for files (CASSANDRA-10421)
+ * Fix handling of range tombstones when reading old format sstables (CASSANDRA-10360)
+ * Aggregate with Initial Condition fails with C* 3.0 (CASSANDRA-10367)
+Merged from 2.2:
+ * Use most up-to-date version of schema for system tables (CASSANDRA-10652)
+ * Deprecate memory_allocator in cassandra.yaml (CASSANDRA-10581,10628)
* Expose phi values from failure detector via JMX and tweak debug
and trace logging (CASSANDRA-9526)
- * Fix RangeNamesQueryPager (CASSANDRA-10509)
- * Deprecate Pig support (CASSANDRA-10542)
- * Reduce contention getting instances of CompositeType (CASSANDRA-10433)
Merged from 2.1:
* Do not run SizeEstimatesRecorder if a node is not a member of the ring (CASSANDRA-9912)
* Improve handling of dead nodes in gossip (CASSANDRA-10298)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/57d56bd1/src/java/org/apache/cassandra/service/MigrationManager.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/service/MigrationManager.java
index 6a21f91,ec7448a..b7f9bf3
--- a/src/java/org/apache/cassandra/service/MigrationManager.java
+++ b/src/java/org/apache/cassandra/service/MigrationManager.java
@@@ -290,13 -277,32 +290,33 @@@ public class MigrationManage
public static void announceNewColumnFamily(CFMetaData cfm, boolean announceLocally) throws ConfigurationException
{
+ announceNewColumnFamily(cfm, announceLocally, true);
+ }
+
+ /**
+ * Announces the table even if the definition is already know locally.
+ * This should generally be avoided but is used internally when we want to force the most up to date version of
+ * a system table schema (Note that we don't know if the schema we force _is_ the most recent version or not, we
+ * just rely on idempotency to basically ignore that announce if it's not. That's why we can't use announceUpdateColumnFamily,
+ * it would for instance delete new columns if this is not called with the most up-to-date version)
+ *
+ * Note that this is only safe for system tables where we know the cfId is fixed and will be the same whatever version
+ * of the definition is used.
+ */
+ public static void forceAnnounceNewColumnFamily(CFMetaData cfm) throws ConfigurationException
+ {
+ announceNewColumnFamily(cfm, false, false);
+ }
+
+ private static void announceNewColumnFamily(CFMetaData cfm, boolean announceLocally, boolean throwOnDuplicate) throws ConfigurationException
+ {
cfm.validate();
- KSMetaData ksm = Schema.instance.getKSMetaData(cfm.ksName);
+ KeyspaceMetadata ksm = Schema.instance.getKSMetaData(cfm.ksName);
if (ksm == null)
throw new ConfigurationException(String.format("Cannot add table '%s' to non existing keyspace '%s'.", cfm.cfName, cfm.ksName));
- else if (throwOnDuplicate && ksm.cfMetaData().containsKey(cfm.cfName))
+ // If we have a table or a view which has the same name, we can't add a new one
- else if (ksm.getTableOrViewNullable(cfm.cfName) != null)
++ else if (throwOnDuplicate && ksm.getTableOrViewNullable(cfm.cfName) != null)
throw new AlreadyExistsException(cfm.ksName, cfm.cfName);
logger.info(String.format("Create new table: %s", cfm));
http://git-wip-us.apache.org/repos/asf/cassandra/blob/57d56bd1/src/java/org/apache/cassandra/service/StorageService.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/service/StorageService.java
index be1e7f5,ad209fc..34df507
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@@ -966,11 -950,8 +966,8 @@@ public class StorageService extends Not
}
// if we don't have system_traces keyspace at this point, then create it manually
- if (Schema.instance.getKSMetaData(TraceKeyspace.NAME) == null)
- maybeAddKeyspace(TraceKeyspace.metadata());
-
- if (Schema.instance.getKSMetaData(SystemDistributedKeyspace.NAME) == null)
- MigrationManager.announceNewKeyspace(SystemDistributedKeyspace.metadata(), 0, false);
- maybeAddOrUpdateKeyspace(TraceKeyspace.definition());
- maybeAddOrUpdateKeyspace(SystemDistributedKeyspace.definition());
++ maybeAddOrUpdateKeyspace(TraceKeyspace.metadata());
++ maybeAddOrUpdateKeyspace(SystemDistributedKeyspace.metadata());
if (!isSurveyMode)
{
@@@ -1036,24 -1017,7 +1033,7 @@@
private void doAuthSetup()
{
- try
- {
- // if we don't have system_auth keyspace at this point, then create it
- if (Schema.instance.getKSMetaData(AuthKeyspace.NAME) == null)
- maybeAddKeyspace(AuthKeyspace.metadata());
- }
- catch (Exception e)
- {
- throw new AssertionError(e); // shouldn't ever happen.
- }
-
- // create any necessary tables as we may be upgrading in which case
- // the ks exists with the only the legacy tables defined.
- // Also, the addKeyspace above can be racy if multiple nodes are started
- // concurrently - see CASSANDRA-9201
- for (CFMetaData table : AuthKeyspace.metadata().tables)
- if (Schema.instance.getCF(table.cfId) == null)
- maybeAddTable(table);
- maybeAddOrUpdateKeyspace(AuthKeyspace.definition());
++ maybeAddOrUpdateKeyspace(AuthKeyspace.metadata());
DatabaseDescriptor.getRoleManager().setup();
DatabaseDescriptor.getAuthenticator().setup();
@@@ -1085,6 -1049,36 +1065,36 @@@
}
}
+ /**
+ * Ensure the schema of a pseudo-system keyspace (a distributed system keyspace: traces, auth and the so-called distributedKeyspace),
+ * is up to date with what we expected (creating it if it doesn't exist and updating tables that may have been upgraded).
+ */
- private void maybeAddOrUpdateKeyspace(KSMetaData expected)
++ private void maybeAddOrUpdateKeyspace(KeyspaceMetadata expected)
+ {
+ // Note that want to deal with the keyspace and its table a bit differently: for the keyspace definition
+ // itself, we want to create it if it doesn't exist yet, but if it does exist, we don't want to modify it,
+ // because user can modify the definition to change the replication factor (#6016) and we don't want to
+ // override it. For the tables however, we have to deal with the fact that new version can add new columns
+ // (#8162 being an example), so even if the table definition exists, we still need to force the "current"
+ // version of the schema, the one the node will be expecting.
+
- KSMetaData defined = Schema.instance.getKSMetaData(expected.name);
++ KeyspaceMetadata defined = Schema.instance.getKSMetaData(expected.name);
+ if (defined == null)
+ {
+ // The keyspace doesn't exist, create it
+ maybeAddKeyspace(expected);
+ return;
+ }
+
+ // While the keyspace exists, it might miss table or have outdated one
- for (CFMetaData expectedTable : expected.cfMetaData().values())
++ for (CFMetaData expectedTable : expected.tables)
+ {
- CFMetaData definedTable = defined.cfMetaData().get(expectedTable.cfName);
++ CFMetaData definedTable = defined.tables.get(expectedTable.cfName).orElse(null);
+ if (definedTable == null || !definedTable.equals(expectedTable))
+ MigrationManager.forceAnnounceNewColumnFamily(expectedTable);
+ }
+ }
+
public boolean isJoined()
{
return joined && !isSurveyMode;