You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by sa...@apache.org on 2015/12/03 13:01:24 UTC

[01/10] cassandra git commit: Always verify expected tables in pseudo-system keyspaces at startup

Repository: cassandra
Updated Branches:
  refs/heads/cassandra-2.2 e491f05b7 -> 15f03ab44
  refs/heads/cassandra-3.0 d7068fd72 -> 9784be5e0
  refs/heads/cassandra-3.1 6405f6c9c -> ebb1f8d16
  refs/heads/trunk 47e133c90 -> 0cb117cd3


Always verify expected tables in pseudo-system keyspaces at startup

Patch by Sam Tunnicliffe; reviewed by Sylvain Lebresne for
CASSANDRA-10761


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/15f03ab4
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/15f03ab4
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/15f03ab4

Branch: refs/heads/cassandra-2.2
Commit: 15f03ab446854cf4d2999c3785d145c89bc3a3e4
Parents: e491f05
Author: Sam Tunnicliffe <sa...@beobal.com>
Authored: Wed Nov 25 16:45:02 2015 +0000
Committer: Sam Tunnicliffe <sa...@beobal.com>
Committed: Thu Dec 3 11:44:25 2015 +0000

----------------------------------------------------------------------
 CHANGES.txt                                      |  4 ++++
 src/java/org/apache/cassandra/config/Schema.java |  4 +++-
 .../apache/cassandra/db/ColumnFamilyStore.java   |  4 ++--
 src/java/org/apache/cassandra/db/Keyspace.java   | 12 ++++++------
 .../apache/cassandra/service/StorageService.java | 19 +++++--------------
 5 files changed, 20 insertions(+), 23 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/15f03ab4/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 787d145..12accac 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,3 +1,7 @@
+2.2.5
+ * Verify tables in pseudo-system keyspaces at startup (CASSANDRA-10761)
+
+
 2.2.4
  * Show CQL help in cqlsh in web browser (CASSANDRA-7225)
  * Serialize on disk the proper SSTable compression ratio (CASSANDRA-10775)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/15f03ab4/src/java/org/apache/cassandra/config/Schema.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/Schema.java b/src/java/org/apache/cassandra/config/Schema.java
index 00c9358..110029e 100644
--- a/src/java/org/apache/cassandra/config/Schema.java
+++ b/src/java/org/apache/cassandra/config/Schema.java
@@ -525,8 +525,10 @@ public class Schema
         // since we're going to call initCf on the new one manually
         Keyspace.open(cfm.ksName);
 
+        // init the new CF before switching the KSM to the new one
+        // to avoid races as in CASSANDRA-10761
+        Keyspace.open(cfm.ksName).initCf(cfm, true);
         setKeyspaceDefinition(ksm);
-        Keyspace.open(ksm.name).initCf(cfm.cfId, cfm.cfName, true);
         MigrationManager.instance.notifyCreateColumnFamily(cfm);
     }
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/15f03ab4/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 8fefae2..5325c8a 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -506,9 +506,9 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
     }
 
 
-    public static ColumnFamilyStore createColumnFamilyStore(Keyspace keyspace, String columnFamily, boolean loadSSTables)
+    public static ColumnFamilyStore createColumnFamilyStore(Keyspace keyspace, CFMetaData metadata, boolean loadSSTables)
     {
-        return createColumnFamilyStore(keyspace, columnFamily, StorageService.getPartitioner(), Schema.instance.getCFMetaData(keyspace.getName(), columnFamily), loadSSTables);
+        return createColumnFamilyStore(keyspace, metadata.cfName, StorageService.getPartitioner(), metadata, loadSSTables);
     }
 
     public static synchronized ColumnFamilyStore createColumnFamilyStore(Keyspace keyspace,

http://git-wip-us.apache.org/repos/asf/cassandra/blob/15f03ab4/src/java/org/apache/cassandra/db/Keyspace.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/Keyspace.java b/src/java/org/apache/cassandra/db/Keyspace.java
index 92a0950..225369c 100644
--- a/src/java/org/apache/cassandra/db/Keyspace.java
+++ b/src/java/org/apache/cassandra/db/Keyspace.java
@@ -267,7 +267,7 @@ public class Keyspace
         for (CFMetaData cfm : new ArrayList<>(metadata.cfMetaData().values()))
         {
             logger.trace("Initializing {}.{}", getName(), cfm.cfName);
-            initCf(cfm.cfId, cfm.cfName, loadSSTables);
+            initCf(cfm, loadSSTables);
         }
     }
 
@@ -330,25 +330,25 @@ public class Keyspace
     /**
      * adds a cf to internal structures, ends up creating disk files).
      */
-    public void initCf(UUID cfId, String cfName, boolean loadSSTables)
+    public void initCf(CFMetaData metadata, boolean loadSSTables)
     {
-        ColumnFamilyStore cfs = columnFamilyStores.get(cfId);
+        ColumnFamilyStore cfs = columnFamilyStores.get(metadata.cfId);
 
         if (cfs == null)
         {
             // CFS being created for the first time, either on server startup or new CF being added.
             // We don't worry about races here; startup is safe, and adding multiple idential CFs
             // simultaneously is a "don't do that" scenario.
-            ColumnFamilyStore oldCfs = columnFamilyStores.putIfAbsent(cfId, ColumnFamilyStore.createColumnFamilyStore(this, cfName, loadSSTables));
+            ColumnFamilyStore oldCfs = columnFamilyStores.putIfAbsent(metadata.cfId, ColumnFamilyStore.createColumnFamilyStore(this, metadata, loadSSTables));
             // CFS mbean instantiation will error out before we hit this, but in case that changes...
             if (oldCfs != null)
-                throw new IllegalStateException("added multiple mappings for cf id " + cfId);
+                throw new IllegalStateException("added multiple mappings for cf id " + metadata.cfId);
         }
         else
         {
             // re-initializing an existing CF.  This will happen if you cleared the schema
             // on this node and it's getting repopulated from the rest of the cluster.
-            assert cfs.name.equals(cfName);
+            assert cfs.name.equals(metadata.cfName);
             cfs.metadata.reload();
             cfs.reload();
         }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/15f03ab4/src/java/org/apache/cassandra/service/StorageService.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java
index fec3750..68e19c5 100644
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@ -1026,18 +1026,6 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         MigrationManager.instance.register(new AuthMigrationListener());
     }
 
-    private void maybeAddTable(CFMetaData cfm)
-    {
-        try
-        {
-            MigrationManager.announceNewColumnFamily(cfm);
-        }
-        catch (AlreadyExistsException e)
-        {
-            logger.debug("Attempted to create new table {}, but it already exists", cfm.cfName);
-        }
-    }
-
     private void maybeAddKeyspace(KSMetaData ksm)
     {
         try
@@ -1064,14 +1052,17 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         // version of the schema, the one the node will be expecting.
 
         KSMetaData defined = Schema.instance.getKSMetaData(expected.name);
+        // If the keyspace doesn't exist, create it
         if (defined == null)
         {
-            // The keyspace doesn't exist, create it
             maybeAddKeyspace(expected);
-            return;
+            defined = Schema.instance.getKSMetaData(expected.name);
         }
 
         // While the keyspace exists, it might miss table or have outdated one
+        // There is also the potential for a race, as schema migrations add the bare
+        // keyspace into Schema.instance before adding its tables, so double check that
+        // all the expected tables are present
         for (CFMetaData expectedTable : expected.cfMetaData().values())
         {
             CFMetaData definedTable = defined.cfMetaData().get(expectedTable.cfName);


[10/10] cassandra git commit: Merge branch 'cassandra-3.1' into trunk

Posted by sa...@apache.org.
Merge branch 'cassandra-3.1' into trunk


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/0cb117cd
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/0cb117cd
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/0cb117cd

Branch: refs/heads/trunk
Commit: 0cb117cd303bbbe0b0c5dd3dc7a2d340ae3a5a14
Parents: 47e133c ebb1f8d
Author: Sam Tunnicliffe <sa...@beobal.com>
Authored: Thu Dec 3 11:58:11 2015 +0000
Committer: Sam Tunnicliffe <sa...@beobal.com>
Committed: Thu Dec 3 11:58:11 2015 +0000

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../org/apache/cassandra/config/Schema.java     | 41 +++++++++-----------
 .../apache/cassandra/db/ColumnFamilyStore.java  |  4 +-
 src/java/org/apache/cassandra/db/Keyspace.java  | 14 +++----
 .../cassandra/service/StorageService.java       | 19 +++------
 5 files changed, 32 insertions(+), 47 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/0cb117cd/CHANGES.txt
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/0cb117cd/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------


[07/10] cassandra git commit: Merge branch 'cassandra-2.2' into cassandra-3.0

Posted by sa...@apache.org.
Merge branch 'cassandra-2.2' into cassandra-3.0


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/9784be5e
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/9784be5e
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/9784be5e

Branch: refs/heads/trunk
Commit: 9784be5e076ce13426813493662378b52228dd57
Parents: d7068fd 15f03ab
Author: Sam Tunnicliffe <sa...@beobal.com>
Authored: Thu Dec 3 11:45:03 2015 +0000
Committer: Sam Tunnicliffe <sa...@beobal.com>
Committed: Thu Dec 3 11:45:50 2015 +0000

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../org/apache/cassandra/config/Schema.java     | 41 +++++++++-----------
 .../apache/cassandra/db/ColumnFamilyStore.java  |  4 +-
 src/java/org/apache/cassandra/db/Keyspace.java  | 14 +++----
 .../cassandra/service/StorageService.java       | 19 +++------
 5 files changed, 32 insertions(+), 47 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/9784be5e/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index a82dcfc,12accac..4dd1b97
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,22 -1,8 +1,23 @@@
 -2.2.5
 +3.0.1
 + * Fix NPE on stream read error (CASSANDRA-10771)
 + * Normalize cqlsh DESC output (CASSANDRA-10431)
 + * Rejects partition range deletions when columns are specified (CASSANDRA-10739)
 + * Fix error when saving cached key for old format sstable (CASSANDRA-10778)
 + * Invalidate prepared statements on DROP INDEX (CASSANDRA-10758)
 + * Fix SELECT statement with IN restrictions on partition key,
 +   ORDER BY and LIMIT (CASSANDRA-10729)
 + * Improve stress performance over 1k threads (CASSANDRA-7217)
 + * Wait for migration responses to complete before bootstrapping (CASSANDRA-10731)
 + * Unable to create a function with argument of type Inet (CASSANDRA-10741)
 + * Fix backward incompatibiliy in CqlInputFormat (CASSANDRA-10717)
 + * Correctly preserve deletion info on updated rows when notifying indexers
 +   of single-row deletions (CASSANDRA-10694)
 + * Notify indexers of partition delete during cleanup (CASSANDRA-10685)
 + * Keep the file open in trySkipCache (CASSANDRA-10669)
 + * Updated trigger example (CASSANDRA-10257)
 +Merged from 2.2:
+  * Verify tables in pseudo-system keyspaces at startup (CASSANDRA-10761)
 -
 -
 -2.2.4
 + * Fix IllegalArgumentException in DataOutputBuffer.reallocate for large buffers (CASSANDRA-10592)
   * Show CQL help in cqlsh in web browser (CASSANDRA-7225)
   * Serialize on disk the proper SSTable compression ratio (CASSANDRA-10775)
   * Reject index queries while the index is building (CASSANDRA-8505)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/9784be5e/src/java/org/apache/cassandra/config/Schema.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/config/Schema.java
index 569c87d,110029e..cf3f569
--- a/src/java/org/apache/cassandra/config/Schema.java
+++ b/src/java/org/apache/cassandra/config/Schema.java
@@@ -605,19 -515,20 +605,17 @@@ public class Schem
      public void addTable(CFMetaData cfm)
      {
          assert getCFMetaData(cfm.ksName, cfm.cfName) == null;
 -        KSMetaData ksm = getKSMetaData(cfm.ksName).cloneWithTableAdded(cfm);
 -
 -        logger.info("Loading {}", cfm);
  
-         update(cfm.ksName, ks ->
-         {
-             load(cfm);
- 
-             // make sure it's init-ed w/ the old definitions first,
-             // since we're going to call initCf on the new one manually
-             Keyspace.open(cfm.ksName);
- 
-             return ks.withSwapped(ks.tables.with(cfm));
-         });
- 
-         Keyspace.open(cfm.ksName).initCf(cfm.cfId, cfm.cfName, true);
++        // Make sure the keyspace is initialized and initialize the table.
++        Keyspace.open(cfm.ksName).initCf(cfm, true);
++        // Update the keyspaces map with the updated metadata
++        update(cfm.ksName, ks -> ks.withSwapped(ks.tables.with(cfm)));
++        // Update the table ID <-> table name map (cfIdMap)
+         load(cfm);
+ 
 -        // make sure it's init-ed w/ the old definitions first,
 -        // since we're going to call initCf on the new one manually
 -        Keyspace.open(cfm.ksName);
 -
+         // init the new CF before switching the KSM to the new one
+         // to avoid races as in CASSANDRA-10761
+         Keyspace.open(cfm.ksName).initCf(cfm, true);
 -        setKeyspaceDefinition(ksm);
          MigrationManager.instance.notifyCreateColumnFamily(cfm);
      }
  
@@@ -659,80 -566,37 +657,77 @@@
          CommitLog.instance.forceRecycleAllSegments(Collections.singleton(cfm.cfId));
      }
  
 -    public void addType(UserType ut)
 +    public void addView(ViewDefinition view)
      {
 -        KSMetaData ksm = getKSMetaData(ut.keyspace);
 -        assert ksm != null;
 +        assert getCFMetaData(view.ksName, view.viewName) == null;
  
-         update(view.ksName, ks ->
-         {
-             load(view);
- 
-             // make sure it's init-ed w/ the old definitions first,
-             // since we're going to call initCf on the new one manually
-             Keyspace.open(view.ksName);
 -        logger.info("Loading {}", ut);
++        Keyspace keyspace = Keyspace.open(view.ksName);
  
-             return ks.withSwapped(ks.views.with(view));
-         });
 -        ksm.userTypes.addType(ut);
++        // Make sure the keyspace is initialized and initialize the table.
++        keyspace.initCf(view.metadata, true);
++        // Update the keyspaces map with the updated metadata
++        update(view.ksName, ks -> ks.withSwapped(ks.views.with(view)));
++        // Update the table ID <-> table name map (cfIdMap)
++        load(view);
  
-         Keyspace.open(view.ksName).initCf(view.metadata.cfId, view.viewName, true);
-         Keyspace.open(view.ksName).viewManager.reload();
 -        MigrationManager.instance.notifyCreateUserType(ut);
++        keyspace.viewManager.reload();
 +        MigrationManager.instance.notifyCreateView(view);
      }
  
 -    public void updateType(UserType ut)
 +    public void updateView(ViewDefinition view)
      {
 -        KSMetaData ksm = getKSMetaData(ut.keyspace);
 -        assert ksm != null;
 +        ViewDefinition current = getKSMetaData(view.ksName).views.get(view.viewName).get();
 +        boolean changeAffectsStatements = current.metadata.apply(view.metadata);
 +
 +        Keyspace keyspace = Keyspace.open(current.ksName);
 +        keyspace.getColumnFamilyStore(current.viewName).reload();
 +        Keyspace.open(current.ksName).viewManager.update(current.viewName);
 +        MigrationManager.instance.notifyUpdateView(current, changeAffectsStatements);
 +    }
 +
 +    public void dropView(String ksName, String viewName)
 +    {
 +        KeyspaceMetadata oldKsm = getKSMetaData(ksName);
 +        assert oldKsm != null;
 +        ColumnFamilyStore cfs = Keyspace.open(ksName).getColumnFamilyStore(viewName);
 +        assert cfs != null;
 +
 +        // make sure all the indexes are dropped, or else.
 +        cfs.indexManager.markAllIndexesRemoved();
  
 -        logger.info("Updating {}", ut);
 +        // reinitialize the keyspace.
 +        ViewDefinition view = oldKsm.views.get(viewName).get();
 +        KeyspaceMetadata newKsm = oldKsm.withSwapped(oldKsm.views.without(viewName));
  
 -        ksm.userTypes.addType(ut);
 +        unload(view);
 +        setKeyspaceMetadata(newKsm);
  
 -        MigrationManager.instance.notifyUpdateUserType(ut);
 +        CompactionManager.instance.interruptCompactionFor(Collections.singleton(view.metadata), true);
 +
 +        if (DatabaseDescriptor.isAutoSnapshot())
 +            cfs.snapshot(Keyspace.getTimestampedSnapshotName(cfs.name));
 +        Keyspace.open(ksName).dropCf(view.metadata.cfId);
 +        Keyspace.open(ksName).viewManager.reload();
 +        MigrationManager.instance.notifyDropView(view);
 +
 +        CommitLog.instance.forceRecycleAllSegments(Collections.singleton(view.metadata.cfId));
      }
  
 -    public void dropType(UserType ut)
 +    public void addType(UserType ut)
      {
 -        KSMetaData ksm = getKSMetaData(ut.keyspace);
 -        assert ksm != null;
 +        update(ut.keyspace, ks -> ks.withSwapped(ks.types.with(ut)));
 +        MigrationManager.instance.notifyCreateUserType(ut);
 +    }
  
 -        ksm.userTypes.removeType(ut);
 +    public void updateType(UserType ut)
 +    {
 +        update(ut.keyspace, ks -> ks.withSwapped(ks.types.without(ut.name).with(ut)));
 +        MigrationManager.instance.notifyUpdateUserType(ut);
 +    }
  
 +    public void dropType(UserType ut)
 +    {
 +        update(ut.keyspace, ks -> ks.withSwapped(ks.types.without(ut.name)));
          MigrationManager.instance.notifyDropUserType(ut);
      }
  

http://git-wip-us.apache.org/repos/asf/cassandra/blob/9784be5e/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 38c99ea,5325c8a..f0adf39
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@@ -532,9 -506,9 +532,9 @@@ public class ColumnFamilyStore implemen
      }
  
  
-     public static ColumnFamilyStore createColumnFamilyStore(Keyspace keyspace, String columnFamily, boolean loadSSTables)
+     public static ColumnFamilyStore createColumnFamilyStore(Keyspace keyspace, CFMetaData metadata, boolean loadSSTables)
      {
-         return createColumnFamilyStore(keyspace, columnFamily, Schema.instance.getCFMetaData(keyspace.getName(), columnFamily), loadSSTables);
 -        return createColumnFamilyStore(keyspace, metadata.cfName, StorageService.getPartitioner(), metadata, loadSSTables);
++        return createColumnFamilyStore(keyspace, metadata.cfName, metadata, loadSSTables);
      }
  
      public static synchronized ColumnFamilyStore createColumnFamilyStore(Keyspace keyspace,

http://git-wip-us.apache.org/repos/asf/cassandra/blob/9784be5e/src/java/org/apache/cassandra/db/Keyspace.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/Keyspace.java
index eb28b2c,225369c..7b4f79b
--- a/src/java/org/apache/cassandra/db/Keyspace.java
+++ b/src/java/org/apache/cassandra/db/Keyspace.java
@@@ -298,16 -264,14 +298,16 @@@ public class Keyspac
          createReplicationStrategy(metadata);
  
          this.metric = new KeyspaceMetrics(this);
 -        for (CFMetaData cfm : new ArrayList<>(metadata.cfMetaData().values()))
 +        this.viewManager = new ViewManager(this);
 +        for (CFMetaData cfm : metadata.tablesAndViews())
          {
              logger.trace("Initializing {}.{}", getName(), cfm.cfName);
-             initCf(cfm.cfId, cfm.cfName, loadSSTables);
+             initCf(cfm, loadSSTables);
          }
 +        this.viewManager.reload();
      }
  
 -    private Keyspace(KSMetaData metadata)
 +    private Keyspace(KeyspaceMetadata metadata)
      {
          this.metadata = metadata;
          createReplicationStrategy(metadata);
@@@ -376,7 -348,8 +374,7 @@@
          {
              // re-initializing an existing CF.  This will happen if you cleared the schema
              // on this node and it's getting repopulated from the rest of the cluster.
-             assert cfs.name.equals(cfName);
+             assert cfs.name.equals(metadata.cfName);
 -            cfs.metadata.reload();
              cfs.reload();
          }
      }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/9784be5e/src/java/org/apache/cassandra/service/StorageService.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/service/StorageService.java
index 0b910f0,68e19c5..2332cea
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@@ -1038,19 -1026,7 +1038,7 @@@ public class StorageService extends Not
          MigrationManager.instance.register(new AuthMigrationListener());
      }
  
-     private void maybeAddTable(CFMetaData cfm)
-     {
-         try
-         {
-             MigrationManager.announceNewColumnFamily(cfm);
-         }
-         catch (AlreadyExistsException e)
-         {
-             logger.debug("Attempted to create new table {}, but it already exists", cfm.cfName);
-         }
-     }
- 
 -    private void maybeAddKeyspace(KSMetaData ksm)
 +    private void maybeAddKeyspace(KeyspaceMetadata ksm)
      {
          try
          {
@@@ -1075,18 -1051,21 +1063,21 @@@
          // (#8162 being an example), so even if the table definition exists, we still need to force the "current"
          // version of the schema, the one the node will be expecting.
  
 -        KSMetaData defined = Schema.instance.getKSMetaData(expected.name);
 +        KeyspaceMetadata defined = Schema.instance.getKSMetaData(expected.name);
+         // If the keyspace doesn't exist, create it
          if (defined == null)
          {
-             // The keyspace doesn't exist, create it
              maybeAddKeyspace(expected);
-             return;
+             defined = Schema.instance.getKSMetaData(expected.name);
          }
  
          // While the keyspace exists, it might miss table or have outdated one
+         // There is also the potential for a race, as schema migrations add the bare
+         // keyspace into Schema.instance before adding its tables, so double check that
+         // all the expected tables are present
 -        for (CFMetaData expectedTable : expected.cfMetaData().values())
 +        for (CFMetaData expectedTable : expected.tables)
          {
 -            CFMetaData definedTable = defined.cfMetaData().get(expectedTable.cfName);
 +            CFMetaData definedTable = defined.tables.get(expectedTable.cfName).orElse(null);
              if (definedTable == null || !definedTable.equals(expectedTable))
                  MigrationManager.forceAnnounceNewColumnFamily(expectedTable);
          }


[08/10] cassandra git commit: Merge branch 'cassandra-3.0' into cassandra-3.1

Posted by sa...@apache.org.
Merge branch 'cassandra-3.0' into cassandra-3.1


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/ebb1f8d1
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/ebb1f8d1
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/ebb1f8d1

Branch: refs/heads/cassandra-3.1
Commit: ebb1f8d16a6e7504298dfabca84e4de7cf2f7cca
Parents: 6405f6c 9784be5
Author: Sam Tunnicliffe <sa...@beobal.com>
Authored: Thu Dec 3 11:56:20 2015 +0000
Committer: Sam Tunnicliffe <sa...@beobal.com>
Committed: Thu Dec 3 11:56:20 2015 +0000

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../org/apache/cassandra/config/Schema.java     | 41 +++++++++-----------
 .../apache/cassandra/db/ColumnFamilyStore.java  |  4 +-
 src/java/org/apache/cassandra/db/Keyspace.java  | 14 +++----
 .../cassandra/service/StorageService.java       | 19 +++------
 5 files changed, 32 insertions(+), 47 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/ebb1f8d1/CHANGES.txt
----------------------------------------------------------------------


[04/10] cassandra git commit: Always verify expected tables in pseudo-system keyspaces at startup

Posted by sa...@apache.org.
Always verify expected tables in pseudo-system keyspaces at startup

Patch by Sam Tunnicliffe; reviewed by Sylvain Lebresne for
CASSANDRA-10761


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/15f03ab4
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/15f03ab4
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/15f03ab4

Branch: refs/heads/trunk
Commit: 15f03ab446854cf4d2999c3785d145c89bc3a3e4
Parents: e491f05
Author: Sam Tunnicliffe <sa...@beobal.com>
Authored: Wed Nov 25 16:45:02 2015 +0000
Committer: Sam Tunnicliffe <sa...@beobal.com>
Committed: Thu Dec 3 11:44:25 2015 +0000

----------------------------------------------------------------------
 CHANGES.txt                                      |  4 ++++
 src/java/org/apache/cassandra/config/Schema.java |  4 +++-
 .../apache/cassandra/db/ColumnFamilyStore.java   |  4 ++--
 src/java/org/apache/cassandra/db/Keyspace.java   | 12 ++++++------
 .../apache/cassandra/service/StorageService.java | 19 +++++--------------
 5 files changed, 20 insertions(+), 23 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/15f03ab4/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 787d145..12accac 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,3 +1,7 @@
+2.2.5
+ * Verify tables in pseudo-system keyspaces at startup (CASSANDRA-10761)
+
+
 2.2.4
  * Show CQL help in cqlsh in web browser (CASSANDRA-7225)
  * Serialize on disk the proper SSTable compression ratio (CASSANDRA-10775)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/15f03ab4/src/java/org/apache/cassandra/config/Schema.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/Schema.java b/src/java/org/apache/cassandra/config/Schema.java
index 00c9358..110029e 100644
--- a/src/java/org/apache/cassandra/config/Schema.java
+++ b/src/java/org/apache/cassandra/config/Schema.java
@@ -525,8 +525,10 @@ public class Schema
         // since we're going to call initCf on the new one manually
         Keyspace.open(cfm.ksName);
 
+        // init the new CF before switching the KSM to the new one
+        // to avoid races as in CASSANDRA-10761
+        Keyspace.open(cfm.ksName).initCf(cfm, true);
         setKeyspaceDefinition(ksm);
-        Keyspace.open(ksm.name).initCf(cfm.cfId, cfm.cfName, true);
         MigrationManager.instance.notifyCreateColumnFamily(cfm);
     }
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/15f03ab4/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 8fefae2..5325c8a 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -506,9 +506,9 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
     }
 
 
-    public static ColumnFamilyStore createColumnFamilyStore(Keyspace keyspace, String columnFamily, boolean loadSSTables)
+    public static ColumnFamilyStore createColumnFamilyStore(Keyspace keyspace, CFMetaData metadata, boolean loadSSTables)
     {
-        return createColumnFamilyStore(keyspace, columnFamily, StorageService.getPartitioner(), Schema.instance.getCFMetaData(keyspace.getName(), columnFamily), loadSSTables);
+        return createColumnFamilyStore(keyspace, metadata.cfName, StorageService.getPartitioner(), metadata, loadSSTables);
     }
 
     public static synchronized ColumnFamilyStore createColumnFamilyStore(Keyspace keyspace,

http://git-wip-us.apache.org/repos/asf/cassandra/blob/15f03ab4/src/java/org/apache/cassandra/db/Keyspace.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/Keyspace.java b/src/java/org/apache/cassandra/db/Keyspace.java
index 92a0950..225369c 100644
--- a/src/java/org/apache/cassandra/db/Keyspace.java
+++ b/src/java/org/apache/cassandra/db/Keyspace.java
@@ -267,7 +267,7 @@ public class Keyspace
         for (CFMetaData cfm : new ArrayList<>(metadata.cfMetaData().values()))
         {
             logger.trace("Initializing {}.{}", getName(), cfm.cfName);
-            initCf(cfm.cfId, cfm.cfName, loadSSTables);
+            initCf(cfm, loadSSTables);
         }
     }
 
@@ -330,25 +330,25 @@ public class Keyspace
     /**
      * adds a cf to internal structures, ends up creating disk files).
      */
-    public void initCf(UUID cfId, String cfName, boolean loadSSTables)
+    public void initCf(CFMetaData metadata, boolean loadSSTables)
     {
-        ColumnFamilyStore cfs = columnFamilyStores.get(cfId);
+        ColumnFamilyStore cfs = columnFamilyStores.get(metadata.cfId);
 
         if (cfs == null)
         {
             // CFS being created for the first time, either on server startup or new CF being added.
             // We don't worry about races here; startup is safe, and adding multiple idential CFs
             // simultaneously is a "don't do that" scenario.
-            ColumnFamilyStore oldCfs = columnFamilyStores.putIfAbsent(cfId, ColumnFamilyStore.createColumnFamilyStore(this, cfName, loadSSTables));
+            ColumnFamilyStore oldCfs = columnFamilyStores.putIfAbsent(metadata.cfId, ColumnFamilyStore.createColumnFamilyStore(this, metadata, loadSSTables));
             // CFS mbean instantiation will error out before we hit this, but in case that changes...
             if (oldCfs != null)
-                throw new IllegalStateException("added multiple mappings for cf id " + cfId);
+                throw new IllegalStateException("added multiple mappings for cf id " + metadata.cfId);
         }
         else
         {
             // re-initializing an existing CF.  This will happen if you cleared the schema
             // on this node and it's getting repopulated from the rest of the cluster.
-            assert cfs.name.equals(cfName);
+            assert cfs.name.equals(metadata.cfName);
             cfs.metadata.reload();
             cfs.reload();
         }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/15f03ab4/src/java/org/apache/cassandra/service/StorageService.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java
index fec3750..68e19c5 100644
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@ -1026,18 +1026,6 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         MigrationManager.instance.register(new AuthMigrationListener());
     }
 
-    private void maybeAddTable(CFMetaData cfm)
-    {
-        try
-        {
-            MigrationManager.announceNewColumnFamily(cfm);
-        }
-        catch (AlreadyExistsException e)
-        {
-            logger.debug("Attempted to create new table {}, but it already exists", cfm.cfName);
-        }
-    }
-
     private void maybeAddKeyspace(KSMetaData ksm)
     {
         try
@@ -1064,14 +1052,17 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         // version of the schema, the one the node will be expecting.
 
         KSMetaData defined = Schema.instance.getKSMetaData(expected.name);
+        // If the keyspace doesn't exist, create it
         if (defined == null)
         {
-            // The keyspace doesn't exist, create it
             maybeAddKeyspace(expected);
-            return;
+            defined = Schema.instance.getKSMetaData(expected.name);
         }
 
         // While the keyspace exists, it might miss table or have outdated one
+        // There is also the potential for a race, as schema migrations add the bare
+        // keyspace into Schema.instance before adding its tables, so double check that
+        // all the expected tables are present
         for (CFMetaData expectedTable : expected.cfMetaData().values())
         {
             CFMetaData definedTable = defined.cfMetaData().get(expectedTable.cfName);


[09/10] cassandra git commit: Merge branch 'cassandra-3.0' into cassandra-3.1

Posted by sa...@apache.org.
Merge branch 'cassandra-3.0' into cassandra-3.1


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/ebb1f8d1
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/ebb1f8d1
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/ebb1f8d1

Branch: refs/heads/trunk
Commit: ebb1f8d16a6e7504298dfabca84e4de7cf2f7cca
Parents: 6405f6c 9784be5
Author: Sam Tunnicliffe <sa...@beobal.com>
Authored: Thu Dec 3 11:56:20 2015 +0000
Committer: Sam Tunnicliffe <sa...@beobal.com>
Committed: Thu Dec 3 11:56:20 2015 +0000

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../org/apache/cassandra/config/Schema.java     | 41 +++++++++-----------
 .../apache/cassandra/db/ColumnFamilyStore.java  |  4 +-
 src/java/org/apache/cassandra/db/Keyspace.java  | 14 +++----
 .../cassandra/service/StorageService.java       | 19 +++------
 5 files changed, 32 insertions(+), 47 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/ebb1f8d1/CHANGES.txt
----------------------------------------------------------------------


[05/10] cassandra git commit: Merge branch 'cassandra-2.2' into cassandra-3.0

Posted by sa...@apache.org.
Merge branch 'cassandra-2.2' into cassandra-3.0


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/9784be5e
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/9784be5e
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/9784be5e

Branch: refs/heads/cassandra-3.1
Commit: 9784be5e076ce13426813493662378b52228dd57
Parents: d7068fd 15f03ab
Author: Sam Tunnicliffe <sa...@beobal.com>
Authored: Thu Dec 3 11:45:03 2015 +0000
Committer: Sam Tunnicliffe <sa...@beobal.com>
Committed: Thu Dec 3 11:45:50 2015 +0000

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../org/apache/cassandra/config/Schema.java     | 41 +++++++++-----------
 .../apache/cassandra/db/ColumnFamilyStore.java  |  4 +-
 src/java/org/apache/cassandra/db/Keyspace.java  | 14 +++----
 .../cassandra/service/StorageService.java       | 19 +++------
 5 files changed, 32 insertions(+), 47 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/9784be5e/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index a82dcfc,12accac..4dd1b97
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,22 -1,8 +1,23 @@@
 -2.2.5
 +3.0.1
 + * Fix NPE on stream read error (CASSANDRA-10771)
 + * Normalize cqlsh DESC output (CASSANDRA-10431)
 + * Rejects partition range deletions when columns are specified (CASSANDRA-10739)
 + * Fix error when saving cached key for old format sstable (CASSANDRA-10778)
 + * Invalidate prepared statements on DROP INDEX (CASSANDRA-10758)
 + * Fix SELECT statement with IN restrictions on partition key,
 +   ORDER BY and LIMIT (CASSANDRA-10729)
 + * Improve stress performance over 1k threads (CASSANDRA-7217)
 + * Wait for migration responses to complete before bootstrapping (CASSANDRA-10731)
 + * Unable to create a function with argument of type Inet (CASSANDRA-10741)
 + * Fix backward incompatibiliy in CqlInputFormat (CASSANDRA-10717)
 + * Correctly preserve deletion info on updated rows when notifying indexers
 +   of single-row deletions (CASSANDRA-10694)
 + * Notify indexers of partition delete during cleanup (CASSANDRA-10685)
 + * Keep the file open in trySkipCache (CASSANDRA-10669)
 + * Updated trigger example (CASSANDRA-10257)
 +Merged from 2.2:
+  * Verify tables in pseudo-system keyspaces at startup (CASSANDRA-10761)
 -
 -
 -2.2.4
 + * Fix IllegalArgumentException in DataOutputBuffer.reallocate for large buffers (CASSANDRA-10592)
   * Show CQL help in cqlsh in web browser (CASSANDRA-7225)
   * Serialize on disk the proper SSTable compression ratio (CASSANDRA-10775)
   * Reject index queries while the index is building (CASSANDRA-8505)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/9784be5e/src/java/org/apache/cassandra/config/Schema.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/config/Schema.java
index 569c87d,110029e..cf3f569
--- a/src/java/org/apache/cassandra/config/Schema.java
+++ b/src/java/org/apache/cassandra/config/Schema.java
@@@ -605,19 -515,20 +605,17 @@@ public class Schem
      public void addTable(CFMetaData cfm)
      {
          assert getCFMetaData(cfm.ksName, cfm.cfName) == null;
 -        KSMetaData ksm = getKSMetaData(cfm.ksName).cloneWithTableAdded(cfm);
 -
 -        logger.info("Loading {}", cfm);
  
-         update(cfm.ksName, ks ->
-         {
-             load(cfm);
- 
-             // make sure it's init-ed w/ the old definitions first,
-             // since we're going to call initCf on the new one manually
-             Keyspace.open(cfm.ksName);
- 
-             return ks.withSwapped(ks.tables.with(cfm));
-         });
- 
-         Keyspace.open(cfm.ksName).initCf(cfm.cfId, cfm.cfName, true);
++        // Make sure the keyspace is initialized and initialize the table.
++        Keyspace.open(cfm.ksName).initCf(cfm, true);
++        // Update the keyspaces map with the updated metadata
++        update(cfm.ksName, ks -> ks.withSwapped(ks.tables.with(cfm)));
++        // Update the table ID <-> table name map (cfIdMap)
+         load(cfm);
+ 
 -        // make sure it's init-ed w/ the old definitions first,
 -        // since we're going to call initCf on the new one manually
 -        Keyspace.open(cfm.ksName);
 -
+         // init the new CF before switching the KSM to the new one
+         // to avoid races as in CASSANDRA-10761
+         Keyspace.open(cfm.ksName).initCf(cfm, true);
 -        setKeyspaceDefinition(ksm);
          MigrationManager.instance.notifyCreateColumnFamily(cfm);
      }
  
@@@ -659,80 -566,37 +657,77 @@@
          CommitLog.instance.forceRecycleAllSegments(Collections.singleton(cfm.cfId));
      }
  
 -    public void addType(UserType ut)
 +    public void addView(ViewDefinition view)
      {
 -        KSMetaData ksm = getKSMetaData(ut.keyspace);
 -        assert ksm != null;
 +        assert getCFMetaData(view.ksName, view.viewName) == null;
  
-         update(view.ksName, ks ->
-         {
-             load(view);
- 
-             // make sure it's init-ed w/ the old definitions first,
-             // since we're going to call initCf on the new one manually
-             Keyspace.open(view.ksName);
 -        logger.info("Loading {}", ut);
++        Keyspace keyspace = Keyspace.open(view.ksName);
  
-             return ks.withSwapped(ks.views.with(view));
-         });
 -        ksm.userTypes.addType(ut);
++        // Make sure the keyspace is initialized and initialize the table.
++        keyspace.initCf(view.metadata, true);
++        // Update the keyspaces map with the updated metadata
++        update(view.ksName, ks -> ks.withSwapped(ks.views.with(view)));
++        // Update the table ID <-> table name map (cfIdMap)
++        load(view);
  
-         Keyspace.open(view.ksName).initCf(view.metadata.cfId, view.viewName, true);
-         Keyspace.open(view.ksName).viewManager.reload();
 -        MigrationManager.instance.notifyCreateUserType(ut);
++        keyspace.viewManager.reload();
 +        MigrationManager.instance.notifyCreateView(view);
      }
  
 -    public void updateType(UserType ut)
 +    public void updateView(ViewDefinition view)
      {
 -        KSMetaData ksm = getKSMetaData(ut.keyspace);
 -        assert ksm != null;
 +        ViewDefinition current = getKSMetaData(view.ksName).views.get(view.viewName).get();
 +        boolean changeAffectsStatements = current.metadata.apply(view.metadata);
 +
 +        Keyspace keyspace = Keyspace.open(current.ksName);
 +        keyspace.getColumnFamilyStore(current.viewName).reload();
 +        Keyspace.open(current.ksName).viewManager.update(current.viewName);
 +        MigrationManager.instance.notifyUpdateView(current, changeAffectsStatements);
 +    }
 +
 +    public void dropView(String ksName, String viewName)
 +    {
 +        KeyspaceMetadata oldKsm = getKSMetaData(ksName);
 +        assert oldKsm != null;
 +        ColumnFamilyStore cfs = Keyspace.open(ksName).getColumnFamilyStore(viewName);
 +        assert cfs != null;
 +
 +        // make sure all the indexes are dropped, or else.
 +        cfs.indexManager.markAllIndexesRemoved();
  
 -        logger.info("Updating {}", ut);
 +        // reinitialize the keyspace.
 +        ViewDefinition view = oldKsm.views.get(viewName).get();
 +        KeyspaceMetadata newKsm = oldKsm.withSwapped(oldKsm.views.without(viewName));
  
 -        ksm.userTypes.addType(ut);
 +        unload(view);
 +        setKeyspaceMetadata(newKsm);
  
 -        MigrationManager.instance.notifyUpdateUserType(ut);
 +        CompactionManager.instance.interruptCompactionFor(Collections.singleton(view.metadata), true);
 +
 +        if (DatabaseDescriptor.isAutoSnapshot())
 +            cfs.snapshot(Keyspace.getTimestampedSnapshotName(cfs.name));
 +        Keyspace.open(ksName).dropCf(view.metadata.cfId);
 +        Keyspace.open(ksName).viewManager.reload();
 +        MigrationManager.instance.notifyDropView(view);
 +
 +        CommitLog.instance.forceRecycleAllSegments(Collections.singleton(view.metadata.cfId));
      }
  
 -    public void dropType(UserType ut)
 +    public void addType(UserType ut)
      {
 -        KSMetaData ksm = getKSMetaData(ut.keyspace);
 -        assert ksm != null;
 +        update(ut.keyspace, ks -> ks.withSwapped(ks.types.with(ut)));
 +        MigrationManager.instance.notifyCreateUserType(ut);
 +    }
  
 -        ksm.userTypes.removeType(ut);
 +    public void updateType(UserType ut)
 +    {
 +        update(ut.keyspace, ks -> ks.withSwapped(ks.types.without(ut.name).with(ut)));
 +        MigrationManager.instance.notifyUpdateUserType(ut);
 +    }
  
 +    public void dropType(UserType ut)
 +    {
 +        update(ut.keyspace, ks -> ks.withSwapped(ks.types.without(ut.name)));
          MigrationManager.instance.notifyDropUserType(ut);
      }
  

http://git-wip-us.apache.org/repos/asf/cassandra/blob/9784be5e/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 38c99ea,5325c8a..f0adf39
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@@ -532,9 -506,9 +532,9 @@@ public class ColumnFamilyStore implemen
      }
  
  
-     public static ColumnFamilyStore createColumnFamilyStore(Keyspace keyspace, String columnFamily, boolean loadSSTables)
+     public static ColumnFamilyStore createColumnFamilyStore(Keyspace keyspace, CFMetaData metadata, boolean loadSSTables)
      {
-         return createColumnFamilyStore(keyspace, columnFamily, Schema.instance.getCFMetaData(keyspace.getName(), columnFamily), loadSSTables);
 -        return createColumnFamilyStore(keyspace, metadata.cfName, StorageService.getPartitioner(), metadata, loadSSTables);
++        return createColumnFamilyStore(keyspace, metadata.cfName, metadata, loadSSTables);
      }
  
      public static synchronized ColumnFamilyStore createColumnFamilyStore(Keyspace keyspace,

http://git-wip-us.apache.org/repos/asf/cassandra/blob/9784be5e/src/java/org/apache/cassandra/db/Keyspace.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/Keyspace.java
index eb28b2c,225369c..7b4f79b
--- a/src/java/org/apache/cassandra/db/Keyspace.java
+++ b/src/java/org/apache/cassandra/db/Keyspace.java
@@@ -298,16 -264,14 +298,16 @@@ public class Keyspac
          createReplicationStrategy(metadata);
  
          this.metric = new KeyspaceMetrics(this);
 -        for (CFMetaData cfm : new ArrayList<>(metadata.cfMetaData().values()))
 +        this.viewManager = new ViewManager(this);
 +        for (CFMetaData cfm : metadata.tablesAndViews())
          {
              logger.trace("Initializing {}.{}", getName(), cfm.cfName);
-             initCf(cfm.cfId, cfm.cfName, loadSSTables);
+             initCf(cfm, loadSSTables);
          }
 +        this.viewManager.reload();
      }
  
 -    private Keyspace(KSMetaData metadata)
 +    private Keyspace(KeyspaceMetadata metadata)
      {
          this.metadata = metadata;
          createReplicationStrategy(metadata);
@@@ -376,7 -348,8 +374,7 @@@
          {
              // re-initializing an existing CF.  This will happen if you cleared the schema
              // on this node and it's getting repopulated from the rest of the cluster.
-             assert cfs.name.equals(cfName);
+             assert cfs.name.equals(metadata.cfName);
 -            cfs.metadata.reload();
              cfs.reload();
          }
      }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/9784be5e/src/java/org/apache/cassandra/service/StorageService.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/service/StorageService.java
index 0b910f0,68e19c5..2332cea
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@@ -1038,19 -1026,7 +1038,7 @@@ public class StorageService extends Not
          MigrationManager.instance.register(new AuthMigrationListener());
      }
  
-     private void maybeAddTable(CFMetaData cfm)
-     {
-         try
-         {
-             MigrationManager.announceNewColumnFamily(cfm);
-         }
-         catch (AlreadyExistsException e)
-         {
-             logger.debug("Attempted to create new table {}, but it already exists", cfm.cfName);
-         }
-     }
- 
 -    private void maybeAddKeyspace(KSMetaData ksm)
 +    private void maybeAddKeyspace(KeyspaceMetadata ksm)
      {
          try
          {
@@@ -1075,18 -1051,21 +1063,21 @@@
          // (#8162 being an example), so even if the table definition exists, we still need to force the "current"
          // version of the schema, the one the node will be expecting.
  
 -        KSMetaData defined = Schema.instance.getKSMetaData(expected.name);
 +        KeyspaceMetadata defined = Schema.instance.getKSMetaData(expected.name);
+         // If the keyspace doesn't exist, create it
          if (defined == null)
          {
-             // The keyspace doesn't exist, create it
              maybeAddKeyspace(expected);
-             return;
+             defined = Schema.instance.getKSMetaData(expected.name);
          }
  
          // While the keyspace exists, it might miss table or have outdated one
+         // There is also the potential for a race, as schema migrations add the bare
+         // keyspace into Schema.instance before adding its tables, so double check that
+         // all the expected tables are present
 -        for (CFMetaData expectedTable : expected.cfMetaData().values())
 +        for (CFMetaData expectedTable : expected.tables)
          {
 -            CFMetaData definedTable = defined.cfMetaData().get(expectedTable.cfName);
 +            CFMetaData definedTable = defined.tables.get(expectedTable.cfName).orElse(null);
              if (definedTable == null || !definedTable.equals(expectedTable))
                  MigrationManager.forceAnnounceNewColumnFamily(expectedTable);
          }


[03/10] cassandra git commit: Always verify expected tables in pseudo-system keyspaces at startup

Posted by sa...@apache.org.
Always verify expected tables in pseudo-system keyspaces at startup

Patch by Sam Tunnicliffe; reviewed by Sylvain Lebresne for
CASSANDRA-10761


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/15f03ab4
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/15f03ab4
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/15f03ab4

Branch: refs/heads/cassandra-3.1
Commit: 15f03ab446854cf4d2999c3785d145c89bc3a3e4
Parents: e491f05
Author: Sam Tunnicliffe <sa...@beobal.com>
Authored: Wed Nov 25 16:45:02 2015 +0000
Committer: Sam Tunnicliffe <sa...@beobal.com>
Committed: Thu Dec 3 11:44:25 2015 +0000

----------------------------------------------------------------------
 CHANGES.txt                                      |  4 ++++
 src/java/org/apache/cassandra/config/Schema.java |  4 +++-
 .../apache/cassandra/db/ColumnFamilyStore.java   |  4 ++--
 src/java/org/apache/cassandra/db/Keyspace.java   | 12 ++++++------
 .../apache/cassandra/service/StorageService.java | 19 +++++--------------
 5 files changed, 20 insertions(+), 23 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/15f03ab4/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 787d145..12accac 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,3 +1,7 @@
+2.2.5
+ * Verify tables in pseudo-system keyspaces at startup (CASSANDRA-10761)
+
+
 2.2.4
  * Show CQL help in cqlsh in web browser (CASSANDRA-7225)
  * Serialize on disk the proper SSTable compression ratio (CASSANDRA-10775)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/15f03ab4/src/java/org/apache/cassandra/config/Schema.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/Schema.java b/src/java/org/apache/cassandra/config/Schema.java
index 00c9358..110029e 100644
--- a/src/java/org/apache/cassandra/config/Schema.java
+++ b/src/java/org/apache/cassandra/config/Schema.java
@@ -525,8 +525,10 @@ public class Schema
         // since we're going to call initCf on the new one manually
         Keyspace.open(cfm.ksName);
 
+        // init the new CF before switching the KSM to the new one
+        // to avoid races as in CASSANDRA-10761
+        Keyspace.open(cfm.ksName).initCf(cfm, true);
         setKeyspaceDefinition(ksm);
-        Keyspace.open(ksm.name).initCf(cfm.cfId, cfm.cfName, true);
         MigrationManager.instance.notifyCreateColumnFamily(cfm);
     }
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/15f03ab4/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 8fefae2..5325c8a 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -506,9 +506,9 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
     }
 
 
-    public static ColumnFamilyStore createColumnFamilyStore(Keyspace keyspace, String columnFamily, boolean loadSSTables)
+    public static ColumnFamilyStore createColumnFamilyStore(Keyspace keyspace, CFMetaData metadata, boolean loadSSTables)
     {
-        return createColumnFamilyStore(keyspace, columnFamily, StorageService.getPartitioner(), Schema.instance.getCFMetaData(keyspace.getName(), columnFamily), loadSSTables);
+        return createColumnFamilyStore(keyspace, metadata.cfName, StorageService.getPartitioner(), metadata, loadSSTables);
     }
 
     public static synchronized ColumnFamilyStore createColumnFamilyStore(Keyspace keyspace,

http://git-wip-us.apache.org/repos/asf/cassandra/blob/15f03ab4/src/java/org/apache/cassandra/db/Keyspace.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/Keyspace.java b/src/java/org/apache/cassandra/db/Keyspace.java
index 92a0950..225369c 100644
--- a/src/java/org/apache/cassandra/db/Keyspace.java
+++ b/src/java/org/apache/cassandra/db/Keyspace.java
@@ -267,7 +267,7 @@ public class Keyspace
         for (CFMetaData cfm : new ArrayList<>(metadata.cfMetaData().values()))
         {
             logger.trace("Initializing {}.{}", getName(), cfm.cfName);
-            initCf(cfm.cfId, cfm.cfName, loadSSTables);
+            initCf(cfm, loadSSTables);
         }
     }
 
@@ -330,25 +330,25 @@ public class Keyspace
     /**
      * adds a cf to internal structures, ends up creating disk files).
      */
-    public void initCf(UUID cfId, String cfName, boolean loadSSTables)
+    public void initCf(CFMetaData metadata, boolean loadSSTables)
     {
-        ColumnFamilyStore cfs = columnFamilyStores.get(cfId);
+        ColumnFamilyStore cfs = columnFamilyStores.get(metadata.cfId);
 
         if (cfs == null)
         {
             // CFS being created for the first time, either on server startup or new CF being added.
             // We don't worry about races here; startup is safe, and adding multiple idential CFs
             // simultaneously is a "don't do that" scenario.
-            ColumnFamilyStore oldCfs = columnFamilyStores.putIfAbsent(cfId, ColumnFamilyStore.createColumnFamilyStore(this, cfName, loadSSTables));
+            ColumnFamilyStore oldCfs = columnFamilyStores.putIfAbsent(metadata.cfId, ColumnFamilyStore.createColumnFamilyStore(this, metadata, loadSSTables));
             // CFS mbean instantiation will error out before we hit this, but in case that changes...
             if (oldCfs != null)
-                throw new IllegalStateException("added multiple mappings for cf id " + cfId);
+                throw new IllegalStateException("added multiple mappings for cf id " + metadata.cfId);
         }
         else
         {
             // re-initializing an existing CF.  This will happen if you cleared the schema
             // on this node and it's getting repopulated from the rest of the cluster.
-            assert cfs.name.equals(cfName);
+            assert cfs.name.equals(metadata.cfName);
             cfs.metadata.reload();
             cfs.reload();
         }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/15f03ab4/src/java/org/apache/cassandra/service/StorageService.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java
index fec3750..68e19c5 100644
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@ -1026,18 +1026,6 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         MigrationManager.instance.register(new AuthMigrationListener());
     }
 
-    private void maybeAddTable(CFMetaData cfm)
-    {
-        try
-        {
-            MigrationManager.announceNewColumnFamily(cfm);
-        }
-        catch (AlreadyExistsException e)
-        {
-            logger.debug("Attempted to create new table {}, but it already exists", cfm.cfName);
-        }
-    }
-
     private void maybeAddKeyspace(KSMetaData ksm)
     {
         try
@@ -1064,14 +1052,17 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         // version of the schema, the one the node will be expecting.
 
         KSMetaData defined = Schema.instance.getKSMetaData(expected.name);
+        // If the keyspace doesn't exist, create it
         if (defined == null)
         {
-            // The keyspace doesn't exist, create it
             maybeAddKeyspace(expected);
-            return;
+            defined = Schema.instance.getKSMetaData(expected.name);
         }
 
         // While the keyspace exists, it might miss table or have outdated one
+        // There is also the potential for a race, as schema migrations add the bare
+        // keyspace into Schema.instance before adding its tables, so double check that
+        // all the expected tables are present
         for (CFMetaData expectedTable : expected.cfMetaData().values())
         {
             CFMetaData definedTable = defined.cfMetaData().get(expectedTable.cfName);


[06/10] cassandra git commit: Merge branch 'cassandra-2.2' into cassandra-3.0

Posted by sa...@apache.org.
Merge branch 'cassandra-2.2' into cassandra-3.0


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/9784be5e
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/9784be5e
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/9784be5e

Branch: refs/heads/cassandra-3.0
Commit: 9784be5e076ce13426813493662378b52228dd57
Parents: d7068fd 15f03ab
Author: Sam Tunnicliffe <sa...@beobal.com>
Authored: Thu Dec 3 11:45:03 2015 +0000
Committer: Sam Tunnicliffe <sa...@beobal.com>
Committed: Thu Dec 3 11:45:50 2015 +0000

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../org/apache/cassandra/config/Schema.java     | 41 +++++++++-----------
 .../apache/cassandra/db/ColumnFamilyStore.java  |  4 +-
 src/java/org/apache/cassandra/db/Keyspace.java  | 14 +++----
 .../cassandra/service/StorageService.java       | 19 +++------
 5 files changed, 32 insertions(+), 47 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/9784be5e/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index a82dcfc,12accac..4dd1b97
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,22 -1,8 +1,23 @@@
 -2.2.5
 +3.0.1
 + * Fix NPE on stream read error (CASSANDRA-10771)
 + * Normalize cqlsh DESC output (CASSANDRA-10431)
 + * Rejects partition range deletions when columns are specified (CASSANDRA-10739)
 + * Fix error when saving cached key for old format sstable (CASSANDRA-10778)
 + * Invalidate prepared statements on DROP INDEX (CASSANDRA-10758)
 + * Fix SELECT statement with IN restrictions on partition key,
 +   ORDER BY and LIMIT (CASSANDRA-10729)
 + * Improve stress performance over 1k threads (CASSANDRA-7217)
 + * Wait for migration responses to complete before bootstrapping (CASSANDRA-10731)
 + * Unable to create a function with argument of type Inet (CASSANDRA-10741)
 + * Fix backward incompatibiliy in CqlInputFormat (CASSANDRA-10717)
 + * Correctly preserve deletion info on updated rows when notifying indexers
 +   of single-row deletions (CASSANDRA-10694)
 + * Notify indexers of partition delete during cleanup (CASSANDRA-10685)
 + * Keep the file open in trySkipCache (CASSANDRA-10669)
 + * Updated trigger example (CASSANDRA-10257)
 +Merged from 2.2:
+  * Verify tables in pseudo-system keyspaces at startup (CASSANDRA-10761)
 -
 -
 -2.2.4
 + * Fix IllegalArgumentException in DataOutputBuffer.reallocate for large buffers (CASSANDRA-10592)
   * Show CQL help in cqlsh in web browser (CASSANDRA-7225)
   * Serialize on disk the proper SSTable compression ratio (CASSANDRA-10775)
   * Reject index queries while the index is building (CASSANDRA-8505)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/9784be5e/src/java/org/apache/cassandra/config/Schema.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/config/Schema.java
index 569c87d,110029e..cf3f569
--- a/src/java/org/apache/cassandra/config/Schema.java
+++ b/src/java/org/apache/cassandra/config/Schema.java
@@@ -605,19 -515,20 +605,17 @@@ public class Schem
      public void addTable(CFMetaData cfm)
      {
          assert getCFMetaData(cfm.ksName, cfm.cfName) == null;
 -        KSMetaData ksm = getKSMetaData(cfm.ksName).cloneWithTableAdded(cfm);
 -
 -        logger.info("Loading {}", cfm);
  
-         update(cfm.ksName, ks ->
-         {
-             load(cfm);
- 
-             // make sure it's init-ed w/ the old definitions first,
-             // since we're going to call initCf on the new one manually
-             Keyspace.open(cfm.ksName);
- 
-             return ks.withSwapped(ks.tables.with(cfm));
-         });
- 
-         Keyspace.open(cfm.ksName).initCf(cfm.cfId, cfm.cfName, true);
++        // Make sure the keyspace is initialized and initialize the table.
++        Keyspace.open(cfm.ksName).initCf(cfm, true);
++        // Update the keyspaces map with the updated metadata
++        update(cfm.ksName, ks -> ks.withSwapped(ks.tables.with(cfm)));
++        // Update the table ID <-> table name map (cfIdMap)
+         load(cfm);
+ 
 -        // make sure it's init-ed w/ the old definitions first,
 -        // since we're going to call initCf on the new one manually
 -        Keyspace.open(cfm.ksName);
 -
+         // init the new CF before switching the KSM to the new one
+         // to avoid races as in CASSANDRA-10761
+         Keyspace.open(cfm.ksName).initCf(cfm, true);
 -        setKeyspaceDefinition(ksm);
          MigrationManager.instance.notifyCreateColumnFamily(cfm);
      }
  
@@@ -659,80 -566,37 +657,77 @@@
          CommitLog.instance.forceRecycleAllSegments(Collections.singleton(cfm.cfId));
      }
  
 -    public void addType(UserType ut)
 +    public void addView(ViewDefinition view)
      {
 -        KSMetaData ksm = getKSMetaData(ut.keyspace);
 -        assert ksm != null;
 +        assert getCFMetaData(view.ksName, view.viewName) == null;
  
-         update(view.ksName, ks ->
-         {
-             load(view);
- 
-             // make sure it's init-ed w/ the old definitions first,
-             // since we're going to call initCf on the new one manually
-             Keyspace.open(view.ksName);
 -        logger.info("Loading {}", ut);
++        Keyspace keyspace = Keyspace.open(view.ksName);
  
-             return ks.withSwapped(ks.views.with(view));
-         });
 -        ksm.userTypes.addType(ut);
++        // Make sure the keyspace is initialized and initialize the table.
++        keyspace.initCf(view.metadata, true);
++        // Update the keyspaces map with the updated metadata
++        update(view.ksName, ks -> ks.withSwapped(ks.views.with(view)));
++        // Update the table ID <-> table name map (cfIdMap)
++        load(view);
  
-         Keyspace.open(view.ksName).initCf(view.metadata.cfId, view.viewName, true);
-         Keyspace.open(view.ksName).viewManager.reload();
 -        MigrationManager.instance.notifyCreateUserType(ut);
++        keyspace.viewManager.reload();
 +        MigrationManager.instance.notifyCreateView(view);
      }
  
 -    public void updateType(UserType ut)
 +    public void updateView(ViewDefinition view)
      {
 -        KSMetaData ksm = getKSMetaData(ut.keyspace);
 -        assert ksm != null;
 +        ViewDefinition current = getKSMetaData(view.ksName).views.get(view.viewName).get();
 +        boolean changeAffectsStatements = current.metadata.apply(view.metadata);
 +
 +        Keyspace keyspace = Keyspace.open(current.ksName);
 +        keyspace.getColumnFamilyStore(current.viewName).reload();
 +        Keyspace.open(current.ksName).viewManager.update(current.viewName);
 +        MigrationManager.instance.notifyUpdateView(current, changeAffectsStatements);
 +    }
 +
 +    public void dropView(String ksName, String viewName)
 +    {
 +        KeyspaceMetadata oldKsm = getKSMetaData(ksName);
 +        assert oldKsm != null;
 +        ColumnFamilyStore cfs = Keyspace.open(ksName).getColumnFamilyStore(viewName);
 +        assert cfs != null;
 +
 +        // make sure all the indexes are dropped, or else.
 +        cfs.indexManager.markAllIndexesRemoved();
  
 -        logger.info("Updating {}", ut);
 +        // reinitialize the keyspace.
 +        ViewDefinition view = oldKsm.views.get(viewName).get();
 +        KeyspaceMetadata newKsm = oldKsm.withSwapped(oldKsm.views.without(viewName));
  
 -        ksm.userTypes.addType(ut);
 +        unload(view);
 +        setKeyspaceMetadata(newKsm);
  
 -        MigrationManager.instance.notifyUpdateUserType(ut);
 +        CompactionManager.instance.interruptCompactionFor(Collections.singleton(view.metadata), true);
 +
 +        if (DatabaseDescriptor.isAutoSnapshot())
 +            cfs.snapshot(Keyspace.getTimestampedSnapshotName(cfs.name));
 +        Keyspace.open(ksName).dropCf(view.metadata.cfId);
 +        Keyspace.open(ksName).viewManager.reload();
 +        MigrationManager.instance.notifyDropView(view);
 +
 +        CommitLog.instance.forceRecycleAllSegments(Collections.singleton(view.metadata.cfId));
      }
  
 -    public void dropType(UserType ut)
 +    public void addType(UserType ut)
      {
 -        KSMetaData ksm = getKSMetaData(ut.keyspace);
 -        assert ksm != null;
 +        update(ut.keyspace, ks -> ks.withSwapped(ks.types.with(ut)));
 +        MigrationManager.instance.notifyCreateUserType(ut);
 +    }
  
 -        ksm.userTypes.removeType(ut);
 +    public void updateType(UserType ut)
 +    {
 +        update(ut.keyspace, ks -> ks.withSwapped(ks.types.without(ut.name).with(ut)));
 +        MigrationManager.instance.notifyUpdateUserType(ut);
 +    }
  
 +    public void dropType(UserType ut)
 +    {
 +        update(ut.keyspace, ks -> ks.withSwapped(ks.types.without(ut.name)));
          MigrationManager.instance.notifyDropUserType(ut);
      }
  

http://git-wip-us.apache.org/repos/asf/cassandra/blob/9784be5e/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 38c99ea,5325c8a..f0adf39
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@@ -532,9 -506,9 +532,9 @@@ public class ColumnFamilyStore implemen
      }
  
  
-     public static ColumnFamilyStore createColumnFamilyStore(Keyspace keyspace, String columnFamily, boolean loadSSTables)
+     public static ColumnFamilyStore createColumnFamilyStore(Keyspace keyspace, CFMetaData metadata, boolean loadSSTables)
      {
-         return createColumnFamilyStore(keyspace, columnFamily, Schema.instance.getCFMetaData(keyspace.getName(), columnFamily), loadSSTables);
 -        return createColumnFamilyStore(keyspace, metadata.cfName, StorageService.getPartitioner(), metadata, loadSSTables);
++        return createColumnFamilyStore(keyspace, metadata.cfName, metadata, loadSSTables);
      }
  
      public static synchronized ColumnFamilyStore createColumnFamilyStore(Keyspace keyspace,

http://git-wip-us.apache.org/repos/asf/cassandra/blob/9784be5e/src/java/org/apache/cassandra/db/Keyspace.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/Keyspace.java
index eb28b2c,225369c..7b4f79b
--- a/src/java/org/apache/cassandra/db/Keyspace.java
+++ b/src/java/org/apache/cassandra/db/Keyspace.java
@@@ -298,16 -264,14 +298,16 @@@ public class Keyspac
          createReplicationStrategy(metadata);
  
          this.metric = new KeyspaceMetrics(this);
 -        for (CFMetaData cfm : new ArrayList<>(metadata.cfMetaData().values()))
 +        this.viewManager = new ViewManager(this);
 +        for (CFMetaData cfm : metadata.tablesAndViews())
          {
              logger.trace("Initializing {}.{}", getName(), cfm.cfName);
-             initCf(cfm.cfId, cfm.cfName, loadSSTables);
+             initCf(cfm, loadSSTables);
          }
 +        this.viewManager.reload();
      }
  
 -    private Keyspace(KSMetaData metadata)
 +    private Keyspace(KeyspaceMetadata metadata)
      {
          this.metadata = metadata;
          createReplicationStrategy(metadata);
@@@ -376,7 -348,8 +374,7 @@@
          {
              // re-initializing an existing CF.  This will happen if you cleared the schema
              // on this node and it's getting repopulated from the rest of the cluster.
-             assert cfs.name.equals(cfName);
+             assert cfs.name.equals(metadata.cfName);
 -            cfs.metadata.reload();
              cfs.reload();
          }
      }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/9784be5e/src/java/org/apache/cassandra/service/StorageService.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/service/StorageService.java
index 0b910f0,68e19c5..2332cea
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@@ -1038,19 -1026,7 +1038,7 @@@ public class StorageService extends Not
          MigrationManager.instance.register(new AuthMigrationListener());
      }
  
-     private void maybeAddTable(CFMetaData cfm)
-     {
-         try
-         {
-             MigrationManager.announceNewColumnFamily(cfm);
-         }
-         catch (AlreadyExistsException e)
-         {
-             logger.debug("Attempted to create new table {}, but it already exists", cfm.cfName);
-         }
-     }
- 
 -    private void maybeAddKeyspace(KSMetaData ksm)
 +    private void maybeAddKeyspace(KeyspaceMetadata ksm)
      {
          try
          {
@@@ -1075,18 -1051,21 +1063,21 @@@
          // (#8162 being an example), so even if the table definition exists, we still need to force the "current"
          // version of the schema, the one the node will be expecting.
  
 -        KSMetaData defined = Schema.instance.getKSMetaData(expected.name);
 +        KeyspaceMetadata defined = Schema.instance.getKSMetaData(expected.name);
+         // If the keyspace doesn't exist, create it
          if (defined == null)
          {
-             // The keyspace doesn't exist, create it
              maybeAddKeyspace(expected);
-             return;
+             defined = Schema.instance.getKSMetaData(expected.name);
          }
  
          // While the keyspace exists, it might miss table or have outdated one
+         // There is also the potential for a race, as schema migrations add the bare
+         // keyspace into Schema.instance before adding its tables, so double check that
+         // all the expected tables are present
 -        for (CFMetaData expectedTable : expected.cfMetaData().values())
 +        for (CFMetaData expectedTable : expected.tables)
          {
 -            CFMetaData definedTable = defined.cfMetaData().get(expectedTable.cfName);
 +            CFMetaData definedTable = defined.tables.get(expectedTable.cfName).orElse(null);
              if (definedTable == null || !definedTable.equals(expectedTable))
                  MigrationManager.forceAnnounceNewColumnFamily(expectedTable);
          }


[02/10] cassandra git commit: Always verify expected tables in pseudo-system keyspaces at startup

Posted by sa...@apache.org.
Always verify expected tables in pseudo-system keyspaces at startup

Patch by Sam Tunnicliffe; reviewed by Sylvain Lebresne for
CASSANDRA-10761


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/15f03ab4
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/15f03ab4
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/15f03ab4

Branch: refs/heads/cassandra-3.0
Commit: 15f03ab446854cf4d2999c3785d145c89bc3a3e4
Parents: e491f05
Author: Sam Tunnicliffe <sa...@beobal.com>
Authored: Wed Nov 25 16:45:02 2015 +0000
Committer: Sam Tunnicliffe <sa...@beobal.com>
Committed: Thu Dec 3 11:44:25 2015 +0000

----------------------------------------------------------------------
 CHANGES.txt                                      |  4 ++++
 src/java/org/apache/cassandra/config/Schema.java |  4 +++-
 .../apache/cassandra/db/ColumnFamilyStore.java   |  4 ++--
 src/java/org/apache/cassandra/db/Keyspace.java   | 12 ++++++------
 .../apache/cassandra/service/StorageService.java | 19 +++++--------------
 5 files changed, 20 insertions(+), 23 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/15f03ab4/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 787d145..12accac 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,3 +1,7 @@
+2.2.5
+ * Verify tables in pseudo-system keyspaces at startup (CASSANDRA-10761)
+
+
 2.2.4
  * Show CQL help in cqlsh in web browser (CASSANDRA-7225)
  * Serialize on disk the proper SSTable compression ratio (CASSANDRA-10775)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/15f03ab4/src/java/org/apache/cassandra/config/Schema.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/Schema.java b/src/java/org/apache/cassandra/config/Schema.java
index 00c9358..110029e 100644
--- a/src/java/org/apache/cassandra/config/Schema.java
+++ b/src/java/org/apache/cassandra/config/Schema.java
@@ -525,8 +525,10 @@ public class Schema
         // since we're going to call initCf on the new one manually
         Keyspace.open(cfm.ksName);
 
+        // init the new CF before switching the KSM to the new one
+        // to avoid races as in CASSANDRA-10761
+        Keyspace.open(cfm.ksName).initCf(cfm, true);
         setKeyspaceDefinition(ksm);
-        Keyspace.open(ksm.name).initCf(cfm.cfId, cfm.cfName, true);
         MigrationManager.instance.notifyCreateColumnFamily(cfm);
     }
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/15f03ab4/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 8fefae2..5325c8a 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -506,9 +506,9 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
     }
 
 
-    public static ColumnFamilyStore createColumnFamilyStore(Keyspace keyspace, String columnFamily, boolean loadSSTables)
+    public static ColumnFamilyStore createColumnFamilyStore(Keyspace keyspace, CFMetaData metadata, boolean loadSSTables)
     {
-        return createColumnFamilyStore(keyspace, columnFamily, StorageService.getPartitioner(), Schema.instance.getCFMetaData(keyspace.getName(), columnFamily), loadSSTables);
+        return createColumnFamilyStore(keyspace, metadata.cfName, StorageService.getPartitioner(), metadata, loadSSTables);
     }
 
     public static synchronized ColumnFamilyStore createColumnFamilyStore(Keyspace keyspace,

http://git-wip-us.apache.org/repos/asf/cassandra/blob/15f03ab4/src/java/org/apache/cassandra/db/Keyspace.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/Keyspace.java b/src/java/org/apache/cassandra/db/Keyspace.java
index 92a0950..225369c 100644
--- a/src/java/org/apache/cassandra/db/Keyspace.java
+++ b/src/java/org/apache/cassandra/db/Keyspace.java
@@ -267,7 +267,7 @@ public class Keyspace
         for (CFMetaData cfm : new ArrayList<>(metadata.cfMetaData().values()))
         {
             logger.trace("Initializing {}.{}", getName(), cfm.cfName);
-            initCf(cfm.cfId, cfm.cfName, loadSSTables);
+            initCf(cfm, loadSSTables);
         }
     }
 
@@ -330,25 +330,25 @@ public class Keyspace
     /**
      * adds a cf to internal structures, ends up creating disk files).
      */
-    public void initCf(UUID cfId, String cfName, boolean loadSSTables)
+    public void initCf(CFMetaData metadata, boolean loadSSTables)
     {
-        ColumnFamilyStore cfs = columnFamilyStores.get(cfId);
+        ColumnFamilyStore cfs = columnFamilyStores.get(metadata.cfId);
 
         if (cfs == null)
         {
             // CFS being created for the first time, either on server startup or new CF being added.
             // We don't worry about races here; startup is safe, and adding multiple idential CFs
             // simultaneously is a "don't do that" scenario.
-            ColumnFamilyStore oldCfs = columnFamilyStores.putIfAbsent(cfId, ColumnFamilyStore.createColumnFamilyStore(this, cfName, loadSSTables));
+            ColumnFamilyStore oldCfs = columnFamilyStores.putIfAbsent(metadata.cfId, ColumnFamilyStore.createColumnFamilyStore(this, metadata, loadSSTables));
             // CFS mbean instantiation will error out before we hit this, but in case that changes...
             if (oldCfs != null)
-                throw new IllegalStateException("added multiple mappings for cf id " + cfId);
+                throw new IllegalStateException("added multiple mappings for cf id " + metadata.cfId);
         }
         else
         {
             // re-initializing an existing CF.  This will happen if you cleared the schema
             // on this node and it's getting repopulated from the rest of the cluster.
-            assert cfs.name.equals(cfName);
+            assert cfs.name.equals(metadata.cfName);
             cfs.metadata.reload();
             cfs.reload();
         }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/15f03ab4/src/java/org/apache/cassandra/service/StorageService.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java
index fec3750..68e19c5 100644
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@ -1026,18 +1026,6 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         MigrationManager.instance.register(new AuthMigrationListener());
     }
 
-    private void maybeAddTable(CFMetaData cfm)
-    {
-        try
-        {
-            MigrationManager.announceNewColumnFamily(cfm);
-        }
-        catch (AlreadyExistsException e)
-        {
-            logger.debug("Attempted to create new table {}, but it already exists", cfm.cfName);
-        }
-    }
-
     private void maybeAddKeyspace(KSMetaData ksm)
     {
         try
@@ -1064,14 +1052,17 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         // version of the schema, the one the node will be expecting.
 
         KSMetaData defined = Schema.instance.getKSMetaData(expected.name);
+        // If the keyspace doesn't exist, create it
         if (defined == null)
         {
-            // The keyspace doesn't exist, create it
             maybeAddKeyspace(expected);
-            return;
+            defined = Schema.instance.getKSMetaData(expected.name);
         }
 
         // While the keyspace exists, it might miss table or have outdated one
+        // There is also the potential for a race, as schema migrations add the bare
+        // keyspace into Schema.instance before adding its tables, so double check that
+        // all the expected tables are present
         for (CFMetaData expectedTable : expected.cfMetaData().values())
         {
             CFMetaData definedTable = defined.cfMetaData().get(expectedTable.cfName);