You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ma...@apache.org on 2016/06/23 13:18:46 UTC

[01/12] cassandra git commit: Disable autocompaction during drain

Repository: cassandra
Updated Branches:
  refs/heads/cassandra-2.1 3c8421a33 -> 9358e589e
  refs/heads/cassandra-2.2 225232a9e -> 5d0d30e4d
  refs/heads/cassandra-3.0 5ee6e7bc1 -> ffd6bab35
  refs/heads/trunk 84e40cb6b -> fc7785bfe


Disable autocompaction during drain

Patch by marcuse; reviewed by Alex Petrov for CASSANDRA-11878


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/5af5bafc
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/5af5bafc
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/5af5bafc

Branch: refs/heads/cassandra-3.0
Commit: 5af5bafc87f3efe0388c003b3923ca3452e1e261
Parents: 5ee6e7b
Author: Marcus Eriksson <ma...@apache.org>
Authored: Thu May 26 13:27:08 2016 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Thu Jun 23 13:47:32 2016 +0200

----------------------------------------------------------------------
 CHANGES.txt                                               | 1 +
 src/java/org/apache/cassandra/service/StorageService.java | 6 ++++++
 2 files changed, 7 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/5af5bafc/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index f7e854d..7db3afb 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 3.0.8
+ * Disable autocompaction during drain (CASSANDRA-11878)
  * Add a metrics timer to MemtablePool and use it to track time spent blocked on memory in MemtableAllocator (CASSANDRA-11327)
  * Fix upgrading schema with super columns with non-text subcomparators (CASSANDRA-12023)
  * Add TimeWindowCompactionStrategy (CASSANDRA-9666)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5af5bafc/src/java/org/apache/cassandra/service/StorageService.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java
index 394220d..5e14d4f 100644
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@ -3978,6 +3978,12 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         StorageProxy.instance.verifyNoHintsInProgress();
 
         setMode(Mode.DRAINING, "flushing column families", false);
+
+        // disable autocompaction - we don't want to start any new compactions while we are draining
+        for (Keyspace keyspace : Keyspace.all())
+            for (ColumnFamilyStore cfs : keyspace.getColumnFamilyStores())
+                cfs.disableAutoCompaction();
+
         // count CFs first, since forceFlush could block for the flushWriter to get a queue slot empty
         totalCFs = 0;
         for (Keyspace keyspace : Keyspace.nonSystem())


[12/12] cassandra git commit: Merge branch 'cassandra-3.0' into trunk

Posted by ma...@apache.org.
Merge branch 'cassandra-3.0' into trunk


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/fc7785bf
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/fc7785bf
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/fc7785bf

Branch: refs/heads/trunk
Commit: fc7785bfe643f516398a3d01655b009102f798ca
Parents: 84e40cb ffd6bab
Author: Marcus Eriksson <ma...@apache.org>
Authored: Thu Jun 23 15:15:25 2016 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Thu Jun 23 15:15:25 2016 +0200

----------------------------------------------------------------------
 CHANGES.txt                                             |  2 ++
 .../apache/cassandra/service/ActiveRepairService.java   | 12 +++++++++++-
 .../org/apache/cassandra/service/StorageService.java    |  6 ++++++
 3 files changed, 19 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/fc7785bf/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 474ad5c,0e9935d..f5843ec
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,22 -1,5 +1,23 @@@
 -3.0.8
 +3.8
 + * Support seek() in EncryptedFileSegmentInputStream (CASSANDRA-11957)
 + * SSTable tools mishandling LocalPartitioner (CASSANDRA-12002)
 + * When SEPWorker assigned work, set thread name to match pool (CASSANDRA-11966)
 + * Add cross-DC latency metrics (CASSANDRA-11596)
 + * Allow terms in selection clause (CASSANDRA-10783)
 + * Add bind variables to trace (CASSANDRA-11719)
 + * Switch counter shards' clock to timestamps (CASSANDRA-9811)
 + * Introduce HdrHistogram and response/service/wait separation to stress tool (CASSANDRA-11853)
 + * entry-weighers in QueryProcessor should respect partitionKeyBindIndexes field (CASSANDRA-11718)
 + * Support older ant versions (CASSANDRA-11807)
 + * Estimate compressed on disk size when deciding if sstable size limit reached (CASSANDRA-11623)
 + * cassandra-stress profiles should support case sensitive schemas (CASSANDRA-11546)
 + * Remove DatabaseDescriptor dependency from FileUtils (CASSANDRA-11578)
 + * Faster streaming (CASSANDRA-9766)
 + * Add prepared query parameter to trace for "Execute CQL3 prepared query" session (CASSANDRA-11425)
 + * Add repaired percentage metric (CASSANDRA-11503)
 + * Add Change-Data-Capture (CASSANDRA-8844)
 +Merged from 3.0:
+  * Disable autocompaction during drain (CASSANDRA-11878)
   * Add a metrics timer to MemtablePool and use it to track time spent blocked on memory in MemtableAllocator (CASSANDRA-11327)
   * Fix upgrading schema with super columns with non-text subcomparators (CASSANDRA-12023)
   * Add TimeWindowCompactionStrategy (CASSANDRA-9666)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/fc7785bf/src/java/org/apache/cassandra/service/ActiveRepairService.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/fc7785bf/src/java/org/apache/cassandra/service/StorageService.java
----------------------------------------------------------------------


[09/12] cassandra git commit: Merge branch 'cassandra-2.1' into cassandra-2.2

Posted by ma...@apache.org.
Merge branch 'cassandra-2.1' into cassandra-2.2


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/5d0d30e4
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/5d0d30e4
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/5d0d30e4

Branch: refs/heads/cassandra-2.2
Commit: 5d0d30e4d3d587a155d42eefe23f8586cdf06d94
Parents: 225232a 9358e58
Author: Marcus Eriksson <ma...@apache.org>
Authored: Thu Jun 23 15:10:35 2016 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Thu Jun 23 15:10:35 2016 +0200

----------------------------------------------------------------------
 CHANGES.txt                                             |  1 +
 .../apache/cassandra/service/ActiveRepairService.java   | 12 +++++++++++-
 2 files changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/5d0d30e4/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index b366d21,620568d..8fbdf3b
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,35 -1,5 +1,36 @@@
 -2.1.15
 +2.2.7
 + * Allow nodetool info to run with readonly JMX access (CASSANDRA-11755)
 + * Validate bloom_filter_fp_chance against lowest supported
 +   value when the table is created (CASSANDRA-11920)
 + * RandomAccessReader: call isEOF() only when rebuffering, not for every read operation (CASSANDRA-12013)
 + * Don't send erroneous NEW_NODE notifications on restart (CASSANDRA-11038)
 + * StorageService shutdown hook should use a volatile variable (CASSANDRA-11984)
 + * Persist local metadata earlier in startup sequence (CASSANDRA-11742)
 + * Run CommitLog tests with different compression settings (CASSANDRA-9039)
 + * cqlsh: fix tab completion for case-sensitive identifiers (CASSANDRA-11664)
 + * Avoid showing estimated key as -1 in tablestats (CASSANDRA-11587)
 + * Fix possible race condition in CommitLog.recover (CASSANDRA-11743)
 + * Enable client encryption in sstableloader with cli options (CASSANDRA-11708)
 + * Possible memory leak in NIODataInputStream (CASSANDRA-11867)
 + * Fix commit log replay after out-of-order flush completion (CASSANDRA-9669)
 + * Add seconds to cqlsh tracing session duration (CASSANDRA-11753)
 + * Prohibit Reverse Counter type as part of the PK (CASSANDRA-9395)
 + * cqlsh: correctly handle non-ascii chars in error messages (CASSANDRA-11626)
 + * Exit JVM if JMX server fails to startup (CASSANDRA-11540)
 + * Produce a heap dump when exiting on OOM (CASSANDRA-9861)
 + * Avoid read repairing purgeable tombstones on range slices (CASSANDRA-11427)
 + * Restore ability to filter on clustering columns when using a 2i (CASSANDRA-11510)
 + * JSON datetime formatting needs timezone (CASSANDRA-11137)
 + * Fix is_dense recalculation for Thrift-updated tables (CASSANDRA-11502)
 + * Remove unnescessary file existence check during anticompaction (CASSANDRA-11660)
 + * Add missing files to debian packages (CASSANDRA-11642)
 + * Avoid calling Iterables::concat in loops during ModificationStatement::getFunctions (CASSANDRA-11621)
 + * cqlsh: COPY FROM should use regular inserts for single statement batches and
 +   report errors correctly if workers processes crash on initialization (CASSANDRA-11474)
 + * Always close cluster with connection in CqlRecordWriter (CASSANDRA-11553)
 + * Fix slice queries on ordered COMPACT tables (CASSANDRA-10988)
 +Merged from 2.1:
+  * Don't try to get sstables for non-repairing column families (CASSANDRA-12077)
   * Prevent select statements with clustering key > 64k (CASSANDRA-11882)
   * Avoid marking too many sstables as repaired (CASSANDRA-11696)
   * Fix clock skew corrupting other nodes with paxos (CASSANDRA-11991)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5d0d30e4/src/java/org/apache/cassandra/service/ActiveRepairService.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/service/ActiveRepairService.java
index e111155,4ca1e42..38804b3
--- a/src/java/org/apache/cassandra/service/ActiveRepairService.java
+++ b/src/java/org/apache/cassandra/service/ActiveRepairService.java
@@@ -542,12 -524,14 +542,17 @@@ public class ActiveRepairService implem
           * @return
           */
          @SuppressWarnings("resource")
 -        public synchronized Refs<SSTableReader> getActiveRepairedSSTableRefsForAntiCompaction(UUID cfId)
 +        public synchronized Refs<SSTableReader> getActiveRepairedSSTableRefsForAntiCompaction(UUID cfId, UUID parentSessionId)
          {
              assert marked.contains(cfId);
 -            ImmutableMap.Builder<SSTableReader, Ref<SSTableReader>> references = ImmutableMap.builder();
 -            Iterable<SSTableReader> sstables = getActiveSSTables(cfId);
 -            if (sstables == null)
++            if (!columnFamilyStores.containsKey(cfId))
+                 throw new RuntimeException("Not possible to get sstables for anticompaction for " + cfId);
 +            boolean isSnapshotRepair = columnFamilyStores.get(cfId).snapshotExists(parentSessionId.toString());
 +            ImmutableMap.Builder<SSTableReader, Ref<SSTableReader>> references = ImmutableMap.builder();
-             for (SSTableReader sstable : isSnapshotRepair ? getSSTablesForSnapshotRepair(cfId, parentSessionId) : getActiveSSTables(cfId))
++            Iterable<SSTableReader> sstables = isSnapshotRepair ? getSSTablesForSnapshotRepair(cfId, parentSessionId) : getActiveSSTables(cfId);
++            // we check this above - if columnFamilyStores contains the cfId sstables will not be null
++            assert sstables != null;
+             for (SSTableReader sstable : sstables)
              {
                  Ref<SSTableReader> ref = sstable.tryRef();
                  if (ref == null)
@@@ -559,57 -543,21 +564,59 @@@
          }
  
          /**
 -         * Marks all the unrepaired sstables as repairing unless we have already done so.
 +         * If we are running a snapshot repair we need to find the 'real' sstables when we start anticompaction
           *
 -         * Any of these sstables that are still on disk are then anticompacted once the streaming and validation phases are done.
 +         * We use the generation of the sstables as identifiers instead of the file name to avoid having to parse out the
 +         * actual filename.
           *
           * @param cfId
 -         * @param parentSessionId used to check that we don't start multiple inc repair sessions over the same sstables
 +         * @param parentSessionId
 +         * @return
           */
 -        public synchronized void markSSTablesRepairing(UUID cfId, UUID parentSessionId)
 +        private Set<SSTableReader> getSSTablesForSnapshotRepair(UUID cfId, UUID parentSessionId)
          {
 -            if (!marked.contains(cfId))
 +            Set<SSTableReader> activeSSTables = new HashSet<>();
 +            ColumnFamilyStore cfs = columnFamilyStores.get(cfId);
++            if (cfs == null)
++                return null;
 +
 +            Set<Integer> snapshotGenerations = new HashSet<>();
 +            try (Refs<SSTableReader> snapshottedSSTables = cfs.getSnapshotSSTableReader(parentSessionId.toString()))
              {
 -                List<SSTableReader> sstables = columnFamilyStores.get(cfId).select(ColumnFamilyStore.UNREPAIRED_SSTABLES).sstables;
 -                Set<SSTableReader> currentlyRepairing = ActiveRepairService.instance.currentlyRepairing(cfId, parentSessionId);
 -                if (!Sets.intersection(currentlyRepairing, Sets.newHashSet(sstables)).isEmpty())
 +                for (SSTableReader sstable : snapshottedSSTables)
 +                {
 +                    snapshotGenerations.add(sstable.descriptor.generation);
 +                }
 +            }
 +            catch (IOException e)
 +            {
 +                throw new RuntimeException(e);
 +            }
 +            for (SSTableReader sstable : cfs.select(ColumnFamilyStore.CANONICAL_SSTABLES).sstables)
 +                if (snapshotGenerations.contains(sstable.descriptor.generation))
 +                    activeSSTables.add(sstable);
 +            return activeSSTables;
 +        }
 +
 +        public synchronized void maybeSnapshot(UUID cfId, UUID parentSessionId)
 +        {
 +            String snapshotName = parentSessionId.toString();
 +            if (!columnFamilyStores.get(cfId).snapshotExists(snapshotName))
 +            {
 +                Set<SSTableReader> snapshottedSSTables = columnFamilyStores.get(cfId).snapshot(snapshotName, new Predicate<SSTableReader>()
                  {
 +                    public boolean apply(SSTableReader sstable)
 +                    {
 +                        return sstable != null &&
 +                               (!isIncremental || !sstable.isRepaired()) &&
 +                               !(sstable.partitioner instanceof LocalPartitioner) && // exclude SSTables from 2i
 +                               new Bounds<>(sstable.first.getToken(), sstable.last.getToken()).intersects(ranges);
 +                    }
 +                }, true);
 +
 +                if (isAlreadyRepairing(cfId, parentSessionId, snapshottedSSTables))
 +                {
 +                    columnFamilyStores.get(cfId).clearSnapshot(parentSessionId.toString());
                      logger.error("Cannot start multiple repair sessions over the same sstables");
                      throw new RuntimeException("Cannot start multiple repair sessions over the same sstables");
                  }
@@@ -618,32 -566,13 +625,35 @@@
              }
          }
  
 +
 +        /**
 +         * Compares other repairing sstables *generation* to the ones we just snapshotted
 +         *
 +         * we compare generations since the sstables have different paths due to snapshot names
 +         *
 +         * @param cfId id of the column family store
 +         * @param parentSessionId parent repair session
 +         * @param sstables the newly snapshotted sstables
 +         * @return
 +         */
 +        private boolean isAlreadyRepairing(UUID cfId, UUID parentSessionId, Collection<SSTableReader> sstables)
 +        {
 +            Set<SSTableReader> currentlyRepairing = ActiveRepairService.instance.currentlyRepairing(cfId, parentSessionId);
 +            Set<Integer> currentlyRepairingGenerations = new HashSet<>();
 +            Set<Integer> newRepairingGenerations = new HashSet<>();
 +            for (SSTableReader sstable : currentlyRepairing)
 +                currentlyRepairingGenerations.add(sstable.descriptor.generation);
 +            for (SSTableReader sstable : sstables)
 +                newRepairingGenerations.add(sstable.descriptor.generation);
 +
 +            return !Sets.intersection(currentlyRepairingGenerations, newRepairingGenerations).isEmpty();
 +        }
 +
          private Set<SSTableReader> getActiveSSTables(UUID cfId)
          {
 -            if (failed)
 -                return Collections.emptySet();
+             if (!columnFamilyStores.containsKey(cfId))
+                 return null;
+ 
              Set<String> repairedSSTables = sstableMap.get(cfId);
              Set<SSTableReader> activeSSTables = new HashSet<>();
              Set<String> activeSSTableNames = new HashSet<>();


[10/12] cassandra git commit: Merge branch 'cassandra-2.2' into cassandra-3.0

Posted by ma...@apache.org.
Merge branch 'cassandra-2.2' into cassandra-3.0


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/ffd6bab3
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/ffd6bab3
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/ffd6bab3

Branch: refs/heads/trunk
Commit: ffd6bab35290f95402a3b6b41c7af6f1c7d26286
Parents: 5af5baf 5d0d30e
Author: Marcus Eriksson <ma...@apache.org>
Authored: Thu Jun 23 15:13:54 2016 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Thu Jun 23 15:13:54 2016 +0200

----------------------------------------------------------------------
 CHANGES.txt                                             |  1 +
 .../apache/cassandra/service/ActiveRepairService.java   | 12 +++++++++++-
 2 files changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/ffd6bab3/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 7db3afb,8fbdf3b..0e9935d
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -7,36 -2,9 +7,37 @@@ Merged from 2.2
   * Allow nodetool info to run with readonly JMX access (CASSANDRA-11755)
   * Validate bloom_filter_fp_chance against lowest supported
     value when the table is created (CASSANDRA-11920)
 - * RandomAccessReader: call isEOF() only when rebuffering, not for every read operation (CASSANDRA-12013)
   * Don't send erroneous NEW_NODE notifications on restart (CASSANDRA-11038)
   * StorageService shutdown hook should use a volatile variable (CASSANDRA-11984)
 +Merged from 2.1:
++ * Don't try to get sstables for non-repairing column families (CASSANDRA-12077)
 + * Avoid marking too many sstables as repaired (CASSANDRA-11696)
 + * Prevent select statements with clustering key > 64k (CASSANDRA-11882)
 + * Fix clock skew corrupting other nodes with paxos (CASSANDRA-11991)
 + * Remove distinction between non-existing static columns and existing but null in LWTs (CASSANDRA-9842)
 + * Cache local ranges when calculating repair neighbors (CASSANDRA-11934)
 + * Allow LWT operation on static column with only partition keys (CASSANDRA-10532)
 + * Create interval tree over canonical sstables to avoid missing sstables during streaming (CASSANDRA-11886)
 + * cqlsh COPY FROM: shutdown parent cluster after forking, to avoid corrupting SSL connections (CASSANDRA-11749)
 +
 +
 +3.0.7
 + * Fix legacy serialization of Thrift-generated non-compound range tombstones
 +   when communicating with 2.x nodes (CASSANDRA-11930)
 + * Fix Directories instantiations where CFS.initialDirectories should be used (CASSANDRA-11849)
 + * Avoid referencing DatabaseDescriptor in AbstractType (CASSANDRA-11912)
 + * Fix sstables not being protected from removal during index build (CASSANDRA-11905)
 + * cqlsh: Suppress stack trace from Read/WriteFailures (CASSANDRA-11032)
 + * Remove unneeded code to repair index summaries that have
 +   been improperly down-sampled (CASSANDRA-11127)
 + * Avoid WriteTimeoutExceptions during commit log replay due to materialized
 +   view lock contention (CASSANDRA-11891)
 + * Prevent OOM failures on SSTable corruption, improve tests for corruption detection (CASSANDRA-9530)
 + * Use CFS.initialDirectories when clearing snapshots (CASSANDRA-11705)
 + * Allow compaction strategies to disable early open (CASSANDRA-11754)
 + * Refactor Materialized View code (CASSANDRA-11475)
 + * Update Java Driver (CASSANDRA-11615)
 +Merged from 2.2:
   * Persist local metadata earlier in startup sequence (CASSANDRA-11742)
   * Run CommitLog tests with different compression settings (CASSANDRA-9039)
   * cqlsh: fix tab completion for case-sensitive identifiers (CASSANDRA-11664)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/ffd6bab3/src/java/org/apache/cassandra/service/ActiveRepairService.java
----------------------------------------------------------------------


[08/12] cassandra git commit: Merge branch 'cassandra-2.1' into cassandra-2.2

Posted by ma...@apache.org.
Merge branch 'cassandra-2.1' into cassandra-2.2


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/5d0d30e4
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/5d0d30e4
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/5d0d30e4

Branch: refs/heads/trunk
Commit: 5d0d30e4d3d587a155d42eefe23f8586cdf06d94
Parents: 225232a 9358e58
Author: Marcus Eriksson <ma...@apache.org>
Authored: Thu Jun 23 15:10:35 2016 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Thu Jun 23 15:10:35 2016 +0200

----------------------------------------------------------------------
 CHANGES.txt                                             |  1 +
 .../apache/cassandra/service/ActiveRepairService.java   | 12 +++++++++++-
 2 files changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/5d0d30e4/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index b366d21,620568d..8fbdf3b
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,35 -1,5 +1,36 @@@
 -2.1.15
 +2.2.7
 + * Allow nodetool info to run with readonly JMX access (CASSANDRA-11755)
 + * Validate bloom_filter_fp_chance against lowest supported
 +   value when the table is created (CASSANDRA-11920)
 + * RandomAccessReader: call isEOF() only when rebuffering, not for every read operation (CASSANDRA-12013)
 + * Don't send erroneous NEW_NODE notifications on restart (CASSANDRA-11038)
 + * StorageService shutdown hook should use a volatile variable (CASSANDRA-11984)
 + * Persist local metadata earlier in startup sequence (CASSANDRA-11742)
 + * Run CommitLog tests with different compression settings (CASSANDRA-9039)
 + * cqlsh: fix tab completion for case-sensitive identifiers (CASSANDRA-11664)
 + * Avoid showing estimated key as -1 in tablestats (CASSANDRA-11587)
 + * Fix possible race condition in CommitLog.recover (CASSANDRA-11743)
 + * Enable client encryption in sstableloader with cli options (CASSANDRA-11708)
 + * Possible memory leak in NIODataInputStream (CASSANDRA-11867)
 + * Fix commit log replay after out-of-order flush completion (CASSANDRA-9669)
 + * Add seconds to cqlsh tracing session duration (CASSANDRA-11753)
 + * Prohibit Reverse Counter type as part of the PK (CASSANDRA-9395)
 + * cqlsh: correctly handle non-ascii chars in error messages (CASSANDRA-11626)
 + * Exit JVM if JMX server fails to startup (CASSANDRA-11540)
 + * Produce a heap dump when exiting on OOM (CASSANDRA-9861)
 + * Avoid read repairing purgeable tombstones on range slices (CASSANDRA-11427)
 + * Restore ability to filter on clustering columns when using a 2i (CASSANDRA-11510)
 + * JSON datetime formatting needs timezone (CASSANDRA-11137)
 + * Fix is_dense recalculation for Thrift-updated tables (CASSANDRA-11502)
 + * Remove unnescessary file existence check during anticompaction (CASSANDRA-11660)
 + * Add missing files to debian packages (CASSANDRA-11642)
 + * Avoid calling Iterables::concat in loops during ModificationStatement::getFunctions (CASSANDRA-11621)
 + * cqlsh: COPY FROM should use regular inserts for single statement batches and
 +   report errors correctly if workers processes crash on initialization (CASSANDRA-11474)
 + * Always close cluster with connection in CqlRecordWriter (CASSANDRA-11553)
 + * Fix slice queries on ordered COMPACT tables (CASSANDRA-10988)
 +Merged from 2.1:
+  * Don't try to get sstables for non-repairing column families (CASSANDRA-12077)
   * Prevent select statements with clustering key > 64k (CASSANDRA-11882)
   * Avoid marking too many sstables as repaired (CASSANDRA-11696)
   * Fix clock skew corrupting other nodes with paxos (CASSANDRA-11991)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5d0d30e4/src/java/org/apache/cassandra/service/ActiveRepairService.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/service/ActiveRepairService.java
index e111155,4ca1e42..38804b3
--- a/src/java/org/apache/cassandra/service/ActiveRepairService.java
+++ b/src/java/org/apache/cassandra/service/ActiveRepairService.java
@@@ -542,12 -524,14 +542,17 @@@ public class ActiveRepairService implem
           * @return
           */
          @SuppressWarnings("resource")
 -        public synchronized Refs<SSTableReader> getActiveRepairedSSTableRefsForAntiCompaction(UUID cfId)
 +        public synchronized Refs<SSTableReader> getActiveRepairedSSTableRefsForAntiCompaction(UUID cfId, UUID parentSessionId)
          {
              assert marked.contains(cfId);
 -            ImmutableMap.Builder<SSTableReader, Ref<SSTableReader>> references = ImmutableMap.builder();
 -            Iterable<SSTableReader> sstables = getActiveSSTables(cfId);
 -            if (sstables == null)
++            if (!columnFamilyStores.containsKey(cfId))
+                 throw new RuntimeException("Not possible to get sstables for anticompaction for " + cfId);
 +            boolean isSnapshotRepair = columnFamilyStores.get(cfId).snapshotExists(parentSessionId.toString());
 +            ImmutableMap.Builder<SSTableReader, Ref<SSTableReader>> references = ImmutableMap.builder();
-             for (SSTableReader sstable : isSnapshotRepair ? getSSTablesForSnapshotRepair(cfId, parentSessionId) : getActiveSSTables(cfId))
++            Iterable<SSTableReader> sstables = isSnapshotRepair ? getSSTablesForSnapshotRepair(cfId, parentSessionId) : getActiveSSTables(cfId);
++            // we check this above - if columnFamilyStores contains the cfId sstables will not be null
++            assert sstables != null;
+             for (SSTableReader sstable : sstables)
              {
                  Ref<SSTableReader> ref = sstable.tryRef();
                  if (ref == null)
@@@ -559,57 -543,21 +564,59 @@@
          }
  
          /**
 -         * Marks all the unrepaired sstables as repairing unless we have already done so.
 +         * If we are running a snapshot repair we need to find the 'real' sstables when we start anticompaction
           *
 -         * Any of these sstables that are still on disk are then anticompacted once the streaming and validation phases are done.
 +         * We use the generation of the sstables as identifiers instead of the file name to avoid having to parse out the
 +         * actual filename.
           *
           * @param cfId
 -         * @param parentSessionId used to check that we don't start multiple inc repair sessions over the same sstables
 +         * @param parentSessionId
 +         * @return
           */
 -        public synchronized void markSSTablesRepairing(UUID cfId, UUID parentSessionId)
 +        private Set<SSTableReader> getSSTablesForSnapshotRepair(UUID cfId, UUID parentSessionId)
          {
 -            if (!marked.contains(cfId))
 +            Set<SSTableReader> activeSSTables = new HashSet<>();
 +            ColumnFamilyStore cfs = columnFamilyStores.get(cfId);
++            if (cfs == null)
++                return null;
 +
 +            Set<Integer> snapshotGenerations = new HashSet<>();
 +            try (Refs<SSTableReader> snapshottedSSTables = cfs.getSnapshotSSTableReader(parentSessionId.toString()))
              {
 -                List<SSTableReader> sstables = columnFamilyStores.get(cfId).select(ColumnFamilyStore.UNREPAIRED_SSTABLES).sstables;
 -                Set<SSTableReader> currentlyRepairing = ActiveRepairService.instance.currentlyRepairing(cfId, parentSessionId);
 -                if (!Sets.intersection(currentlyRepairing, Sets.newHashSet(sstables)).isEmpty())
 +                for (SSTableReader sstable : snapshottedSSTables)
 +                {
 +                    snapshotGenerations.add(sstable.descriptor.generation);
 +                }
 +            }
 +            catch (IOException e)
 +            {
 +                throw new RuntimeException(e);
 +            }
 +            for (SSTableReader sstable : cfs.select(ColumnFamilyStore.CANONICAL_SSTABLES).sstables)
 +                if (snapshotGenerations.contains(sstable.descriptor.generation))
 +                    activeSSTables.add(sstable);
 +            return activeSSTables;
 +        }
 +
 +        public synchronized void maybeSnapshot(UUID cfId, UUID parentSessionId)
 +        {
 +            String snapshotName = parentSessionId.toString();
 +            if (!columnFamilyStores.get(cfId).snapshotExists(snapshotName))
 +            {
 +                Set<SSTableReader> snapshottedSSTables = columnFamilyStores.get(cfId).snapshot(snapshotName, new Predicate<SSTableReader>()
                  {
 +                    public boolean apply(SSTableReader sstable)
 +                    {
 +                        return sstable != null &&
 +                               (!isIncremental || !sstable.isRepaired()) &&
 +                               !(sstable.partitioner instanceof LocalPartitioner) && // exclude SSTables from 2i
 +                               new Bounds<>(sstable.first.getToken(), sstable.last.getToken()).intersects(ranges);
 +                    }
 +                }, true);
 +
 +                if (isAlreadyRepairing(cfId, parentSessionId, snapshottedSSTables))
 +                {
 +                    columnFamilyStores.get(cfId).clearSnapshot(parentSessionId.toString());
                      logger.error("Cannot start multiple repair sessions over the same sstables");
                      throw new RuntimeException("Cannot start multiple repair sessions over the same sstables");
                  }
@@@ -618,32 -566,13 +625,35 @@@
              }
          }
  
 +
 +        /**
 +         * Compares other repairing sstables *generation* to the ones we just snapshotted
 +         *
 +         * we compare generations since the sstables have different paths due to snapshot names
 +         *
 +         * @param cfId id of the column family store
 +         * @param parentSessionId parent repair session
 +         * @param sstables the newly snapshotted sstables
 +         * @return
 +         */
 +        private boolean isAlreadyRepairing(UUID cfId, UUID parentSessionId, Collection<SSTableReader> sstables)
 +        {
 +            Set<SSTableReader> currentlyRepairing = ActiveRepairService.instance.currentlyRepairing(cfId, parentSessionId);
 +            Set<Integer> currentlyRepairingGenerations = new HashSet<>();
 +            Set<Integer> newRepairingGenerations = new HashSet<>();
 +            for (SSTableReader sstable : currentlyRepairing)
 +                currentlyRepairingGenerations.add(sstable.descriptor.generation);
 +            for (SSTableReader sstable : sstables)
 +                newRepairingGenerations.add(sstable.descriptor.generation);
 +
 +            return !Sets.intersection(currentlyRepairingGenerations, newRepairingGenerations).isEmpty();
 +        }
 +
          private Set<SSTableReader> getActiveSSTables(UUID cfId)
          {
 -            if (failed)
 -                return Collections.emptySet();
+             if (!columnFamilyStores.containsKey(cfId))
+                 return null;
+ 
              Set<String> repairedSSTables = sstableMap.get(cfId);
              Set<SSTableReader> activeSSTables = new HashSet<>();
              Set<String> activeSSTableNames = new HashSet<>();


[03/12] cassandra git commit: Don't try to get sstables for non-repairing column families

Posted by ma...@apache.org.
Don't try to get sstables for non-repairing column families

Patch by marcuse; reviewed by Paulo Motta for CASSANDRA-12077


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/9358e589
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/9358e589
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/9358e589

Branch: refs/heads/cassandra-2.2
Commit: 9358e589e292a2c851d069aebd36819f8e767f5b
Parents: 3c8421a
Author: Marcus Eriksson <ma...@apache.org>
Authored: Thu Jun 23 09:46:00 2016 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Thu Jun 23 14:49:59 2016 +0200

----------------------------------------------------------------------
 CHANGES.txt                                                  | 1 +
 .../org/apache/cassandra/service/ActiveRepairService.java    | 8 +++++++-
 2 files changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/9358e589/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 03246ae..620568d 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 2.1.15
+ * Don't try to get sstables for non-repairing column families (CASSANDRA-12077)
  * Prevent select statements with clustering key > 64k (CASSANDRA-11882)
  * Avoid marking too many sstables as repaired (CASSANDRA-11696)
  * Fix clock skew corrupting other nodes with paxos (CASSANDRA-11991)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/9358e589/src/java/org/apache/cassandra/service/ActiveRepairService.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/ActiveRepairService.java b/src/java/org/apache/cassandra/service/ActiveRepairService.java
index bab244d..4ca1e42 100644
--- a/src/java/org/apache/cassandra/service/ActiveRepairService.java
+++ b/src/java/org/apache/cassandra/service/ActiveRepairService.java
@@ -528,7 +528,10 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
         {
             assert marked.contains(cfId);
             ImmutableMap.Builder<SSTableReader, Ref<SSTableReader>> references = ImmutableMap.builder();
-            for (SSTableReader sstable : getActiveSSTables(cfId))
+            Iterable<SSTableReader> sstables = getActiveSSTables(cfId);
+            if (sstables == null)
+                throw new RuntimeException("Not possible to get sstables for anticompaction for " + cfId);
+            for (SSTableReader sstable : sstables)
             {
                 Ref<SSTableReader> ref = sstable.tryRef();
                 if (ref == null)
@@ -567,6 +570,9 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
         {
             if (failed)
                 return Collections.emptySet();
+            if (!columnFamilyStores.containsKey(cfId))
+                return null;
+
             Set<String> repairedSSTables = sstableMap.get(cfId);
             Set<SSTableReader> activeSSTables = new HashSet<>();
             Set<String> activeSSTableNames = new HashSet<>();


[02/12] cassandra git commit: Disable autocompaction during drain

Posted by ma...@apache.org.
Disable autocompaction during drain

Patch by marcuse; reviewed by Alex Petrov for CASSANDRA-11878


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/5af5bafc
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/5af5bafc
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/5af5bafc

Branch: refs/heads/trunk
Commit: 5af5bafc87f3efe0388c003b3923ca3452e1e261
Parents: 5ee6e7b
Author: Marcus Eriksson <ma...@apache.org>
Authored: Thu May 26 13:27:08 2016 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Thu Jun 23 13:47:32 2016 +0200

----------------------------------------------------------------------
 CHANGES.txt                                               | 1 +
 src/java/org/apache/cassandra/service/StorageService.java | 6 ++++++
 2 files changed, 7 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/5af5bafc/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index f7e854d..7db3afb 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 3.0.8
+ * Disable autocompaction during drain (CASSANDRA-11878)
  * Add a metrics timer to MemtablePool and use it to track time spent blocked on memory in MemtableAllocator (CASSANDRA-11327)
  * Fix upgrading schema with super columns with non-text subcomparators (CASSANDRA-12023)
  * Add TimeWindowCompactionStrategy (CASSANDRA-9666)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5af5bafc/src/java/org/apache/cassandra/service/StorageService.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java
index 394220d..5e14d4f 100644
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@ -3978,6 +3978,12 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         StorageProxy.instance.verifyNoHintsInProgress();
 
         setMode(Mode.DRAINING, "flushing column families", false);
+
+        // disable autocompaction - we don't want to start any new compactions while we are draining
+        for (Keyspace keyspace : Keyspace.all())
+            for (ColumnFamilyStore cfs : keyspace.getColumnFamilyStores())
+                cfs.disableAutoCompaction();
+
         // count CFs first, since forceFlush could block for the flushWriter to get a queue slot empty
         totalCFs = 0;
         for (Keyspace keyspace : Keyspace.nonSystem())


[11/12] cassandra git commit: Merge branch 'cassandra-2.2' into cassandra-3.0

Posted by ma...@apache.org.
Merge branch 'cassandra-2.2' into cassandra-3.0


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/ffd6bab3
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/ffd6bab3
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/ffd6bab3

Branch: refs/heads/cassandra-3.0
Commit: ffd6bab35290f95402a3b6b41c7af6f1c7d26286
Parents: 5af5baf 5d0d30e
Author: Marcus Eriksson <ma...@apache.org>
Authored: Thu Jun 23 15:13:54 2016 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Thu Jun 23 15:13:54 2016 +0200

----------------------------------------------------------------------
 CHANGES.txt                                             |  1 +
 .../apache/cassandra/service/ActiveRepairService.java   | 12 +++++++++++-
 2 files changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/ffd6bab3/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 7db3afb,8fbdf3b..0e9935d
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -7,36 -2,9 +7,37 @@@ Merged from 2.2
   * Allow nodetool info to run with readonly JMX access (CASSANDRA-11755)
   * Validate bloom_filter_fp_chance against lowest supported
     value when the table is created (CASSANDRA-11920)
 - * RandomAccessReader: call isEOF() only when rebuffering, not for every read operation (CASSANDRA-12013)
   * Don't send erroneous NEW_NODE notifications on restart (CASSANDRA-11038)
   * StorageService shutdown hook should use a volatile variable (CASSANDRA-11984)
 +Merged from 2.1:
++ * Don't try to get sstables for non-repairing column families (CASSANDRA-12077)
 + * Avoid marking too many sstables as repaired (CASSANDRA-11696)
 + * Prevent select statements with clustering key > 64k (CASSANDRA-11882)
 + * Fix clock skew corrupting other nodes with paxos (CASSANDRA-11991)
 + * Remove distinction between non-existing static columns and existing but null in LWTs (CASSANDRA-9842)
 + * Cache local ranges when calculating repair neighbors (CASSANDRA-11934)
 + * Allow LWT operation on static column with only partition keys (CASSANDRA-10532)
 + * Create interval tree over canonical sstables to avoid missing sstables during streaming (CASSANDRA-11886)
 + * cqlsh COPY FROM: shutdown parent cluster after forking, to avoid corrupting SSL connections (CASSANDRA-11749)
 +
 +
 +3.0.7
 + * Fix legacy serialization of Thrift-generated non-compound range tombstones
 +   when communicating with 2.x nodes (CASSANDRA-11930)
 + * Fix Directories instantiations where CFS.initialDirectories should be used (CASSANDRA-11849)
 + * Avoid referencing DatabaseDescriptor in AbstractType (CASSANDRA-11912)
 + * Fix sstables not being protected from removal during index build (CASSANDRA-11905)
 + * cqlsh: Suppress stack trace from Read/WriteFailures (CASSANDRA-11032)
 + * Remove unneeded code to repair index summaries that have
 +   been improperly down-sampled (CASSANDRA-11127)
 + * Avoid WriteTimeoutExceptions during commit log replay due to materialized
 +   view lock contention (CASSANDRA-11891)
 + * Prevent OOM failures on SSTable corruption, improve tests for corruption detection (CASSANDRA-9530)
 + * Use CFS.initialDirectories when clearing snapshots (CASSANDRA-11705)
 + * Allow compaction strategies to disable early open (CASSANDRA-11754)
 + * Refactor Materialized View code (CASSANDRA-11475)
 + * Update Java Driver (CASSANDRA-11615)
 +Merged from 2.2:
   * Persist local metadata earlier in startup sequence (CASSANDRA-11742)
   * Run CommitLog tests with different compression settings (CASSANDRA-9039)
   * cqlsh: fix tab completion for case-sensitive identifiers (CASSANDRA-11664)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/ffd6bab3/src/java/org/apache/cassandra/service/ActiveRepairService.java
----------------------------------------------------------------------


[06/12] cassandra git commit: Don't try to get sstables for non-repairing column families

Posted by ma...@apache.org.
Don't try to get sstables for non-repairing column families

Patch by marcuse; reviewed by Paulo Motta for CASSANDRA-12077


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/9358e589
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/9358e589
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/9358e589

Branch: refs/heads/trunk
Commit: 9358e589e292a2c851d069aebd36819f8e767f5b
Parents: 3c8421a
Author: Marcus Eriksson <ma...@apache.org>
Authored: Thu Jun 23 09:46:00 2016 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Thu Jun 23 14:49:59 2016 +0200

----------------------------------------------------------------------
 CHANGES.txt                                                  | 1 +
 .../org/apache/cassandra/service/ActiveRepairService.java    | 8 +++++++-
 2 files changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/9358e589/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 03246ae..620568d 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 2.1.15
+ * Don't try to get sstables for non-repairing column families (CASSANDRA-12077)
  * Prevent select statements with clustering key > 64k (CASSANDRA-11882)
  * Avoid marking too many sstables as repaired (CASSANDRA-11696)
  * Fix clock skew corrupting other nodes with paxos (CASSANDRA-11991)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/9358e589/src/java/org/apache/cassandra/service/ActiveRepairService.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/ActiveRepairService.java b/src/java/org/apache/cassandra/service/ActiveRepairService.java
index bab244d..4ca1e42 100644
--- a/src/java/org/apache/cassandra/service/ActiveRepairService.java
+++ b/src/java/org/apache/cassandra/service/ActiveRepairService.java
@@ -528,7 +528,10 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
         {
             assert marked.contains(cfId);
             ImmutableMap.Builder<SSTableReader, Ref<SSTableReader>> references = ImmutableMap.builder();
-            for (SSTableReader sstable : getActiveSSTables(cfId))
+            Iterable<SSTableReader> sstables = getActiveSSTables(cfId);
+            if (sstables == null)
+                throw new RuntimeException("Not possible to get sstables for anticompaction for " + cfId);
+            for (SSTableReader sstable : sstables)
             {
                 Ref<SSTableReader> ref = sstable.tryRef();
                 if (ref == null)
@@ -567,6 +570,9 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
         {
             if (failed)
                 return Collections.emptySet();
+            if (!columnFamilyStores.containsKey(cfId))
+                return null;
+
             Set<String> repairedSSTables = sstableMap.get(cfId);
             Set<SSTableReader> activeSSTables = new HashSet<>();
             Set<String> activeSSTableNames = new HashSet<>();


[07/12] cassandra git commit: Merge branch 'cassandra-2.1' into cassandra-2.2

Posted by ma...@apache.org.
Merge branch 'cassandra-2.1' into cassandra-2.2


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/5d0d30e4
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/5d0d30e4
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/5d0d30e4

Branch: refs/heads/cassandra-3.0
Commit: 5d0d30e4d3d587a155d42eefe23f8586cdf06d94
Parents: 225232a 9358e58
Author: Marcus Eriksson <ma...@apache.org>
Authored: Thu Jun 23 15:10:35 2016 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Thu Jun 23 15:10:35 2016 +0200

----------------------------------------------------------------------
 CHANGES.txt                                             |  1 +
 .../apache/cassandra/service/ActiveRepairService.java   | 12 +++++++++++-
 2 files changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/5d0d30e4/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index b366d21,620568d..8fbdf3b
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,35 -1,5 +1,36 @@@
 -2.1.15
 +2.2.7
 + * Allow nodetool info to run with readonly JMX access (CASSANDRA-11755)
 + * Validate bloom_filter_fp_chance against lowest supported
 +   value when the table is created (CASSANDRA-11920)
 + * RandomAccessReader: call isEOF() only when rebuffering, not for every read operation (CASSANDRA-12013)
 + * Don't send erroneous NEW_NODE notifications on restart (CASSANDRA-11038)
 + * StorageService shutdown hook should use a volatile variable (CASSANDRA-11984)
 + * Persist local metadata earlier in startup sequence (CASSANDRA-11742)
 + * Run CommitLog tests with different compression settings (CASSANDRA-9039)
 + * cqlsh: fix tab completion for case-sensitive identifiers (CASSANDRA-11664)
 + * Avoid showing estimated key as -1 in tablestats (CASSANDRA-11587)
 + * Fix possible race condition in CommitLog.recover (CASSANDRA-11743)
 + * Enable client encryption in sstableloader with cli options (CASSANDRA-11708)
 + * Possible memory leak in NIODataInputStream (CASSANDRA-11867)
 + * Fix commit log replay after out-of-order flush completion (CASSANDRA-9669)
 + * Add seconds to cqlsh tracing session duration (CASSANDRA-11753)
 + * Prohibit Reverse Counter type as part of the PK (CASSANDRA-9395)
 + * cqlsh: correctly handle non-ascii chars in error messages (CASSANDRA-11626)
 + * Exit JVM if JMX server fails to startup (CASSANDRA-11540)
 + * Produce a heap dump when exiting on OOM (CASSANDRA-9861)
 + * Avoid read repairing purgeable tombstones on range slices (CASSANDRA-11427)
 + * Restore ability to filter on clustering columns when using a 2i (CASSANDRA-11510)
 + * JSON datetime formatting needs timezone (CASSANDRA-11137)
 + * Fix is_dense recalculation for Thrift-updated tables (CASSANDRA-11502)
 + * Remove unnescessary file existence check during anticompaction (CASSANDRA-11660)
 + * Add missing files to debian packages (CASSANDRA-11642)
 + * Avoid calling Iterables::concat in loops during ModificationStatement::getFunctions (CASSANDRA-11621)
 + * cqlsh: COPY FROM should use regular inserts for single statement batches and
 +   report errors correctly if workers processes crash on initialization (CASSANDRA-11474)
 + * Always close cluster with connection in CqlRecordWriter (CASSANDRA-11553)
 + * Fix slice queries on ordered COMPACT tables (CASSANDRA-10988)
 +Merged from 2.1:
+  * Don't try to get sstables for non-repairing column families (CASSANDRA-12077)
   * Prevent select statements with clustering key > 64k (CASSANDRA-11882)
   * Avoid marking too many sstables as repaired (CASSANDRA-11696)
   * Fix clock skew corrupting other nodes with paxos (CASSANDRA-11991)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5d0d30e4/src/java/org/apache/cassandra/service/ActiveRepairService.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/service/ActiveRepairService.java
index e111155,4ca1e42..38804b3
--- a/src/java/org/apache/cassandra/service/ActiveRepairService.java
+++ b/src/java/org/apache/cassandra/service/ActiveRepairService.java
@@@ -542,12 -524,14 +542,17 @@@ public class ActiveRepairService implem
           * @return
           */
          @SuppressWarnings("resource")
 -        public synchronized Refs<SSTableReader> getActiveRepairedSSTableRefsForAntiCompaction(UUID cfId)
 +        public synchronized Refs<SSTableReader> getActiveRepairedSSTableRefsForAntiCompaction(UUID cfId, UUID parentSessionId)
          {
              assert marked.contains(cfId);
 -            ImmutableMap.Builder<SSTableReader, Ref<SSTableReader>> references = ImmutableMap.builder();
 -            Iterable<SSTableReader> sstables = getActiveSSTables(cfId);
 -            if (sstables == null)
++            if (!columnFamilyStores.containsKey(cfId))
+                 throw new RuntimeException("Not possible to get sstables for anticompaction for " + cfId);
 +            boolean isSnapshotRepair = columnFamilyStores.get(cfId).snapshotExists(parentSessionId.toString());
 +            ImmutableMap.Builder<SSTableReader, Ref<SSTableReader>> references = ImmutableMap.builder();
-             for (SSTableReader sstable : isSnapshotRepair ? getSSTablesForSnapshotRepair(cfId, parentSessionId) : getActiveSSTables(cfId))
++            Iterable<SSTableReader> sstables = isSnapshotRepair ? getSSTablesForSnapshotRepair(cfId, parentSessionId) : getActiveSSTables(cfId);
++            // we check this above - if columnFamilyStores contains the cfId sstables will not be null
++            assert sstables != null;
+             for (SSTableReader sstable : sstables)
              {
                  Ref<SSTableReader> ref = sstable.tryRef();
                  if (ref == null)
@@@ -559,57 -543,21 +564,59 @@@
          }
  
          /**
 -         * Marks all the unrepaired sstables as repairing unless we have already done so.
 +         * If we are running a snapshot repair we need to find the 'real' sstables when we start anticompaction
           *
 -         * Any of these sstables that are still on disk are then anticompacted once the streaming and validation phases are done.
 +         * We use the generation of the sstables as identifiers instead of the file name to avoid having to parse out the
 +         * actual filename.
           *
           * @param cfId
 -         * @param parentSessionId used to check that we don't start multiple inc repair sessions over the same sstables
 +         * @param parentSessionId
 +         * @return
           */
 -        public synchronized void markSSTablesRepairing(UUID cfId, UUID parentSessionId)
 +        private Set<SSTableReader> getSSTablesForSnapshotRepair(UUID cfId, UUID parentSessionId)
          {
 -            if (!marked.contains(cfId))
 +            Set<SSTableReader> activeSSTables = new HashSet<>();
 +            ColumnFamilyStore cfs = columnFamilyStores.get(cfId);
++            if (cfs == null)
++                return null;
 +
 +            Set<Integer> snapshotGenerations = new HashSet<>();
 +            try (Refs<SSTableReader> snapshottedSSTables = cfs.getSnapshotSSTableReader(parentSessionId.toString()))
              {
 -                List<SSTableReader> sstables = columnFamilyStores.get(cfId).select(ColumnFamilyStore.UNREPAIRED_SSTABLES).sstables;
 -                Set<SSTableReader> currentlyRepairing = ActiveRepairService.instance.currentlyRepairing(cfId, parentSessionId);
 -                if (!Sets.intersection(currentlyRepairing, Sets.newHashSet(sstables)).isEmpty())
 +                for (SSTableReader sstable : snapshottedSSTables)
 +                {
 +                    snapshotGenerations.add(sstable.descriptor.generation);
 +                }
 +            }
 +            catch (IOException e)
 +            {
 +                throw new RuntimeException(e);
 +            }
 +            for (SSTableReader sstable : cfs.select(ColumnFamilyStore.CANONICAL_SSTABLES).sstables)
 +                if (snapshotGenerations.contains(sstable.descriptor.generation))
 +                    activeSSTables.add(sstable);
 +            return activeSSTables;
 +        }
 +
 +        public synchronized void maybeSnapshot(UUID cfId, UUID parentSessionId)
 +        {
 +            String snapshotName = parentSessionId.toString();
 +            if (!columnFamilyStores.get(cfId).snapshotExists(snapshotName))
 +            {
 +                Set<SSTableReader> snapshottedSSTables = columnFamilyStores.get(cfId).snapshot(snapshotName, new Predicate<SSTableReader>()
                  {
 +                    public boolean apply(SSTableReader sstable)
 +                    {
 +                        return sstable != null &&
 +                               (!isIncremental || !sstable.isRepaired()) &&
 +                               !(sstable.partitioner instanceof LocalPartitioner) && // exclude SSTables from 2i
 +                               new Bounds<>(sstable.first.getToken(), sstable.last.getToken()).intersects(ranges);
 +                    }
 +                }, true);
 +
 +                if (isAlreadyRepairing(cfId, parentSessionId, snapshottedSSTables))
 +                {
 +                    columnFamilyStores.get(cfId).clearSnapshot(parentSessionId.toString());
                      logger.error("Cannot start multiple repair sessions over the same sstables");
                      throw new RuntimeException("Cannot start multiple repair sessions over the same sstables");
                  }
@@@ -618,32 -566,13 +625,35 @@@
              }
          }
  
 +
 +        /**
 +         * Compares other repairing sstables *generation* to the ones we just snapshotted
 +         *
 +         * we compare generations since the sstables have different paths due to snapshot names
 +         *
 +         * @param cfId id of the column family store
 +         * @param parentSessionId parent repair session
 +         * @param sstables the newly snapshotted sstables
 +         * @return
 +         */
 +        private boolean isAlreadyRepairing(UUID cfId, UUID parentSessionId, Collection<SSTableReader> sstables)
 +        {
 +            Set<SSTableReader> currentlyRepairing = ActiveRepairService.instance.currentlyRepairing(cfId, parentSessionId);
 +            Set<Integer> currentlyRepairingGenerations = new HashSet<>();
 +            Set<Integer> newRepairingGenerations = new HashSet<>();
 +            for (SSTableReader sstable : currentlyRepairing)
 +                currentlyRepairingGenerations.add(sstable.descriptor.generation);
 +            for (SSTableReader sstable : sstables)
 +                newRepairingGenerations.add(sstable.descriptor.generation);
 +
 +            return !Sets.intersection(currentlyRepairingGenerations, newRepairingGenerations).isEmpty();
 +        }
 +
          private Set<SSTableReader> getActiveSSTables(UUID cfId)
          {
 -            if (failed)
 -                return Collections.emptySet();
+             if (!columnFamilyStores.containsKey(cfId))
+                 return null;
+ 
              Set<String> repairedSSTables = sstableMap.get(cfId);
              Set<SSTableReader> activeSSTables = new HashSet<>();
              Set<String> activeSSTableNames = new HashSet<>();


[05/12] cassandra git commit: Don't try to get sstables for non-repairing column families

Posted by ma...@apache.org.
Don't try to get sstables for non-repairing column families

Patch by marcuse; reviewed by Paulo Motta for CASSANDRA-12077


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/9358e589
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/9358e589
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/9358e589

Branch: refs/heads/cassandra-2.1
Commit: 9358e589e292a2c851d069aebd36819f8e767f5b
Parents: 3c8421a
Author: Marcus Eriksson <ma...@apache.org>
Authored: Thu Jun 23 09:46:00 2016 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Thu Jun 23 14:49:59 2016 +0200

----------------------------------------------------------------------
 CHANGES.txt                                                  | 1 +
 .../org/apache/cassandra/service/ActiveRepairService.java    | 8 +++++++-
 2 files changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/9358e589/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 03246ae..620568d 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 2.1.15
+ * Don't try to get sstables for non-repairing column families (CASSANDRA-12077)
  * Prevent select statements with clustering key > 64k (CASSANDRA-11882)
  * Avoid marking too many sstables as repaired (CASSANDRA-11696)
  * Fix clock skew corrupting other nodes with paxos (CASSANDRA-11991)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/9358e589/src/java/org/apache/cassandra/service/ActiveRepairService.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/ActiveRepairService.java b/src/java/org/apache/cassandra/service/ActiveRepairService.java
index bab244d..4ca1e42 100644
--- a/src/java/org/apache/cassandra/service/ActiveRepairService.java
+++ b/src/java/org/apache/cassandra/service/ActiveRepairService.java
@@ -528,7 +528,10 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
         {
             assert marked.contains(cfId);
             ImmutableMap.Builder<SSTableReader, Ref<SSTableReader>> references = ImmutableMap.builder();
-            for (SSTableReader sstable : getActiveSSTables(cfId))
+            Iterable<SSTableReader> sstables = getActiveSSTables(cfId);
+            if (sstables == null)
+                throw new RuntimeException("Not possible to get sstables for anticompaction for " + cfId);
+            for (SSTableReader sstable : sstables)
             {
                 Ref<SSTableReader> ref = sstable.tryRef();
                 if (ref == null)
@@ -567,6 +570,9 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
         {
             if (failed)
                 return Collections.emptySet();
+            if (!columnFamilyStores.containsKey(cfId))
+                return null;
+
             Set<String> repairedSSTables = sstableMap.get(cfId);
             Set<SSTableReader> activeSSTables = new HashSet<>();
             Set<String> activeSSTableNames = new HashSet<>();


[04/12] cassandra git commit: Don't try to get sstables for non-repairing column families

Posted by ma...@apache.org.
Don't try to get sstables for non-repairing column families

Patch by marcuse; reviewed by Paulo Motta for CASSANDRA-12077


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/9358e589
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/9358e589
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/9358e589

Branch: refs/heads/cassandra-3.0
Commit: 9358e589e292a2c851d069aebd36819f8e767f5b
Parents: 3c8421a
Author: Marcus Eriksson <ma...@apache.org>
Authored: Thu Jun 23 09:46:00 2016 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Thu Jun 23 14:49:59 2016 +0200

----------------------------------------------------------------------
 CHANGES.txt                                                  | 1 +
 .../org/apache/cassandra/service/ActiveRepairService.java    | 8 +++++++-
 2 files changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/9358e589/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 03246ae..620568d 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 2.1.15
+ * Don't try to get sstables for non-repairing column families (CASSANDRA-12077)
  * Prevent select statements with clustering key > 64k (CASSANDRA-11882)
  * Avoid marking too many sstables as repaired (CASSANDRA-11696)
  * Fix clock skew corrupting other nodes with paxos (CASSANDRA-11991)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/9358e589/src/java/org/apache/cassandra/service/ActiveRepairService.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/ActiveRepairService.java b/src/java/org/apache/cassandra/service/ActiveRepairService.java
index bab244d..4ca1e42 100644
--- a/src/java/org/apache/cassandra/service/ActiveRepairService.java
+++ b/src/java/org/apache/cassandra/service/ActiveRepairService.java
@@ -528,7 +528,10 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
         {
             assert marked.contains(cfId);
             ImmutableMap.Builder<SSTableReader, Ref<SSTableReader>> references = ImmutableMap.builder();
-            for (SSTableReader sstable : getActiveSSTables(cfId))
+            Iterable<SSTableReader> sstables = getActiveSSTables(cfId);
+            if (sstables == null)
+                throw new RuntimeException("Not possible to get sstables for anticompaction for " + cfId);
+            for (SSTableReader sstable : sstables)
             {
                 Ref<SSTableReader> ref = sstable.tryRef();
                 if (ref == null)
@@ -567,6 +570,9 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
         {
             if (failed)
                 return Collections.emptySet();
+            if (!columnFamilyStores.containsKey(cfId))
+                return null;
+
             Set<String> repairedSSTables = sstableMap.get(cfId);
             Set<SSTableReader> activeSSTables = new HashSet<>();
             Set<String> activeSSTableNames = new HashSet<>();