You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ma...@apache.org on 2015/04/01 13:01:32 UTC

[1/3] cassandra git commit: More aggressive check for expired sstables in DTCS

Repository: cassandra
Updated Branches:
  refs/heads/trunk 7f0a8542d -> b09e39295


More aggressive check for expired sstables in DTCS

Patch by Björn Hegerfors; reviewed by marcuse for CASSANDRA-8359


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/3d0c4e78
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/3d0c4e78
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/3d0c4e78

Branch: refs/heads/trunk
Commit: 3d0c4e78c6bb3f8767aa0720b7de579908e2bf59
Parents: 74bfa77
Author: Björn Hegerfors <bj...@spotify.com>
Authored: Wed Apr 1 12:26:00 2015 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Wed Apr 1 12:32:09 2015 +0200

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../DateTieredCompactionStrategy.java           | 31 ++++++++++----
 .../DateTieredCompactionStrategyTest.java       | 43 ++++++++++++++++++++
 3 files changed, 67 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/3d0c4e78/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index a956eb6..1afe6fe 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 2.0.14:
+ * More aggressive check for expired sstables in DTCS (CASSANDRA-8359)
  * Don't set clientMode to true when bulk-loading sstables to avoid
    a NullPointerException (CASSANDRA-8238)
  * Fix ignored index_interval change in ALTER TABLE statements (CASSANDRA-7976)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/3d0c4e78/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
index 6b3e800..cfa9c8a 100644
--- a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
@@ -53,7 +53,7 @@ public class DateTieredCompactionStrategy extends AbstractCompactionStrategy
 
         while (true)
         {
-            List<SSTableReader> latestBucket = getNextBackgroundSStables(gcBefore);
+            List<SSTableReader> latestBucket = getNextBackgroundSSTables(gcBefore);
 
             if (latestBucket.isEmpty())
                 return null;
@@ -68,24 +68,39 @@ public class DateTieredCompactionStrategy extends AbstractCompactionStrategy
      * @param gcBefore
      * @return
      */
-    private List<SSTableReader> getNextBackgroundSStables(final int gcBefore)
+    private List<SSTableReader> getNextBackgroundSSTables(final int gcBefore)
     {
         if (!isEnabled() || cfs.getSSTables().isEmpty())
             return Collections.emptyList();
 
+        Set<SSTableReader> uncompacting = cfs.getUncompactingSSTables();
+
+        // Find fully expired SSTables. Those will be included no matter what.
+        Set<SSTableReader> expired = CompactionController.getFullyExpiredSSTables(cfs, uncompacting, cfs.getOverlappingSSTables(uncompacting), gcBefore);
+        Set<SSTableReader> candidates = Sets.newHashSet(filterSuspectSSTables(uncompacting));
+
+        List<SSTableReader> compactionCandidates = new ArrayList<>(getNextNonExpiredSSTables(Sets.difference(candidates, expired), gcBefore));
+        if (!expired.isEmpty())
+        {
+            logger.debug("Including expired sstables: {}", expired);
+            compactionCandidates.addAll(expired);
+        }
+        return compactionCandidates;
+    }
+
+    private List<SSTableReader> getNextNonExpiredSSTables(Iterable<SSTableReader> nonExpiringSSTables, final int gcBefore)
+    {
         int base = cfs.getMinimumCompactionThreshold();
         long now = getNow();
 
-        Iterable<SSTableReader> candidates = filterSuspectSSTables(cfs.getUncompactingSSTables());
-
-        List<SSTableReader> mostInteresting = getCompactionCandidates(candidates, now, base);
+        List<SSTableReader> mostInteresting = getCompactionCandidates(nonExpiringSSTables, now, base);
         if (mostInteresting != null)
             return mostInteresting;
 
         // if there is no sstable to compact in standard way, try compacting single sstable whose droppable tombstone
         // ratio is greater than threshold.
         List<SSTableReader> sstablesWithTombstones = Lists.newArrayList();
-        for (SSTableReader sstable : candidates)
+        for (SSTableReader sstable : nonExpiringSSTables)
         {
             if (worthDroppingTombstones(sstable, gcBefore))
                 sstablesWithTombstones.add(sstable);
@@ -106,8 +121,8 @@ public class DateTieredCompactionStrategy extends AbstractCompactionStrategy
         List<SSTableReader> mostInteresting = newestBucket(buckets,
                                                            cfs.getMinimumCompactionThreshold(),
                                                            cfs.getMaximumCompactionThreshold(),
-                                                           options.baseTime,
-                                                           now);
+                                                           now,
+                                                           options.baseTime);
         if (!mostInteresting.isEmpty())
             return mostInteresting;
         return null;

http://git-wip-us.apache.org/repos/asf/cassandra/blob/3d0c4e78/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
index f98e372..1fa41a3 100644
--- a/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
@@ -273,4 +273,47 @@ public class DateTieredCompactionStrategyTest extends SchemaLoader
         filtered = filterOldSSTables(sstrs, 1, 4);
         assertEquals("no sstables should remain when all are too old", 0, Iterables.size(filtered));
     }
+
+
+    @Test
+    public void testDropExpiredSSTables() throws InterruptedException
+    {
+        Keyspace keyspace = Keyspace.open(KEYSPACE1);
+        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD1);
+        cfs.truncateBlocking();
+        cfs.disableAutoCompaction();
+
+        ByteBuffer value = ByteBuffer.wrap(new byte[100]);
+
+        // create 2 sstables
+        DecoratedKey key = Util.dk(String.valueOf("expired"));
+        RowMutation rm = new RowMutation(KEYSPACE1, key.key);
+        rm.add(CF_STANDARD1, ByteBufferUtil.bytes("column"), value, System.currentTimeMillis(), 5);
+        rm.apply();
+        cfs.forceBlockingFlush();
+        SSTableReader expiredSSTable = cfs.getSSTables().iterator().next();
+        Thread.sleep(10);
+        key = Util.dk(String.valueOf("nonexpired"));
+        rm = new RowMutation(KEYSPACE1, key.key);
+        rm.add(CF_STANDARD1, ByteBufferUtil.bytes("column"), value, System.currentTimeMillis());
+        rm.apply();
+        cfs.forceBlockingFlush();
+        assertEquals(cfs.getSSTables().size(), 2);
+
+        Map<String, String> options = new HashMap<>();
+
+        options.put(DateTieredCompactionStrategyOptions.BASE_TIME_KEY, "30");
+        options.put(DateTieredCompactionStrategyOptions.TIMESTAMP_RESOLUTION_KEY, "MILLISECONDS");
+        options.put(DateTieredCompactionStrategyOptions.MAX_SSTABLE_AGE_KEY, Double.toString((1d / (24 * 60 * 60))));
+        DateTieredCompactionStrategy dtcs = new DateTieredCompactionStrategy(cfs, options);
+        dtcs.startup();
+        assertNull(dtcs.getNextBackgroundTask((int) (System.currentTimeMillis() / 1000)));
+        Thread.sleep(7000);
+        AbstractCompactionTask t = dtcs.getNextBackgroundTask((int) (System.currentTimeMillis()/1000));
+        assertNotNull(t);
+        assertEquals(1, Iterables.size(t.sstables));
+        SSTableReader sstable = t.sstables.iterator().next();
+        assertEquals(sstable, expiredSSTable);
+    }
+
 }


[2/3] cassandra git commit: Merge branch 'cassandra-2.0' into cassandra-2.1

Posted by ma...@apache.org.
Merge branch 'cassandra-2.0' into cassandra-2.1

Conflicts:
	CHANGES.txt
	src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/52191ebe
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/52191ebe
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/52191ebe

Branch: refs/heads/trunk
Commit: 52191ebe33d82bf89a57f019c026c88a8d82ee55
Parents: c9b83e0 3d0c4e7
Author: Marcus Eriksson <ma...@apache.org>
Authored: Wed Apr 1 12:34:01 2015 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Wed Apr 1 12:56:21 2015 +0200

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../DateTieredCompactionStrategy.java           | 31 ++++++++++----
 .../DateTieredCompactionStrategyTest.java       | 43 ++++++++++++++++++++
 3 files changed, 67 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/52191ebe/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 7ded07b,1afe6fe..ab5fb2d
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,69 -1,7 +1,70 @@@
 -2.0.14:
 +2.1.4
 + * Buffer bloom filter serialization (CASSANDRA-9066)
 + * Fix anti-compaction target bloom filter size (CASSANDRA-9060)
 + * Make FROZEN and TUPLE unreserved keywords in CQL (CASSANDRA-9047)
 + * Prevent AssertionError from SizeEstimatesRecorder (CASSANDRA-9034)
 + * Avoid overwriting index summaries for sstables with an older format that
 +   does not support downsampling; rebuild summaries on startup when this
 +   is detected (CASSANDRA-8993)
 + * Fix potential data loss in CompressedSequentialWriter (CASSANDRA-8949)
 + * Make PasswordAuthenticator number of hashing rounds configurable (CASSANDRA-8085)
 + * Fix AssertionError when binding nested collections in DELETE (CASSANDRA-8900)
 + * Check for overlap with non-early sstables in LCS (CASSANDRA-8739)
 + * Only calculate max purgable timestamp if we have to (CASSANDRA-8914)
 + * (cqlsh) Greatly improve performance of COPY FROM (CASSANDRA-8225)
 + * IndexSummary effectiveIndexInterval is now a guideline, not a rule (CASSANDRA-8993)
 + * Use correct bounds for page cache eviction of compressed files (CASSANDRA-8746)
 + * SSTableScanner enforces its bounds (CASSANDRA-8946)
 + * Cleanup cell equality (CASSANDRA-8947)
 + * Introduce intra-cluster message coalescing (CASSANDRA-8692)
 + * DatabaseDescriptor throws NPE when rpc_interface is used (CASSANDRA-8839)
 + * Don't check if an sstable is live for offline compactions (CASSANDRA-8841)
 + * Don't set clientMode in SSTableLoader (CASSANDRA-8238)
 + * Fix SSTableRewriter with disabled early open (CASSANDRA-8535)
 + * Allow invalidating permissions and cache time (CASSANDRA-8722)
 + * Log warning when queries that will require ALLOW FILTERING in Cassandra 3.0
 +   are executed (CASSANDRA-8418)
 + * Fix cassandra-stress so it respects the CL passed in user mode (CASSANDRA-8948)
 + * Fix rare NPE in ColumnDefinition#hasIndexOption() (CASSANDRA-8786)
 + * cassandra-stress reports per-operation statistics, plus misc (CASSANDRA-8769)
 + * Add SimpleDate (cql date) and Time (cql time) types (CASSANDRA-7523)
 + * Use long for key count in cfstats (CASSANDRA-8913)
 + * Make SSTableRewriter.abort() more robust to failure (CASSANDRA-8832)
 + * Remove cold_reads_to_omit from STCS (CASSANDRA-8860)
 + * Make EstimatedHistogram#percentile() use ceil instead of floor (CASSANDRA-8883)
 + * Fix top partitions reporting wrong cardinality (CASSANDRA-8834)
 + * Fix rare NPE in KeyCacheSerializer (CASSANDRA-8067)
 + * Pick sstables for validation as late as possible inc repairs (CASSANDRA-8366)
 + * Fix commitlog getPendingTasks to not increment (CASSANDRA-8856)
 + * Fix parallelism adjustment in range and secondary index queries
 +   when the first fetch does not satisfy the limit (CASSANDRA-8856)
 + * Check if the filtered sstables is non-empty in STCS (CASSANDRA-8843)
 + * Upgrade java-driver used for cassandra-stress (CASSANDRA-8842)
 + * Fix CommitLog.forceRecycleAllSegments() memory access error (CASSANDRA-8812)
 + * Improve assertions in Memory (CASSANDRA-8792)
 + * Fix SSTableRewriter cleanup (CASSANDRA-8802)
 + * Introduce SafeMemory for CompressionMetadata.Writer (CASSANDRA-8758)
 + * 'nodetool info' prints exception against older node (CASSANDRA-8796)
 + * Ensure SSTableReader.last corresponds exactly with the file end (CASSANDRA-8750)
 + * Make SSTableWriter.openEarly more robust and obvious (CASSANDRA-8747)
 + * Enforce SSTableReader.first/last (CASSANDRA-8744)
 + * Cleanup SegmentedFile API (CASSANDRA-8749)
 + * Avoid overlap with early compaction replacement (CASSANDRA-8683)
 + * Safer Resource Management++ (CASSANDRA-8707)
 + * Write partition size estimates into a system table (CASSANDRA-7688)
 + * cqlsh: Fix keys() and full() collection indexes in DESCRIBE output
 +   (CASSANDRA-8154)
 + * Show progress of streaming in nodetool netstats (CASSANDRA-8886)
 + * IndexSummaryBuilder utilises offheap memory, and shares data between
 +   each IndexSummary opened from it (CASSANDRA-8757)
 + * markCompacting only succeeds if the exact SSTableReader instances being 
 +   marked are in the live set (CASSANDRA-8689)
 + * cassandra-stress support for varint (CASSANDRA-8882)
 + * Fix Adler32 digest for compressed sstables (CASSANDRA-8778)
 + * Add nodetool statushandoff/statusbackup (CASSANDRA-8912)
 + * Use stdout for progress and stats in sstableloader (CASSANDRA-8982)
 +Merged from 2.0:
+  * More aggressive check for expired sstables in DTCS (CASSANDRA-8359)
 - * Don't set clientMode to true when bulk-loading sstables to avoid
 -   a NullPointerException (CASSANDRA-8238)
   * Fix ignored index_interval change in ALTER TABLE statements (CASSANDRA-7976)
   * Do more aggressive compaction in old time windows in DTCS (CASSANDRA-8360)
   * java.lang.AssertionError when reading saved cache (CASSANDRA-8740)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/52191ebe/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
index 2c3bf33,cfa9c8a..7841c93
--- a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
@@@ -74,15 -73,29 +74,30 @@@ public class DateTieredCompactionStrate
          if (!isEnabled() || cfs.getSSTables().isEmpty())
              return Collections.emptyList();
  
+         Set<SSTableReader> uncompacting = cfs.getUncompactingSSTables();
+ 
+         // Find fully expired SSTables. Those will be included no matter what.
+         Set<SSTableReader> expired = CompactionController.getFullyExpiredSSTables(cfs, uncompacting, cfs.getOverlappingSSTables(uncompacting), gcBefore);
+         Set<SSTableReader> candidates = Sets.newHashSet(filterSuspectSSTables(uncompacting));
+ 
+         List<SSTableReader> compactionCandidates = new ArrayList<>(getNextNonExpiredSSTables(Sets.difference(candidates, expired), gcBefore));
+         if (!expired.isEmpty())
+         {
+             logger.debug("Including expired sstables: {}", expired);
+             compactionCandidates.addAll(expired);
+         }
+         return compactionCandidates;
+     }
+ 
+     private List<SSTableReader> getNextNonExpiredSSTables(Iterable<SSTableReader> nonExpiringSSTables, final int gcBefore)
+     {
          int base = cfs.getMinimumCompactionThreshold();
          long now = getNow();
-         Iterable<SSTableReader> candidates = filterSuspectSSTables(Sets.intersection(cfs.getUncompactingSSTables(), sstables));
--
-         List<SSTableReader> mostInteresting = getCompactionCandidates(candidates, now, base);
+         List<SSTableReader> mostInteresting = getCompactionCandidates(nonExpiringSSTables, now, base);
          if (mostInteresting != null)
 +        {
              return mostInteresting;
 +        }
  
          // if there is no sstable to compact in standard way, try compacting single sstable whose droppable tombstone
          // ratio is greater than threshold.

http://git-wip-us.apache.org/repos/asf/cassandra/blob/52191ebe/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
index cedd821,1fa41a3..becddeb
--- a/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
@@@ -273,4 -273,47 +273,47 @@@ public class DateTieredCompactionStrate
          filtered = filterOldSSTables(sstrs, 1, 4);
          assertEquals("no sstables should remain when all are too old", 0, Iterables.size(filtered));
      }
+ 
+ 
+     @Test
+     public void testDropExpiredSSTables() throws InterruptedException
+     {
+         Keyspace keyspace = Keyspace.open(KEYSPACE1);
+         ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD1);
+         cfs.truncateBlocking();
+         cfs.disableAutoCompaction();
+ 
+         ByteBuffer value = ByteBuffer.wrap(new byte[100]);
+ 
+         // create 2 sstables
+         DecoratedKey key = Util.dk(String.valueOf("expired"));
 -        RowMutation rm = new RowMutation(KEYSPACE1, key.key);
 -        rm.add(CF_STANDARD1, ByteBufferUtil.bytes("column"), value, System.currentTimeMillis(), 5);
++        Mutation rm = new Mutation(KEYSPACE1, key.getKey());
++        rm.add(CF_STANDARD1, Util.cellname("column"), value, System.currentTimeMillis(), 5);
+         rm.apply();
+         cfs.forceBlockingFlush();
+         SSTableReader expiredSSTable = cfs.getSSTables().iterator().next();
+         Thread.sleep(10);
+         key = Util.dk(String.valueOf("nonexpired"));
 -        rm = new RowMutation(KEYSPACE1, key.key);
 -        rm.add(CF_STANDARD1, ByteBufferUtil.bytes("column"), value, System.currentTimeMillis());
++        rm = new Mutation(KEYSPACE1, key.getKey());
++        rm.add(CF_STANDARD1, Util.cellname("column"), value, System.currentTimeMillis());
+         rm.apply();
+         cfs.forceBlockingFlush();
+         assertEquals(cfs.getSSTables().size(), 2);
+ 
+         Map<String, String> options = new HashMap<>();
+ 
+         options.put(DateTieredCompactionStrategyOptions.BASE_TIME_KEY, "30");
+         options.put(DateTieredCompactionStrategyOptions.TIMESTAMP_RESOLUTION_KEY, "MILLISECONDS");
+         options.put(DateTieredCompactionStrategyOptions.MAX_SSTABLE_AGE_KEY, Double.toString((1d / (24 * 60 * 60))));
+         DateTieredCompactionStrategy dtcs = new DateTieredCompactionStrategy(cfs, options);
+         dtcs.startup();
+         assertNull(dtcs.getNextBackgroundTask((int) (System.currentTimeMillis() / 1000)));
+         Thread.sleep(7000);
+         AbstractCompactionTask t = dtcs.getNextBackgroundTask((int) (System.currentTimeMillis()/1000));
+         assertNotNull(t);
+         assertEquals(1, Iterables.size(t.sstables));
+         SSTableReader sstable = t.sstables.iterator().next();
+         assertEquals(sstable, expiredSSTable);
+     }
+ 
  }


[3/3] cassandra git commit: Merge branch 'cassandra-2.1' into trunk

Posted by ma...@apache.org.
Merge branch 'cassandra-2.1' into trunk


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/b09e3929
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/b09e3929
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/b09e3929

Branch: refs/heads/trunk
Commit: b09e39295cdd26690a7d6f9a7c5417c26e9376fc
Parents: 7f0a854 52191eb
Author: Marcus Eriksson <ma...@apache.org>
Authored: Wed Apr 1 12:56:34 2015 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Wed Apr 1 12:56:34 2015 +0200

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../DateTieredCompactionStrategy.java           | 31 ++++++++++----
 .../DateTieredCompactionStrategyTest.java       | 43 ++++++++++++++++++++
 3 files changed, 67 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/b09e3929/CHANGES.txt
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/b09e3929/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/b09e3929/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
----------------------------------------------------------------------