You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ma...@apache.org on 2014/11/14 11:11:29 UTC
[1/2] cassandra git commit: Do more aggressive ttl expiration checks
to be able to drop more sstables
Repository: cassandra
Updated Branches:
refs/heads/trunk 3b6edc6af -> 41a35ec74
Do more aggressive ttl expiration checks to be able to drop more sstables
Patch by Bjorn Hegerfors; reviewed by marcuse and slebresne for CASSANDRA-8243
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/abbcfc5f
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/abbcfc5f
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/abbcfc5f
Branch: refs/heads/trunk
Commit: abbcfc5f72323d0c6040a3cc02aba8f2c0058d95
Parents: 054beee
Author: Marcus Eriksson <ma...@apache.org>
Authored: Fri Nov 14 10:54:39 2014 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Fri Nov 14 11:02:45 2014 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../db/compaction/CompactionController.java | 20 ++++----
.../cassandra/db/compaction/TTLExpiryTest.java | 50 ++++++++++++++++++++
3 files changed, 60 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/abbcfc5f/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 6228893..2476d25 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.1.3
+ * Do more aggressive entire-sstable TTL expiry checks (CASSANDRA-8243)
* Add more log info if readMeter is null (CASSANDRA-8238)
* add check of the system wall clock time at startup (CASSANDRA-8305)
* Support for frozen collections (CASSANDRA-7859)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/abbcfc5f/src/java/org/apache/cassandra/db/compaction/CompactionController.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionController.java b/src/java/org/apache/cassandra/db/compaction/CompactionController.java
index ef27805..f23d39a 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionController.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionController.java
@@ -92,12 +92,11 @@ public class CompactionController implements AutoCloseable
* Finds expired sstables
*
* works something like this;
- * 1. find "global" minTimestamp of overlapping sstables (excluding the possibly droppable ones)
- * 2. build a list of candidates to be dropped
- * 3. sort the candidate list, biggest maxTimestamp first in list
- * 4. check if the candidates to be dropped actually can be dropped (maxTimestamp < global minTimestamp) and it is included in the compaction
- * - if not droppable, update global minTimestamp and remove from candidates
- * 5. return candidates.
+ * 1. find "global" minTimestamp of overlapping sstables and compacting sstables containing any non-expired data
+ * 2. build a list of fully expired candidates
+ * 3. check if the candidates to be dropped actually can be dropped (maxTimestamp < global minTimestamp)
+ * - if not droppable, remove from candidates
+ * 4. return candidates.
*
* @param cfStore
* @param compacting we take the drop-candidates from this set, it is usually the sstables included in the compaction
@@ -127,10 +126,10 @@ public class CompactionController implements AutoCloseable
minTimestamp = Math.min(minTimestamp, candidate.getMinTimestamp());
}
- // we still need to keep candidates that might shadow something in a
- // non-candidate sstable. And if we remove a sstable from the candidates, we
- // must take it's timestamp into account (hence the sorting below).
- Collections.sort(candidates, SSTableReader.maxTimestampComparator);
+ // At this point, minTimestamp denotes the lowest timestamp of any relevant
+ // SSTable that contains a constructive value. candidates contains all the
+ // candidates with no constructive values. The ones out of these that have
+ // (getMaxTimestamp() < minTimestamp) serve no purpose anymore.
Iterator<SSTableReader> iterator = candidates.iterator();
while (iterator.hasNext())
@@ -138,7 +137,6 @@ public class CompactionController implements AutoCloseable
SSTableReader candidate = iterator.next();
if (candidate.getMaxTimestamp() >= minTimestamp)
{
- minTimestamp = Math.min(candidate.getMinTimestamp(), minTimestamp);
iterator.remove();
}
else
http://git-wip-us.apache.org/repos/asf/cassandra/blob/abbcfc5f/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java b/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
index b98af68..4fe5cfb 100644
--- a/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
@@ -20,6 +20,7 @@ package org.apache.cassandra.db.compaction;
*
*/
+import com.google.common.collect.Sets;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -31,6 +32,10 @@ import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.io.sstable.SSTableScanner;
import org.apache.cassandra.utils.ByteBufferUtil;
+
+import java.util.Collections;
+import java.util.Set;
+
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -38,6 +43,51 @@ import static org.junit.Assert.assertTrue;
public class TTLExpiryTest extends SchemaLoader
{
@Test
+ public void testAggressiveFullyExpired()
+ {
+ ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
+ cfs.disableAutoCompaction();
+ cfs.metadata.gcGraceSeconds(0);
+
+ DecoratedKey ttlKey = Util.dk("ttl");
+ Mutation rm = new Mutation("Keyspace1", ttlKey.getKey());
+ rm.add("Standard1", Util.cellname("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 1, 1);
+ rm.add("Standard1", Util.cellname("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 3, 1);
+ rm.applyUnsafe();
+ cfs.forceBlockingFlush();
+
+ rm = new Mutation("Keyspace1", ttlKey.getKey());
+ rm.add("Standard1", Util.cellname("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 2, 1);
+ rm.add("Standard1", Util.cellname("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 5, 1);
+ rm.applyUnsafe();
+ cfs.forceBlockingFlush();
+
+ rm = new Mutation("Keyspace1", ttlKey.getKey());
+ rm.add("Standard1", Util.cellname("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 4, 1);
+ rm.add("Standard1", Util.cellname("shadow"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 7, 1);
+ rm.applyUnsafe();
+ cfs.forceBlockingFlush();
+
+ rm = new Mutation("Keyspace1", ttlKey.getKey());
+ rm.add("Standard1", Util.cellname("shadow"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 6, 3);
+ rm.add("Standard1", Util.cellname("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 8, 1);
+ rm.applyUnsafe();
+ cfs.forceBlockingFlush();
+
+ Set<SSTableReader> sstables = Sets.newHashSet(cfs.getSSTables());
+ int now = (int)(System.currentTimeMillis() / 1000);
+ int gcBefore = now + 2;
+ Set<SSTableReader> expired = CompactionController.getFullyExpiredSSTables(
+ cfs,
+ sstables,
+ Collections.EMPTY_SET,
+ gcBefore);
+ assertEquals(2, expired.size());
+
+ cfs.clearUnsafe();
+ }
+
+ @Test
public void testSimpleExpire() throws InterruptedException
{
ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
[2/2] cassandra git commit: Merge branch 'cassandra-2.1' into trunk
Posted by ma...@apache.org.
Merge branch 'cassandra-2.1' into trunk
Conflicts:
test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/41a35ec7
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/41a35ec7
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/41a35ec7
Branch: refs/heads/trunk
Commit: 41a35ec74e7bc27111cfb994bafb7b9389f94d2b
Parents: 3b6edc6 abbcfc5
Author: Marcus Eriksson <ma...@apache.org>
Authored: Fri Nov 14 11:10:09 2014 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Fri Nov 14 11:10:09 2014 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../db/compaction/CompactionController.java | 20 ++++----
.../cassandra/db/compaction/TTLExpiryTest.java | 50 ++++++++++++++++++++
3 files changed, 60 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/41a35ec7/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 208381f,2476d25..f250edc
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,38 -1,5 +1,39 @@@
+3.0
+ * Fix aggregate fn results on empty selection, result column name,
+ and cqlsh parsing (CASSANDRA-8229)
+ * Mark sstables as repaired after full repair (CASSANDRA-7586)
+ * Extend Descriptor to include a format value and refactor reader/writer apis (CASSANDRA-7443)
+ * Integrate JMH for microbenchmarks (CASSANDRA-8151)
+ * Keep sstable levels when bootstrapping (CASSANDRA-7460)
+ * Add Sigar library and perform basic OS settings check on startup (CASSANDRA-7838)
+ * Support for aggregation functions (CASSANDRA-4914)
+ * Remove cassandra-cli (CASSANDRA-7920)
+ * Accept dollar quoted strings in CQL (CASSANDRA-7769)
+ * Make assassinate a first class command (CASSANDRA-7935)
+ * Support IN clause on any clustering column (CASSANDRA-4762)
+ * Improve compaction logging (CASSANDRA-7818)
+ * Remove YamlFileNetworkTopologySnitch (CASSANDRA-7917)
+ * Do anticompaction in groups (CASSANDRA-6851)
+ * Support pure user-defined functions (CASSANDRA-7395, 7526, 7562, 7740, 7781, 7929,
+ 7924, 7812, 8063)
+ * Permit configurable timestamps with cassandra-stress (CASSANDRA-7416)
+ * Move sstable RandomAccessReader to nio2, which allows using the
+ FILE_SHARE_DELETE flag on Windows (CASSANDRA-4050)
+ * Remove CQL2 (CASSANDRA-5918)
+ * Add Thrift get_multi_slice call (CASSANDRA-6757)
+ * Optimize fetching multiple cells by name (CASSANDRA-6933)
+ * Allow compilation in java 8 (CASSANDRA-7028)
+ * Make incremental repair default (CASSANDRA-7250)
+ * Enable code coverage thru JaCoCo (CASSANDRA-7226)
+ * Switch external naming of 'column families' to 'tables' (CASSANDRA-4369)
+ * Shorten SSTable path (CASSANDRA-6962)
+ * Use unsafe mutations for most unit tests (CASSANDRA-6969)
+ * Fix race condition during calculation of pending ranges (CASSANDRA-7390)
+ * Fail on very large batch sizes (CASSANDRA-8011)
+ * improve concurrency of repair (CASSANDRA-6455, 8208)
+
2.1.3
+ * Do more aggressive entire-sstable TTL expiry checks (CASSANDRA-8243)
* Add more log info if readMeter is null (CASSANDRA-8238)
* add check of the system wall clock time at startup (CASSANDRA-8305)
* Support for frozen collections (CASSANDRA-7859)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/41a35ec7/src/java/org/apache/cassandra/db/compaction/CompactionController.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/41a35ec7/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
index 636370b,4fe5cfb..924c4b5
--- a/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
@@@ -20,43 -20,77 +20,93 @@@ package org.apache.cassandra.db.compact
*
*/
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.junit.BeforeClass;
+ import com.google.common.collect.Sets;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.apache.cassandra.OrderedJUnit4ClassRunner;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
+import org.apache.cassandra.config.KSMetaData;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
-import org.apache.cassandra.io.sstable.SSTableReader;
-import org.apache.cassandra.io.sstable.SSTableScanner;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.cassandra.utils.ByteBufferUtil;
+
+ import java.util.Collections;
+ import java.util.Set;
+
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@RunWith(OrderedJUnit4ClassRunner.class)
-public class TTLExpiryTest extends SchemaLoader
+public class TTLExpiryTest
{
+ public static final String KEYSPACE1 = "TTLExpiryTest";
+ private static final String CF_STANDARD1 = "Standard1";
+
+ @BeforeClass
+ public static void defineSchema() throws ConfigurationException
+ {
+ SchemaLoader.prepareServer();
+ SchemaLoader.createKeyspace(KEYSPACE1,
+ SimpleStrategy.class,
+ KSMetaData.optsWithRF(1),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1));
+ }
+
@Test
+ public void testAggressiveFullyExpired()
+ {
- ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
++ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore("Standard1");
+ cfs.disableAutoCompaction();
+ cfs.metadata.gcGraceSeconds(0);
+
+ DecoratedKey ttlKey = Util.dk("ttl");
- Mutation rm = new Mutation("Keyspace1", ttlKey.getKey());
++ Mutation rm = new Mutation(KEYSPACE1, ttlKey.getKey());
+ rm.add("Standard1", Util.cellname("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 1, 1);
+ rm.add("Standard1", Util.cellname("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 3, 1);
+ rm.applyUnsafe();
+ cfs.forceBlockingFlush();
+
- rm = new Mutation("Keyspace1", ttlKey.getKey());
++ rm = new Mutation(KEYSPACE1, ttlKey.getKey());
+ rm.add("Standard1", Util.cellname("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 2, 1);
+ rm.add("Standard1", Util.cellname("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 5, 1);
+ rm.applyUnsafe();
+ cfs.forceBlockingFlush();
+
- rm = new Mutation("Keyspace1", ttlKey.getKey());
++ rm = new Mutation(KEYSPACE1, ttlKey.getKey());
+ rm.add("Standard1", Util.cellname("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 4, 1);
+ rm.add("Standard1", Util.cellname("shadow"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 7, 1);
+ rm.applyUnsafe();
+ cfs.forceBlockingFlush();
+
- rm = new Mutation("Keyspace1", ttlKey.getKey());
++ rm = new Mutation(KEYSPACE1, ttlKey.getKey());
+ rm.add("Standard1", Util.cellname("shadow"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 6, 3);
+ rm.add("Standard1", Util.cellname("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 8, 1);
+ rm.applyUnsafe();
+ cfs.forceBlockingFlush();
+
+ Set<SSTableReader> sstables = Sets.newHashSet(cfs.getSSTables());
+ int now = (int)(System.currentTimeMillis() / 1000);
+ int gcBefore = now + 2;
+ Set<SSTableReader> expired = CompactionController.getFullyExpiredSSTables(
+ cfs,
+ sstables,
+ Collections.EMPTY_SET,
+ gcBefore);
+ assertEquals(2, expired.size());
+
+ cfs.clearUnsafe();
+ }
+
+ @Test
public void testSimpleExpire() throws InterruptedException
{
- ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore("Standard1");
cfs.disableAutoCompaction();
cfs.metadata.gcGraceSeconds(0);
long timestamp = System.currentTimeMillis();