You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by yu...@apache.org on 2012/10/08 20:37:15 UTC
git commit: configurable bucket size for STC;
patch by Radim Kolar, reviewed by yukim for CASSANDRA-4704
Updated Branches:
refs/heads/trunk 5160de591 -> 7a2b45ae4
configurable bucket size for STC; patch by Radim Kolar, reviewed by yukim for CASSANDRA-4704
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/7a2b45ae
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/7a2b45ae
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/7a2b45ae
Branch: refs/heads/trunk
Commit: 7a2b45ae4205572e6b8b4309bbac80859c709c2b
Parents: 5160de5
Author: Yuki Morishita <yu...@apache.org>
Authored: Mon Oct 8 13:34:30 2012 -0500
Committer: Yuki Morishita <yu...@apache.org>
Committed: Mon Oct 8 13:35:33 2012 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../compaction/SizeTieredCompactionStrategy.java | 23 +++++++++++++--
.../SizeTieredCompactionStrategyTest.java | 17 ++++++++--
3 files changed, 34 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7a2b45ae/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 5f02032..6ffa9c9 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -22,6 +22,7 @@
* Use rpc_address for binary protocol and change default port (CASSANRA-4751)
* Fix use of collections in prepared statements (CASSANDRA-4739)
* Store more information into peers table (CASSANDRA-4351)
+ * Configurable bucket size for size tiered compaction (CASSANDRA-4704)
1.2-beta1
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7a2b45ae/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
index 6b9715e..a56345f 100644
--- a/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
@@ -32,8 +32,15 @@ public class SizeTieredCompactionStrategy extends AbstractCompactionStrategy
{
private static final Logger logger = LoggerFactory.getLogger(SizeTieredCompactionStrategy.class);
protected static final long DEFAULT_MIN_SSTABLE_SIZE = 50L * 1024L * 1024L;
+ protected static final double DEFAULT_BUCKET_LOW = 0.5;
+ protected static final double DEFAULT_BUCKET_HIGH = 1.5;
protected static final String MIN_SSTABLE_SIZE_KEY = "min_sstable_size";
+ protected static final String BUCKET_LOW_KEY = "bucket_low";
+ protected static final String BUCKET_HIGH_KEY = "bucket_high";
+
protected long minSSTableSize;
+ protected double bucketLow;
+ protected double bucketHigh;
protected volatile int estimatedRemainingTasks;
public SizeTieredCompactionStrategy(ColumnFamilyStore cfs, Map<String, String> options)
@@ -42,6 +49,16 @@ public class SizeTieredCompactionStrategy extends AbstractCompactionStrategy
this.estimatedRemainingTasks = 0;
String optionValue = options.get(MIN_SSTABLE_SIZE_KEY);
minSSTableSize = optionValue == null ? DEFAULT_MIN_SSTABLE_SIZE : Long.parseLong(optionValue);
+ optionValue = options.get(BUCKET_LOW_KEY);
+ bucketLow = optionValue == null ? DEFAULT_BUCKET_LOW : Double.parseDouble(optionValue);
+ optionValue = options.get(BUCKET_HIGH_KEY);
+ bucketHigh = optionValue == null ? DEFAULT_BUCKET_HIGH : Double.parseDouble(optionValue);
+ if (bucketHigh <= bucketLow)
+ {
+ logger.warn("Bucket low/high marks for {} incorrect, using defaults.", cfs.getColumnFamilyName());
+ bucketLow = DEFAULT_BUCKET_LOW;
+ bucketHigh = DEFAULT_BUCKET_HIGH;
+ }
cfs.setCompactionThresholds(cfs.metadata.getMinCompactionThreshold(), cfs.metadata.getMaxCompactionThreshold());
}
@@ -54,7 +71,7 @@ public class SizeTieredCompactionStrategy extends AbstractCompactionStrategy
}
Set<SSTableReader> candidates = cfs.getUncompactingSSTables();
- List<List<SSTableReader>> buckets = getBuckets(createSSTableAndLengthPairs(filterSuspectSSTables(candidates)), minSSTableSize);
+ List<List<SSTableReader>> buckets = getBuckets(createSSTableAndLengthPairs(filterSuspectSSTables(candidates)));
logger.debug("Compaction buckets are {}", buckets);
updateEstimatedCompactionsByTasks(buckets);
@@ -141,7 +158,7 @@ public class SizeTieredCompactionStrategy extends AbstractCompactionStrategy
/*
* Group files of similar size into buckets.
*/
- static <T> List<List<T>> getBuckets(Collection<Pair<T, Long>> files, long minSSTableSize)
+ <T> List<List<T>> getBuckets(Collection<Pair<T, Long>> files)
{
// Sort the list in order to get deterministic results during the grouping below
List<Pair<T, Long>> sortedFiles = new ArrayList<Pair<T, Long>>(files);
@@ -167,7 +184,7 @@ public class SizeTieredCompactionStrategy extends AbstractCompactionStrategy
{
List<T> bucket = entry.getValue();
long oldAverageSize = entry.getKey();
- if ((size > (oldAverageSize / 2) && size < (3 * oldAverageSize) / 2)
+ if ((size > (oldAverageSize * bucketLow) && size < (oldAverageSize * bucketHigh))
|| (size < minSSTableSize && oldAverageSize < minSSTableSize))
{
// remove and re-add under new new average size
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7a2b45ae/test/unit/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategyTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategyTest.java
index 89e0a6a..3df02e5 100644
--- a/test/unit/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategyTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategyTest.java
@@ -22,9 +22,12 @@ import java.util.*;
import org.junit.Test;
import static org.junit.Assert.*;
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Table;
import org.apache.cassandra.utils.Pair;
-public class SizeTieredCompactionStrategyTest
+public class SizeTieredCompactionStrategyTest extends SchemaLoader
{
@Test
public void testGetBuckets()
@@ -37,7 +40,11 @@ public class SizeTieredCompactionStrategyTest
pairs.add(pair);
}
- List<List<String>> buckets = SizeTieredCompactionStrategy.getBuckets(pairs, 2);
+ ColumnFamilyStore cfs = Table.open("Keyspace1").getColumnFamilyStore("Standard1");
+ Map<String, String> opts = new HashMap<String, String>();
+ opts.put(SizeTieredCompactionStrategy.MIN_SSTABLE_SIZE_KEY, "2");
+ SizeTieredCompactionStrategy strategy = new SizeTieredCompactionStrategy(cfs, opts);
+ List<List<String>> buckets = strategy.getBuckets(pairs);
assertEquals(3, buckets.size());
for (List<String> bucket : buckets)
@@ -57,7 +64,7 @@ public class SizeTieredCompactionStrategyTest
pairs.add(pair);
}
- buckets = SizeTieredCompactionStrategy.getBuckets(pairs, 2);
+ buckets = strategy.getBuckets(pairs);
assertEquals(2, buckets.size());
for (List<String> bucket : buckets)
@@ -78,7 +85,9 @@ public class SizeTieredCompactionStrategyTest
pairs.add(pair);
}
- buckets = SizeTieredCompactionStrategy.getBuckets(pairs, 10); // notice the min is 10
+ opts.put(SizeTieredCompactionStrategy.MIN_SSTABLE_SIZE_KEY, "10");
+ strategy = new SizeTieredCompactionStrategy(cfs, opts);
+ buckets = strategy.getBuckets(pairs); // notice the min is 10
assertEquals(1, buckets.size());
}
}