You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2011/09/22 22:10:30 UTC
svn commit: r1174364 - in
/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction:
AbstractCompactionStrategy.java CompactionTask.java
LeveledCompactionStrategy.java SizeTieredCompactionStrategy.java
Author: jbellis
Date: Thu Sep 22 20:10:28 2011
New Revision: 1174364
URL: http://svn.apache.org/viewvc?rev=1174364&view=rev
Log:
fix leveled BF size calculation
patch by jbellis; tested by brandonwilliams for CASSANDRA-3234
Modified:
cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java?rev=1174364&r1=1174363&r2=1174364&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java (original)
+++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java Thu Sep 22 20:10:28 2011
@@ -92,4 +92,9 @@ public abstract class AbstractCompaction
* @return the number of background tasks estimated to still be needed for this columnfamilystore
*/
public abstract int getEstimatedRemainingTasks();
+
+ /**
+ * @return size in bytes of the largest sstables for this strategy
+ */
+ public abstract long getMaxSSTableSize();
}
Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionTask.java?rev=1174364&r1=1174363&r2=1174364&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionTask.java (original)
+++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionTask.java Thu Sep 22 20:10:28 2011
@@ -120,10 +120,11 @@ public class CompactionTask extends Abst
long startTime = System.currentTimeMillis();
long totalkeysWritten = 0;
- // TODO the int cast here is potentially buggy
- int expectedBloomFilterSize = Math.max(DatabaseDescriptor.getIndexInterval(), (int)SSTableReader.getApproximateKeyCount(toCompact));
+ long estimatedTotalKeys = Math.max(DatabaseDescriptor.getIndexInterval(), SSTableReader.getApproximateKeyCount(toCompact));
+ long estimatedSSTables = Math.max(1, SSTable.getTotalBytes(toCompact) / cfs.getCompactionStrategy().getMaxSSTableSize());
+ long keysPerSSTable = (long) Math.ceil((double) estimatedTotalKeys / estimatedSSTables);
if (logger.isDebugEnabled())
- logger.debug("Expected bloom filter size : " + expectedBloomFilterSize);
+ logger.debug("Expected bloom filter size : " + keysPerSSTable);
AbstractCompactionIterable ci = DatabaseDescriptor.isMultithreadedCompaction()
? new ParallelCompactionIterable(OperationType.COMPACTION, toCompact, controller)
@@ -152,7 +153,7 @@ public class CompactionTask extends Abst
return 0;
}
- SSTableWriter writer = cfs.createCompactionWriter(expectedBloomFilterSize, compactionFileLocation, toCompact);
+ SSTableWriter writer = cfs.createCompactionWriter(keysPerSSTable, compactionFileLocation, toCompact);
writers.add(writer);
while (nni.hasNext())
{
@@ -179,7 +180,7 @@ public class CompactionTask extends Abst
SSTableReader toIndex = writer.closeAndOpenReader(getMaxDataAge(toCompact));
cachedKeyMap.put(toIndex, cachedKeys);
sstables.add(toIndex);
- writer = cfs.createCompactionWriter(expectedBloomFilterSize, compactionFileLocation, toCompact);
+ writer = cfs.createCompactionWriter(keysPerSSTable, compactionFileLocation, toCompact);
writers.add(writer);
cachedKeys = new HashMap<DecoratedKey, Long>();
}
Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java?rev=1174364&r1=1174363&r2=1174364&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java (original)
+++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java Thu Sep 22 20:10:28 2011
@@ -28,11 +28,9 @@ import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
-import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.concurrent.DebuggableScheduledThreadPoolExecutor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.notifications.INotification;
@@ -47,7 +45,7 @@ public class LeveledCompactionStrategy e
private LeveledManifest manifest;
private final String SSTABLE_SIZE_OPTION = "sstable_size_in_mb";
- private final int maxSSTableSize;
+ private final int maxSSTableSizeInMB;
private final AtomicReference<LeveledCompactionTask> task = new AtomicReference<LeveledCompactionTask>();
public LeveledCompactionStrategy(ColumnFamilyStore cfs, Map<String, String> options)
@@ -70,12 +68,12 @@ public class LeveledCompactionStrategy e
}
}
}
- maxSSTableSize = configuredMaxSSTableSize;
+ maxSSTableSizeInMB = configuredMaxSSTableSize;
cfs.getDataTracker().subscribe(this);
logger.info(this + " subscribed to the data tracker.");
- manifest = LeveledManifest.create(cfs, this.maxSSTableSize);
+ manifest = LeveledManifest.create(cfs, this.maxSSTableSizeInMB);
logger.debug("Created {}", manifest);
// override min/max for this strategy
cfs.setMaximumCompactionThreshold(Integer.MAX_VALUE);
@@ -119,7 +117,7 @@ public class LeveledCompactionStrategy e
return Collections.emptyList();
}
- LeveledCompactionTask newTask = new LeveledCompactionTask(cfs, sstables, gcBefore, this.maxSSTableSize);
+ LeveledCompactionTask newTask = new LeveledCompactionTask(cfs, sstables, gcBefore, this.maxSSTableSizeInMB);
return task.compareAndSet(currentTask, newTask)
? Collections.<AbstractCompactionTask>singletonList(newTask)
: Collections.<AbstractCompactionTask>emptyList();
@@ -156,6 +154,11 @@ public class LeveledCompactionStrategy e
}
}
+ public long getMaxSSTableSize()
+ {
+ return maxSSTableSizeInMB * 1024 * 1024;
+ }
+
@Override
public String toString()
{
Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java?rev=1174364&r1=1174363&r2=1174364&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java (original)
+++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java Thu Sep 22 20:10:28 2011
@@ -176,6 +176,11 @@ public class SizeTieredCompactionStrateg
return minSSTableSize;
}
+ public long getMaxSSTableSize()
+ {
+ return Long.MAX_VALUE;
+ }
+
public String toString()
{
return String.format("SizeTieredCompactionStrategy[%s/%s]",