You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2009/12/23 20:03:27 UTC
svn commit: r893599 - in /incubator/cassandra/trunk: ./
interface/gen-java/org/apache/cassandra/service/ src/java/org/
src/java/org/apache/cassandra/db/ src/java/org/apache/cassandra/io/
src/java/org/apache/cassandra/utils/ test/unit/org/
Author: jbellis
Date: Wed Dec 23 19:03:25 2009
New Revision: 893599
URL: http://svn.apache.org/viewvc?rev=893599&view=rev
Log:
merge from 0.5
Modified:
incubator/cassandra/trunk/ (props changed)
incubator/cassandra/trunk/CHANGES.txt
incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/Cassandra.java (props changed)
incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/Column.java (props changed)
incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java (props changed)
incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/NotFoundException.java (props changed)
incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/SuperColumn.java (props changed)
incubator/cassandra/trunk/src/java/org/ (props changed)
incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/io/CompactionIterator.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/utils/FBUtilities.java
incubator/cassandra/trunk/test/unit/org/ (props changed)
Propchange: incubator/cassandra/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 23 19:03:25 2009
@@ -1,3 +1,3 @@
/incubator/cassandra/branches/cassandra-0.3:774578-796573
/incubator/cassandra/branches/cassandra-0.4:810145-834239,834349-834350
-/incubator/cassandra/branches/cassandra-0.5:888872-893584
+/incubator/cassandra/branches/cassandra-0.5:888872-893596
Modified: incubator/cassandra/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/CHANGES.txt?rev=893599&r1=893598&r2=893599&view=diff
==============================================================================
--- incubator/cassandra/trunk/CHANGES.txt (original)
+++ incubator/cassandra/trunk/CHANGES.txt Wed Dec 23 19:03:25 2009
@@ -19,6 +19,7 @@
(CASSANDRA-643)
* fix bug determining closest neighbor for use in multiple datacenters
(CASSANDRA-648)
+ * Vast improvements in anticompaction speed (CASSANDRA-607)
0.5.0 beta 2
Propchange: incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/Cassandra.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 23 19:03:25 2009
@@ -1,4 +1,4 @@
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/Cassandra.java:774578-796573
/incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/Cassandra.java:810145-834239,834349-834350
-/incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/Cassandra.java:888872-893584
+/incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/Cassandra.java:888872-893596
/incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/Cassandra.java:749219-768588
Propchange: incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/Column.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 23 19:03:25 2009
@@ -1,5 +1,5 @@
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/column_t.java:774578-792198
/incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/Column.java:810145-834239,834349-834350
-/incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/Column.java:888872-893584
+/incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/Column.java:888872-893596
/incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/Column.java:749219-794428
/incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/column_t.java:749219-768588
Propchange: incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 23 19:03:25 2009
@@ -1,4 +1,4 @@
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:774578-796573
/incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:810145-834239,834349-834350
-/incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:888872-893584
+/incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:888872-893596
/incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:749219-768588
Propchange: incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/NotFoundException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 23 19:03:25 2009
@@ -1,4 +1,4 @@
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:774578-796573
/incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:810145-834239,834349-834350
-/incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:888872-893584
+/incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:888872-893596
/incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:749219-768588
Propchange: incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/SuperColumn.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 23 19:03:25 2009
@@ -1,5 +1,5 @@
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/superColumn_t.java:774578-792198
/incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/SuperColumn.java:810145-834239,834349-834350
-/incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/SuperColumn.java:888872-893584
+/incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/SuperColumn.java:888872-893596
/incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/SuperColumn.java:749219-794428
/incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/superColumn_t.java:749219-768588
Propchange: incubator/cassandra/trunk/src/java/org/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 23 19:03:25 2009
@@ -1,4 +1,4 @@
/incubator/cassandra/branches/cassandra-0.3/src/java/org:774578-796573
/incubator/cassandra/branches/cassandra-0.4/src/java/org:810145-834239,834349-834350
-/incubator/cassandra/branches/cassandra-0.5/src/java/org:888872-893584
+/incubator/cassandra/branches/cassandra-0.5/src/java/org:888872-893596
/incubator/cassandra/trunk/src/java/org:749219-769885
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=893599&r1=893598&r2=893599&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java Wed Dec 23 19:03:25 2009
@@ -53,12 +53,12 @@
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.collections.IteratorUtils;
import org.apache.commons.collections.PredicateUtils;
+import org.apache.commons.collections.iterators.CollatingIterator;
import org.apache.commons.collections.iterators.FilterIterator;
import org.cliffc.high_scale_lib.NonBlockingHashMap;
-
-import com.google.common.base.Predicate;
import com.google.common.collect.Iterators;
+import com.google.common.base.Predicate;
public final class ColumnFamilyStore implements ColumnFamilyStoreMBean
{
@@ -655,7 +655,6 @@
// if we have too many to compact all at once, compact older ones first -- this avoids
// re-compacting files we just created.
Collections.sort(sstables);
- boolean major = sstables.size() == ssTables_.size();
filesCompacted += doFileCompaction(sstables.subList(0, Math.min(sstables.size(), maxThreshold)));
}
logger_.debug(filesCompacted + " files compacted");
@@ -812,7 +811,7 @@
logger_.debug("Expected bloom filter size : " + expectedBloomFilterSize);
SSTableWriter writer = null;
- CompactionIterator ci = new CompactionIterator(sstables, getDefaultGCBefore(), sstables.size() == ssTables_.size());
+ CompactionIterator ci = new AntiCompactionIterator(sstables, ranges, getDefaultGCBefore(), sstables.size() == ssTables_.size());
Iterator<CompactionIterator.CompactedRow> nni = new FilterIterator(ci, PredicateUtils.notNullPredicate());
try
@@ -825,17 +824,14 @@
while (nni.hasNext())
{
CompactionIterator.CompactedRow row = nni.next();
- if (Range.isTokenInRanges(row.key.token, ranges))
- {
- if (writer == null)
- {
- FileUtils.createDirectory(compactionFileLocation);
- String newFilename = new File(compactionFileLocation, getTempSSTableFileName()).getAbsolutePath();
- writer = new SSTableWriter(newFilename, expectedBloomFilterSize, StorageService.getPartitioner());
- }
- writer.append(row.key, row.buffer);
- totalkeysWritten++;
- }
+ if (writer == null)
+ {
+ FileUtils.createDirectory(compactionFileLocation);
+ String newFilename = new File(compactionFileLocation, getTempSSTableFileName()).getAbsolutePath();
+ writer = new SSTableWriter(newFilename, expectedBloomFilterSize, StorageService.getPartitioner());
+ }
+ writer.append(row.key, row.buffer);
+ totalkeysWritten++;
}
}
finally
@@ -1599,4 +1595,39 @@
ssTables_.clearUnsafe();
}
+ private static class AntiCompactionIterator extends CompactionIterator
+ {
+ public AntiCompactionIterator(Collection<SSTableReader> sstables, Collection<Range> ranges, int gcBefore, boolean isMajor)
+ throws IOException
+ {
+ super(getCollatedRangeIterator(sstables, ranges), gcBefore, isMajor);
+ }
+
+ private static Iterator getCollatedRangeIterator(Collection<SSTableReader> sstables, final Collection<Range> ranges)
+ throws IOException
+ {
+ org.apache.commons.collections.Predicate rangesPredicate = new org.apache.commons.collections.Predicate()
+ {
+ public boolean evaluate(Object row)
+ {
+ return Range.isTokenInRanges(((IteratingRow)row).getKey().token, ranges);
+ }
+ };
+ CollatingIterator iter = FBUtilities.<IteratingRow>getCollatingIterator();
+ for (SSTableReader sstable : sstables)
+ {
+ SSTableScanner scanner = sstable.getScanner(FILE_BUFFER_SIZE);
+ iter.addIterator(new FilterIterator(scanner, rangesPredicate));
+ }
+ return iter;
+ }
+
+ public void close() throws IOException
+ {
+ for (Object o : ((CollatingIterator)source).getIterators())
+ {
+ ((SSTableScanner)((FilterIterator)o).getIterator()).close();
+ }
+ }
+ }
}
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/io/CompactionIterator.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/io/CompactionIterator.java?rev=893599&r1=893598&r2=893599&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/io/CompactionIterator.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/io/CompactionIterator.java Wed Dec 23 19:03:25 2009
@@ -26,12 +26,13 @@
import java.io.IOError;
import java.util.List;
import java.util.ArrayList;
-import java.util.Comparator;
+import java.util.Iterator;
import org.apache.log4j.Logger;
import org.apache.commons.collections.iterators.CollatingIterator;
import org.apache.cassandra.utils.ReducingIterator;
+import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.db.ColumnFamily;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.ColumnFamilyStore;
@@ -40,31 +41,29 @@
{
private static Logger logger = Logger.getLogger(CompactionIterator.class);
- private static final int FILE_BUFFER_SIZE = 1024 * 1024;
+ protected static final int FILE_BUFFER_SIZE = 1024 * 1024;
private final List<IteratingRow> rows = new ArrayList<IteratingRow>();
private final int gcBefore;
private boolean major;
- @SuppressWarnings("unchecked")
public CompactionIterator(Iterable<SSTableReader> sstables, int gcBefore, boolean major) throws IOException
{
- super(getCollatingIterator(sstables));
+ this(getCollatingIterator(sstables), gcBefore, major);
+ }
+
+ @SuppressWarnings("unchecked")
+ protected CompactionIterator(Iterator iter, int gcBefore, boolean major)
+ {
+ super(iter);
this.gcBefore = gcBefore;
this.major = major;
}
@SuppressWarnings("unchecked")
- private static CollatingIterator getCollatingIterator(Iterable<SSTableReader> sstables) throws IOException
+ protected static CollatingIterator getCollatingIterator(Iterable<SSTableReader> sstables) throws IOException
{
- // CollatingIterator has a bug that causes NPE when you try to use default comparator. :(
- CollatingIterator iter = new CollatingIterator(new Comparator()
- {
- public int compare(Object o1, Object o2)
- {
- return ((Comparable)o1).compareTo(o2);
- }
- });
+ CollatingIterator iter = FBUtilities.<IteratingRow>getCollatingIterator();
for (SSTableReader sstable : sstables)
{
iter.addIterator(sstable.getScanner(FILE_BUFFER_SIZE));
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/utils/FBUtilities.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/utils/FBUtilities.java?rev=893599&r1=893598&r2=893599&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/utils/FBUtilities.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/utils/FBUtilities.java Wed Dec 23 19:03:25 2009
@@ -30,6 +30,8 @@
import org.apache.log4j.Logger;
+import org.apache.commons.collections.iterators.CollatingIterator;
+
import org.apache.cassandra.config.DatabaseDescriptor;
public class FBUtilities
@@ -304,4 +306,16 @@
throw new IOException("rename failed of " + filename);
}
}
+
+ public static <T extends Comparable<T>> CollatingIterator getCollatingIterator()
+ {
+ // CollatingIterator will happily NPE if you do not specify a comparator explicitly
+ return new CollatingIterator(new Comparator<T>()
+ {
+ public int compare(T o1, T o2)
+ {
+ return o1.compareTo(o2);
+ }
+ });
+ }
}
Propchange: incubator/cassandra/trunk/test/unit/org/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 23 19:03:25 2009
@@ -1,4 +1,4 @@
/incubator/cassandra/branches/cassandra-0.3/test/unit/org:774578-796573
/incubator/cassandra/branches/cassandra-0.4/test/unit/org:810145-834239,834349-834350
-/incubator/cassandra/branches/cassandra-0.5/test/unit/org:888872-893584
+/incubator/cassandra/branches/cassandra-0.5/test/unit/org:888872-893596
/incubator/cassandra/trunk/test/unit/org:749219-768583