You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2009/12/23 18:46:06 UTC
svn commit: r893584 - in
/incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra:
db/ColumnFamilyStore.java io/CompactionIterator.java io/SSTableReader.java
io/SSTableScanner.java tools/SSTableExport.java
Author: jbellis
Date: Wed Dec 23 17:45:58 2009
New Revision: 893584
URL: http://svn.apache.org/viewvc?rev=893584&view=rev
Log:
allow specifying buffer size to SSTableScanner. patch by Stu Hood and jbellis for CASSANDRA-607
Modified:
incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/io/CompactionIterator.java
incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/io/SSTableReader.java
incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/io/SSTableScanner.java
incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/tools/SSTableExport.java
Modified: incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=893584&r1=893583&r2=893584&view=diff
==============================================================================
--- incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/db/ColumnFamilyStore.java (original)
+++ incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/db/ColumnFamilyStore.java Wed Dec 23 17:45:58 2009
@@ -96,6 +96,8 @@
new NamedThreadFactory("FLUSH-WRITER-POOL"));
private static ExecutorService commitLogUpdater_ = new DebuggableThreadPoolExecutor("MEMTABLE-POST-FLUSHER");
+ private static final int KEY_RANGE_FILE_BUFFER_SIZE = 256 * 1024;
+
private final String table_;
public final String columnFamily_;
private final boolean isSuper_;
@@ -785,7 +787,7 @@
* @return
* @throws IOException
*/
- List<SSTableReader> doFileAntiCompaction(Collection<SSTableReader> sstables, Collection<Range> ranges, InetAddress target) throws IOException
+ List<SSTableReader> doFileAntiCompaction(Collection<SSTableReader> sstables, final Collection<Range> ranges, InetAddress target) throws IOException
{
logger_.info("AntiCompacting [" + StringUtils.join(sstables, ",") + "]");
// Calculate the expected compacted filesize
@@ -1315,7 +1317,7 @@
// sstables
for (SSTableReader sstable : ssTables_)
{
- final SSTableScanner scanner = sstable.getScanner();
+ final SSTableScanner scanner = sstable.getScanner(KEY_RANGE_FILE_BUFFER_SIZE);
scanner.seekTo(startWithDK);
Iterator<DecoratedKey> iter = new CloseableIterator<DecoratedKey>()
{
@@ -1434,7 +1436,7 @@
// sstables
for (SSTableReader sstable : ssTables_)
{
- final SSTableScanner scanner = sstable.getScanner();
+ final SSTableScanner scanner = sstable.getScanner(KEY_RANGE_FILE_BUFFER_SIZE);
scanner.seekTo(startWith);
Iterator<DecoratedKey> iter = new CloseableIterator<DecoratedKey>()
{
Modified: incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/io/CompactionIterator.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/io/CompactionIterator.java?rev=893584&r1=893583&r2=893584&view=diff
==============================================================================
--- incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/io/CompactionIterator.java (original)
+++ incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/io/CompactionIterator.java Wed Dec 23 17:45:58 2009
@@ -40,6 +40,8 @@
{
private static Logger logger = Logger.getLogger(CompactionIterator.class);
+ private static final int FILE_BUFFER_SIZE = 1024 * 1024;
+
private final List<IteratingRow> rows = new ArrayList<IteratingRow>();
private final int gcBefore;
private boolean major;
@@ -65,7 +67,7 @@
});
for (SSTableReader sstable : sstables)
{
- iter.addIterator(sstable.getScanner());
+ iter.addIterator(sstable.getScanner(FILE_BUFFER_SIZE));
}
return iter;
}
Modified: incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/io/SSTableReader.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/io/SSTableReader.java?rev=893584&r1=893583&r2=893584&view=diff
==============================================================================
--- incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/io/SSTableReader.java (original)
+++ incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/io/SSTableReader.java Wed Dec 23 17:45:58 2009
@@ -403,9 +403,9 @@
return partitioner;
}
- public SSTableScanner getScanner() throws IOException
+ public SSTableScanner getScanner(int bufferSize) throws IOException
{
- return new SSTableScanner(this);
+ return new SSTableScanner(this, bufferSize);
}
public AbstractType getColumnComparator()
Modified: incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/io/SSTableScanner.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/io/SSTableScanner.java?rev=893584&r1=893583&r2=893584&view=diff
==============================================================================
--- incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/io/SSTableScanner.java (original)
+++ incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/io/SSTableScanner.java Wed Dec 23 17:45:58 2009
@@ -38,13 +38,13 @@
private SSTableReader sstable;
private Iterator<IteratingRow> iterator;
- SSTableScanner(SSTableReader sstable) throws IOException
+ /**
+ * @param sstable SSTable to scan.
+ * @param bufferSize Number of bytes to buffer the file while scanning.
+ */
+ SSTableScanner(SSTableReader sstable, int bufferSize) throws IOException
{
- // TODO this is used for both compactions and key ranges. the buffer sizes we want
- // to use for these ops are very different. here we are leaning towards the key-range
- // use case since that is more common. What we really want is to split those
- // two uses of this class up.
- this.file = new BufferedRandomAccessFile(sstable.getFilename(), "r", 256 * 1024);
+ this.file = new BufferedRandomAccessFile(sstable.getFilename(), "r", bufferSize);
this.sstable = sstable;
}
Modified: incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/tools/SSTableExport.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/tools/SSTableExport.java?rev=893584&r1=893583&r2=893584&view=diff
==============================================================================
--- incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/tools/SSTableExport.java (original)
+++ incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/tools/SSTableExport.java Wed Dec 23 17:45:58 2009
@@ -40,6 +40,8 @@
*/
public class SSTableExport
{
+ private static int INPUT_FILE_BUFFER_SIZE = 8 * 1024 * 1024;
+
private static final String OUTFILE_OPTION = "f";
private static final String KEY_OPTION = "k";
private static Options options;
@@ -143,7 +145,7 @@
throws IOException
{
SSTableReader reader = SSTableReader.open(ssTableFile);
- SSTableScanner scanner = reader.getScanner();
+ SSTableScanner scanner = reader.getScanner(INPUT_FILE_BUFFER_SIZE);
IPartitioner<?> partitioner = DatabaseDescriptor.getPartitioner();
int i = 0;
@@ -201,7 +203,7 @@
// than once from within the same process.
static void export(SSTableReader reader, PrintStream outs) throws IOException
{
- SSTableScanner scanner = reader.getScanner();
+ SSTableScanner scanner = reader.getScanner(INPUT_FILE_BUFFER_SIZE);
outs.println("{");
@@ -321,4 +323,4 @@
}
System.exit(0);
}
-}
\ No newline at end of file
+}