You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2009/08/11 16:13:47 UTC
svn commit: r803135 - in /incubator/cassandra/trunk: conf/
src/java/org/apache/cassandra/config/ src/java/org/apache/cassandra/db/
src/java/org/apache/cassandra/db/filter/ src/java/org/apache/cassandra/io/
Author: jbellis
Date: Tue Aug 11 14:13:47 2009
New Revision: 803135
URL: http://svn.apache.org/viewvc?rev=803135&view=rev
Log:
add settings for buffer sizes; make unconfigurable ones more sane.
patch by jbellis; reviewed by Jun Rao and Eric Evans for CASSANDRA-355
Modified:
incubator/cassandra/trunk/conf/storage-conf.xml
incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/db/CommitLog.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableNamesIterator.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableSliceIterator.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/io/FileStruct.java
Modified: incubator/cassandra/trunk/conf/storage-conf.xml
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/conf/storage-conf.xml?rev=803135&r1=803134&r2=803135&view=diff
==============================================================================
--- incubator/cassandra/trunk/conf/storage-conf.xml (original)
+++ incubator/cassandra/trunk/conf/storage-conf.xml Tue Aug 11 14:13:47 2009
@@ -198,6 +198,12 @@
<!-- Memory, Disk, and Performance -->
<!--======================================================================-->
+ <!-- Buffer size to use when performing contiguous column slices.
+ Increase this to the size of the column slices you typically
+ perform. (Name-based queries are performed with a buffer size
+ of ColumnIndexSizeInKB.) -->
+ <SlicedBufferSizeInKB>64</SlicedBufferSizeInKB>
+
<!-- Buffer size to use when flushing memtables to disk.
(Only one memtable is ever flushed at a time.)
Increase (decrease) the index buffer size relative to the data buffer
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java?rev=803135&r1=803134&r2=803135&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java Tue Aug 11 14:13:47 2009
@@ -71,6 +71,7 @@
private static double flushDataBufferSizeInMB_ = 32;
private static double flushIndexBufferSizeInMB_ = 8;
+ private static int slicedReadBufferSizeInKB_ = 64;
private static List<String> tables_ = new ArrayList<String>();
private static Set<String> applicationColumnFamilies_ = new HashSet<String>();
@@ -239,6 +240,12 @@
flushIndexBufferSizeInMB_ = Double.parseDouble(rawFlushIndex);
}
+ String rawSlicedBuffer = xmlUtils.getNodeValue("/Storage/SlicedBufferSizeInKB");
+ if (rawSlicedBuffer != null)
+ {
+ slicedReadBufferSizeInKB_ = Integer.parseInt(rawSlicedBuffer);
+ }
+
/* TCP port on which the storage system listens */
String port = xmlUtils.getNodeValue("/Storage/StoragePort");
if ( port != null )
@@ -956,4 +963,14 @@
{
return flushIndexBufferSizeInMB_;
}
+
+ public static int getIndexedReadBufferSizeInKB()
+ {
+ return columnIndexSizeInKB_;
+ }
+
+ public static int getSlicedReadBufferSizeInKB()
+ {
+ return slicedReadBufferSizeInKB_;
+ }
}
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/CommitLog.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/CommitLog.java?rev=803135&r1=803134&r2=803135&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/CommitLog.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/CommitLog.java Tue Aug 11 14:13:47 2009
@@ -249,7 +249,8 @@
for (File file : clogs)
{
- BufferedRandomAccessFile reader = new BufferedRandomAccessFile(file.getAbsolutePath(), "r");
+ int bufferSize = (int)Math.min(file.length(), 32 * 1024 * 1024);
+ BufferedRandomAccessFile reader = new BufferedRandomAccessFile(file.getAbsolutePath(), "r", bufferSize);
CommitLogHeader clHeader = readCommitLogHeader(reader);
/* seek to the lowest position */
int lowPos = CommitLogHeader.getLowestPosition(clHeader);
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableNamesIterator.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableNamesIterator.java?rev=803135&r1=803134&r2=803135&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableNamesIterator.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableNamesIterator.java Tue Aug 11 14:13:47 2009
@@ -27,7 +27,7 @@
if (position < 0)
return;
- BufferedRandomAccessFile file = new BufferedRandomAccessFile(filename, "r");
+ BufferedRandomAccessFile file = new BufferedRandomAccessFile(filename, "r", DatabaseDescriptor.getIndexedReadBufferSizeInKB() * 1024);
try
{
file.seek(position);
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableSliceIterator.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableSliceIterator.java?rev=803135&r1=803134&r2=803135&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableSliceIterator.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableSliceIterator.java Tue Aug 11 14:13:47 2009
@@ -7,6 +7,7 @@
import org.apache.cassandra.db.ColumnFamily;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.io.*;
+import org.apache.cassandra.config.DatabaseDescriptor;
import com.google.common.collect.AbstractIterator;
/**
@@ -86,7 +87,7 @@
public ColumnGroupReader(String filename, String key, long position) throws IOException
{
- this.file = new BufferedRandomAccessFile(filename, "r");
+ this.file = new BufferedRandomAccessFile(filename, "r", DatabaseDescriptor.getSlicedReadBufferSizeInKB() * 1024);
file.seek(position);
String keyInDisk = file.readUTF();
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/io/FileStruct.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/io/FileStruct.java?rev=803135&r1=803134&r2=803135&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/io/FileStruct.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/io/FileStruct.java Tue Aug 11 14:13:47 2009
@@ -19,10 +19,12 @@
package org.apache.cassandra.io;
import java.io.IOException;
+import java.io.File;
import java.util.Iterator;
import org.apache.cassandra.db.IColumn;
import org.apache.cassandra.db.ColumnFamily;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.log4j.Logger;
import com.google.common.collect.AbstractIterator;
@@ -40,7 +42,11 @@
FileStruct(SSTableReader sstable) throws IOException
{
- this.file = new BufferedRandomAccessFile(sstable.getFilename(), "r", 1024 * 1024);
+ // TODO this is used for both compactions and key ranges. the buffer sizes we want
+ // to use for these ops are very different. here we are leaning towards the key-range
+ // use case since that is more common. What we really want is to split those
+ // two uses of this class up.
+ this.file = new BufferedRandomAccessFile(sstable.getFilename(), "r", 256 * 1024);
this.sstable = sstable;
}