You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2010/07/12 01:07:51 UTC
svn commit: r963159 - in /cassandra/trunk: conf/
src/java/org/apache/cassandra/config/ src/java/org/apache/cassandra/db/
src/java/org/apache/cassandra/io/sstable/
src/java/org/apache/cassandra/io/util/ src/java/org/apache/cassandra/tools/
test/ test/co...
Author: jbellis
Date: Sun Jul 11 23:07:50 2010
New Revision: 963159
URL: http://svn.apache.org/viewvc?rev=963159&view=rev
Log:
avoid double-copy of flushed data by serializing directly to output file. patch by jbellis; reviewed by mdennis for CASSANDRA-270
Modified:
cassandra/trunk/conf/cassandra.yaml
cassandra/trunk/src/java/org/apache/cassandra/config/Config.java
cassandra/trunk/src/java/org/apache/cassandra/config/Converter.java
cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
cassandra/trunk/src/java/org/apache/cassandra/io/util/BufferedRandomAccessFile.java
cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java
cassandra/trunk/test/cassandra.in.sh
cassandra/trunk/test/conf/cassandra.yaml
cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java
cassandra/trunk/test/unit/org/apache/cassandra/tools/SSTableExportTest.java
Modified: cassandra/trunk/conf/cassandra.yaml
URL: http://svn.apache.org/viewvc/cassandra/trunk/conf/cassandra.yaml?rev=963159&r1=963158&r2=963159&view=diff
==============================================================================
--- cassandra/trunk/conf/cassandra.yaml (original)
+++ cassandra/trunk/conf/cassandra.yaml Sun Jul 11 23:07:50 2010
@@ -95,14 +95,13 @@ memtable_flush_after_mins: 60
memtable_throughput_in_mb: 64
# Number of objects in millions in the memtable before it is flushed
memtable_operations_in_millions: 0.3
-# Buffer size to use when flushing !memtables to disk.
-flush_data_buffer_size_in_mb: 32
# Increase (decrease) the index buffer size relative to the data
# buffer if you have few (many) columns per key.
flush_index_buffer_size_in_mb: 8
column_index_size_in_kb: 64
-in_memory_compaction_limit_in_mb: 128
+
+in_memory_compaction_limit_in_mb: 64
# commit log
commitlog_directory: /var/lib/cassandra/commitlog
Modified: cassandra/trunk/src/java/org/apache/cassandra/config/Config.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/config/Config.java?rev=963159&r1=963158&r2=963159&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/config/Config.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/config/Config.java Sun Jul 11 23:07:50 2010
@@ -36,9 +36,6 @@ public class Config {
public Integer memtable_flush_writers = null; // will get set to the length of data dirs in DatabaseDescriptor
- public Double flush_data_buffer_size_in_mb = new Double(32);
- public Double flush_index_buffer_size_in_mb = new Double(8);
-
public Integer sliced_buffer_size_in_kb = 64;
public Integer storage_port = 7000;
Modified: cassandra/trunk/src/java/org/apache/cassandra/config/Converter.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/config/Converter.java?rev=963159&r1=963158&r2=963159&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/config/Converter.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/config/Converter.java Sun Jul 11 23:07:50 2010
@@ -41,7 +41,7 @@ public class Converter {
int size = tablesxml.getLength();
for ( int i = 0; i < size; ++i )
{
- String value = null;
+ String value;
Keyspace ks = new Keyspace();
Node table = tablesxml.item(i);
/* parsing out the table ksName */
@@ -158,18 +158,6 @@ public class Converter {
conf.concurrent_writes = Integer.parseInt(rawWriters);
}
- String rawFlushData = xmlUtils.getNodeValue("/Storage/FlushDataBufferSizeInMB");
- if (rawFlushData != null)
- {
- conf.flush_data_buffer_size_in_mb = Double.parseDouble(rawFlushData);
- }
-
- String rawFlushIndex = xmlUtils.getNodeValue("/Storage/FlushIndexBufferSizeInMB");
- if (rawFlushIndex != null)
- {
- conf.flush_index_buffer_size_in_mb = Double.parseDouble(rawFlushIndex);
- }
-
String rawSlicedBuffer = xmlUtils.getNodeValue("/Storage/SlicedBufferSizeInKB");
if (rawSlicedBuffer != null)
{
@@ -287,7 +275,7 @@ public class Converter {
{
try
{
- String configname = null;
+ String configname;
ClassLoader loader = Converter.class.getClassLoader();
URL scpurl = loader.getResource(PREVIOUS_CONF_FILE);
if (scpurl == null)
Modified: cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java?rev=963159&r1=963158&r2=963159&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java Sun Jul 11 23:07:50 2010
@@ -935,7 +935,7 @@ public class DatabaseDescriptor
return conf.memtable_flush_writers;
}
- public static long getInMemoryCompactionLimit()
+ public static int getInMemoryCompactionLimit()
{
return conf.in_memory_compaction_limit_in_mb * 1024 * 1024;
}
@@ -1126,16 +1126,6 @@ public class DatabaseDescriptor
return indexAccessMode;
}
- public static double getFlushDataBufferSizeInMB()
- {
- return conf.flush_data_buffer_size_in_mb;
- }
-
- public static double getFlushIndexBufferSizeInMB()
- {
- return conf.flush_index_buffer_size_in_mb;
- }
-
public static int getIndexedReadBufferSizeInKB()
{
return conf.column_index_size_in_kb;
Modified: cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java?rev=963159&r1=963158&r2=963159&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java Sun Jul 11 23:07:50 2010
@@ -150,15 +150,8 @@ public class Memtable implements Compara
logger.info("Writing " + this);
SSTableWriter writer = new SSTableWriter(cfs.getFlushPath(), columnFamilies.size(), partitioner);
- DataOutputBuffer buffer = new DataOutputBuffer();
for (Map.Entry<DecoratedKey, ColumnFamily> entry : columnFamilies.entrySet())
- {
- buffer.reset();
- /* serialize the cf with column indexes */
- ColumnFamily.serializer().serializeWithIndexes(entry.getValue(), buffer);
- /* Now write the key and value to disk */
- writer.append(entry.getKey(), buffer);
- }
+ writer.append(entry.getKey(), entry.getValue());
SSTableReader ssTable = writer.closeAndOpenReader();
logger.info("Completed flushing " + ssTable.getFilename());
Modified: cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java?rev=963159&r1=963158&r2=963159&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java Sun Jul 11 23:07:50 2010
@@ -43,11 +43,11 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamily;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.io.AbstractCompactedRow;
import org.apache.cassandra.io.util.BufferedRandomAccessFile;
-import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.io.util.SegmentedFile;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.BloomFilter;
@@ -67,7 +67,7 @@ public class SSTableWriter extends SSTab
super(filename, partitioner);
iwriter = new IndexWriter(desc, partitioner, keyCount);
dbuilder = SegmentedFile.getBuilder();
- dataFile = new BufferedRandomAccessFile(getFilename(), "rw", (int)(DatabaseDescriptor.getFlushDataBufferSizeInMB() * 1024 * 1024));
+ dataFile = new BufferedRandomAccessFile(getFilename(), "rw", DatabaseDescriptor.getInMemoryCompactionLimit());
}
private long beforeAppend(DecoratedKey decoratedKey) throws IOException
@@ -104,16 +104,22 @@ public class SSTableWriter extends SSTab
afterAppend(row.key, currentPosition);
}
- // TODO make this take a DataOutputStream and wrap the byte[] version to combine them
- public void append(DecoratedKey decoratedKey, DataOutputBuffer buffer) throws IOException
+ public void append(DecoratedKey decoratedKey, ColumnFamily cf) throws IOException
{
- long currentPosition = beforeAppend(decoratedKey);
+ long startPosition = beforeAppend(decoratedKey);
FBUtilities.writeShortByteArray(partitioner.convertToDiskFormat(decoratedKey), dataFile);
- int length = buffer.getLength();
- assert length > 0;
- dataFile.writeLong(length);
- dataFile.write(buffer.getData(), 0, length);
- afterAppend(decoratedKey, currentPosition);
+ // write placeholder for the row size, since we don't know it yet
+ long sizePosition = dataFile.getFilePointer();
+ dataFile.writeLong(-1);
+ // write out row data
+ ColumnFamily.serializer().serializeWithIndexes(cf, dataFile);
+ // seek back and write the row size (not including the size Long itself)
+ long endPosition = dataFile.getFilePointer();
+ dataFile.seek(sizePosition);
+ dataFile.writeLong(endPosition - (sizePosition + 8));
+ // finally, reset for next row
+ dataFile.seek(endPosition);
+ afterAppend(decoratedKey, startPosition);
}
public void append(DecoratedKey decoratedKey, byte[] value) throws IOException
@@ -209,7 +215,7 @@ public class SSTableWriter extends SSTab
ffile.delete();
// open the data file for input, and an IndexWriter for output
- BufferedRandomAccessFile dfile = new BufferedRandomAccessFile(desc.filenameFor(SSTable.COMPONENT_DATA), "r", (int)(DatabaseDescriptor.getFlushDataBufferSizeInMB() * 1024 * 1024));
+ BufferedRandomAccessFile dfile = new BufferedRandomAccessFile(desc.filenameFor(SSTable.COMPONENT_DATA), "r", 8 * 1024 * 1024);
IndexWriter iwriter;
long estimatedRows;
try
@@ -285,8 +291,7 @@ public class SSTableWriter extends SSTab
{
this.desc = desc;
this.partitioner = part;
- int bufferbytes = (int)(DatabaseDescriptor.getFlushIndexBufferSizeInMB() * 1024 * 1024);
- indexFile = new BufferedRandomAccessFile(desc.filenameFor(SSTable.COMPONENT_INDEX), "rw", bufferbytes);
+ indexFile = new BufferedRandomAccessFile(desc.filenameFor(SSTable.COMPONENT_INDEX), "rw", 8 * 1024 * 1024);
builder = SegmentedFile.getBuilder();
summary = new IndexSummary();
bf = BloomFilter.getFilter(keyCount, 15);
Modified: cassandra/trunk/src/java/org/apache/cassandra/io/util/BufferedRandomAccessFile.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/util/BufferedRandomAccessFile.java?rev=963159&r1=963158&r2=963159&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/util/BufferedRandomAccessFile.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/util/BufferedRandomAccessFile.java Sun Jul 11 23:07:50 2010
@@ -40,7 +40,6 @@ public class BufferedRandomAccessFile ex
{
static final int LogBuffSz_ = 16; // 64K buffer
public static final int BuffSz_ = (1 << LogBuffSz_);
- static final long BuffMask_ = ~(((long) BuffSz_) - 1L);
private String path_;
@@ -84,22 +83,22 @@ public class BufferedRandomAccessFile ex
*
* V3. Any (possibly) unflushed characters are stored in "f.buff":
*
- * (forall i in [f.lo, f.curr): c(f)[i] == f.buff[i - f.lo])
+ * (forall i in [f.lo, f.hi): c(f)[i] == f.buff[i - f.lo])
*
* V4. For all characters not covered by V3, c(f) and disk(f) agree:
*
- * (forall i in [f.lo, len(f)): i not in [f.lo, f.curr) => c(f)[i] ==
+ * (forall i in [f.lo, len(f)): i not in [f.lo, f.hi) => c(f)[i] ==
* disk(f)[i])
*
* V5. "f.dirty" is true iff the buffer contains bytes that should be
* flushed to the file; by V3 and V4, only part of the buffer can be dirty.
*
- * f.dirty == (exists i in [f.lo, f.curr): c(f)[i] != f.buff[i - f.lo])
+ * f.dirty == (exists i in [f.lo, f.hi): c(f)[i] != f.buff[i - f.lo])
*
* V6. this.maxHi == this.lo + this.buff.length
*
* Note that "f.buff" can be "null" in a valid file, since the range of
- * characters in V3 is empty when "f.lo == f.curr".
+ * characters in V3 is empty when "f.lo == f.hi".
*
* A file is said to be *ready* if the buffer contains the current position,
* i.e., when:
@@ -189,9 +188,9 @@ public class BufferedRandomAccessFile ex
{
if (this.diskPos_ != this.lo_)
super.seek(this.lo_);
- int len = (int) (this.curr_ - this.lo_);
+ int len = (int) (this.hi_ - this.lo_);
super.write(this.buff_, 0, len);
- this.diskPos_ = this.curr_;
+ this.diskPos_ = this.hi_;
this.dirty_ = false;
}
}
@@ -203,60 +202,43 @@ public class BufferedRandomAccessFile ex
*/
private int fillBuffer() throws IOException
{
- int cnt = 0;
- int rem = this.buff_.length;
- while (rem > 0)
+ int count = 0;
+ int remainder = this.buff_.length;
+ while (remainder > 0)
{
- int n = super.read(this.buff_, cnt, rem);
+ int n = super.read(this.buff_, count, remainder);
if (n < 0)
break;
- cnt += n;
- rem -= n;
+ count += n;
+ remainder -= n;
}
- if ( (cnt < 0) && (this.hitEOF_ = (cnt < this.buff_.length)) )
- {
- // make sure buffer that wasn't read is initialized with -1
- Arrays.fill(this.buff_, cnt, this.buff_.length, (byte) 0xff);
- }
- this.diskPos_ += cnt;
- return cnt;
+ this.hitEOF_ = (count < this.buff_.length);
+ this.diskPos_ += count;
+ return count;
}
+ public void seek(long pos) throws IOException
+ {
+ this.curr_ = pos;
+ }
+
/*
- * This method positions <code>this.curr</code> at position <code>pos</code>.
- * If <code>pos</code> does not fall in the current buffer, it flushes the
- * current buffer and loads the correct one.<p>
- *
* On exit from this routine <code>this.curr == this.hi</code> iff <code>pos</code>
* is at or past the end-of-file, which can only happen if the file was
* opened in read-only mode.
*/
- public void seek(long pos) throws IOException
+ private void reBuffer() throws IOException
{
- if (pos >= this.hi_ || pos < this.lo_)
+ this.flushBuffer();
+ this.lo_ = this.curr_;
+ this.maxHi_ = this.lo_ + (long) this.buff_.length;
+ if (this.diskPos_ != this.lo_)
{
- // seeking outside of current buffer -- flush and read
- this.flushBuffer();
- this.lo_ = pos & BuffMask_; // start at BuffSz boundary
- this.maxHi_ = this.lo_ + (long) this.buff_.length;
- if (this.diskPos_ != this.lo_)
- {
- super.seek(this.lo_);
- this.diskPos_ = this.lo_;
- }
- int n = this.fillBuffer();
- this.hi_ = this.lo_ + (long) n;
+ super.seek(this.lo_);
+ this.diskPos_ = this.lo_;
}
- else
- {
- // seeking inside current buffer -- no read required
- if (pos < this.curr_)
- {
- // if seeking backwards, we must flush to maintain V4
- this.flushBuffer();
- }
- }
- this.curr_ = pos;
+ int n = this.fillBuffer();
+ this.hi_ = this.lo_ + (long) n;
}
public long getFilePointer()
@@ -280,16 +262,10 @@ public class BufferedRandomAccessFile ex
public int read() throws IOException
{
- if (this.curr_ >= this.hi_)
+ if (this.lo_ > this.curr_ || this.curr_ >= this.hi_)
{
- // test for EOF
- // if (this.hi < this.maxHi) return -1;
- if (this.hitEOF_)
- return -1;
-
- // slow path -- read another buffer
- this.seek(this.curr_);
- if (this.curr_ == this.hi_)
+ this.reBuffer();
+ if (this.curr_ == this.hi_ && this.hitEOF_)
return -1;
}
byte res = this.buff_[(int) (this.curr_ - this.lo_)];
@@ -304,16 +280,10 @@ public class BufferedRandomAccessFile ex
public int read(byte[] b, int off, int len) throws IOException
{
- if (this.curr_ >= this.hi_)
+ if (this.lo_ > this.curr_ || this.curr_ >= this.hi_)
{
- // test for EOF
- // if (this.hi < this.maxHi) return -1;
- if (this.hitEOF_)
- return -1;
-
- // slow path -- read another buffer
- this.seek(this.curr_);
- if (this.curr_ == this.hi_)
+ this.reBuffer();
+ if (this.curr_ == this.hi_ && this.hitEOF_)
return -1;
}
len = Math.min(len, (int) (this.hi_ - this.curr_));
@@ -325,26 +295,14 @@ public class BufferedRandomAccessFile ex
public void write(int b) throws IOException
{
- if (this.curr_ >= this.hi_)
+ if (this.lo_ > this.curr_ || this.curr_ > this.hi_ || this.curr_ >= maxHi_)
{
- if (this.hitEOF_ && this.hi_ < this.maxHi_)
- {
- // at EOF -- bump "hi"
- this.hi_++;
- }
- else
- {
- // slow path -- write current buffer; read next one
- this.seek(this.curr_);
- if (this.curr_ == this.hi_)
- {
- // appending to EOF -- bump "hi"
- this.hi_++;
- }
- }
+ this.reBuffer();
}
this.buff_[(int) (this.curr_ - this.lo_)] = (byte) b;
this.curr_++;
+ if (this.curr_ > this.hi_)
+ this.hi_ = this.curr_;
this.dirty_ = true;
syncNeeded_ = true;
}
@@ -368,32 +326,20 @@ public class BufferedRandomAccessFile ex
/*
* Write at most "len" bytes to "b" starting at position "off", and return
- * the number of bytes written.
+ * the number of bytes written. caller is responsible for setting dirty, syncNeeded.
*/
private int writeAtMost(byte[] b, int off, int len) throws IOException
- {
- if (this.curr_ >= this.hi_)
+ {
+ if (this.lo_ > this.curr_ || this.curr_ > this.hi_ || this.curr_ >= maxHi_)
{
- if (this.hitEOF_ && this.hi_ < this.maxHi_)
- {
- // at EOF -- bump "hi"
- this.hi_ = this.maxHi_;
- }
- else
- {
- // slow path -- write current buffer; read next one
- this.seek(this.curr_);
- if (this.curr_ == this.hi_)
- {
- // appending to EOF -- bump "hi"
- this.hi_ = this.maxHi_;
- }
- }
+ this.reBuffer();
}
- len = Math.min(len, (int) (this.hi_ - this.curr_));
+ len = Math.min(len, (int) (this.maxHi_ - this.curr_));
int buffOff = (int) (this.curr_ - this.lo_);
System.arraycopy(b, off, this.buff_, buffOff, len);
this.curr_ += len;
+ if (this.curr_ > this.hi_)
+ this.hi_ = this.curr_;
return len;
}
Modified: cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java?rev=963159&r1=963158&r2=963159&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java Sun Jul 11 23:07:50 2010
@@ -154,8 +154,7 @@ public class SSTableImport
ColumnFamily cfamily = ColumnFamily.create(keyspace, cf);
ColumnFamilyType cfType = cfamily.getColumnFamilyType(); // Super or Standard
IPartitioner<?> partitioner = DatabaseDescriptor.getPartitioner();
- DataOutputBuffer dob = new DataOutputBuffer();
-
+
try
{
JSONObject json = (JSONObject)JSONValue.parseWithException(new FileReader(jsonFile));
@@ -174,9 +173,7 @@ public class SSTableImport
else
addToStandardCF((JSONArray)json.get(rowKey.getValue()), cfamily);
- ColumnFamily.serializer().serializeWithIndexes(cfamily, dob);
- writer.append(rowKey.getKey(), dob);
- dob.reset();
+ writer.append(rowKey.getKey(), cfamily);
cfamily.clear();
}
Modified: cassandra/trunk/test/cassandra.in.sh
URL: http://svn.apache.org/viewvc/cassandra/trunk/test/cassandra.in.sh?rev=963159&r1=963158&r2=963159&view=diff
==============================================================================
--- cassandra/trunk/test/cassandra.in.sh (original)
+++ cassandra/trunk/test/cassandra.in.sh Sun Jul 11 23:07:50 2010
@@ -38,6 +38,7 @@ JVM_OPTS=" \
-Xrunjdwp:transport=dt_socket,server=y,address=8898,suspend=n \
-Xms128M \
-Xmx1G \
+ -Xss128k \
-XX:SurvivorRatio=8 \
-XX:TargetSurvivorRatio=90 \
-XX:+AggressiveOpts \
Modified: cassandra/trunk/test/conf/cassandra.yaml
URL: http://svn.apache.org/viewvc/cassandra/trunk/test/conf/cassandra.yaml?rev=963159&r1=963158&r2=963159&view=diff
==============================================================================
--- cassandra/trunk/test/conf/cassandra.yaml (original)
+++ cassandra/trunk/test/conf/cassandra.yaml Sun Jul 11 23:07:50 2010
@@ -1,6 +1,5 @@
cluster_name: Test Cluster
-flush_data_buffer_size_in_mb: 1
-flush_index_buffer_size_in_mb: 8
+in_memory_compaction_limit_in_mb: 1
commitlog_sync: batch
commitlog_sync_batch_window_in_ms: 1.0
partitioner: org.apache.cassandra.dht.CollatingOrderPreservingPartitioner
Modified: cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java?rev=963159&r1=963158&r2=963159&view=diff
==============================================================================
--- cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java (original)
+++ cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java Sun Jul 11 23:07:50 2010
@@ -85,28 +85,16 @@ public class SSTableUtils
public static SSTableReader writeRawSSTable(String tablename, String cfname, Map<byte[], byte[]> entries) throws IOException
{
- return writeRawSSTable(null, tablename, cfname, entries);
- }
-
- public static SSTableReader writeRawSSTable(File datafile, String tablename, String cfname, Map<byte[], byte[]> entries) throws IOException
- {
- boolean temporary = false;
- if (datafile == null)
- {
- datafile = tempSSTableFile(tablename, cfname);
- temporary = true;
- }
+ File datafile = tempSSTableFile(tablename, cfname);
SSTableWriter writer = new SSTableWriter(datafile.getAbsolutePath(), entries.size(), StorageService.getPartitioner());
SortedMap<DecoratedKey, byte[]> sortedEntries = new TreeMap<DecoratedKey, byte[]>();
for (Map.Entry<byte[], byte[]> entry : entries.entrySet())
sortedEntries.put(writer.partitioner.decorateKey(entry.getKey()), entry.getValue());
for (Map.Entry<DecoratedKey, byte[]> entry : sortedEntries.entrySet())
writer.append(entry.getKey(), entry.getValue());
- if (temporary)
- {
- new File(writer.indexFilename()).deleteOnExit();
- new File(writer.filterFilename()).deleteOnExit();
- }
+ new File(writer.indexFilename()).deleteOnExit();
+ new File(writer.filterFilename()).deleteOnExit();
return writer.closeAndOpenReader();
}
+
}
Modified: cassandra/trunk/test/unit/org/apache/cassandra/tools/SSTableExportTest.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/tools/SSTableExportTest.java?rev=963159&r1=963158&r2=963159&view=diff
==============================================================================
--- cassandra/trunk/test/unit/org/apache/cassandra/tools/SSTableExportTest.java (original)
+++ cassandra/trunk/test/unit/org/apache/cassandra/tools/SSTableExportTest.java Sun Jul 11 23:07:50 2010
@@ -60,21 +60,16 @@ public class SSTableExportTest extends S
File tempSS = tempSSTableFile("Keyspace1", "Standard1");
ColumnFamily cfamily = ColumnFamily.create("Keyspace1", "Standard1");
IPartitioner<?> partitioner = DatabaseDescriptor.getPartitioner();
- DataOutputBuffer dob = new DataOutputBuffer();
SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2, partitioner);
// Add rowA
cfamily.addColumn(new QueryPath("Standard1", null, "colA".getBytes()), "valA".getBytes(), new TimestampClock(1));
- ColumnFamily.serializer().serializeWithIndexes(cfamily, dob);
- writer.append(Util.dk("rowA"), dob);
- dob.reset();
+ writer.append(Util.dk("rowA"), cfamily);
cfamily.clear();
// Add rowB
cfamily.addColumn(new QueryPath("Standard1", null, "colB".getBytes()), "valB".getBytes(), new TimestampClock(1));
- ColumnFamily.serializer().serializeWithIndexes(cfamily, dob);
- writer.append(Util.dk("rowB"), dob);
- dob.reset();
+ writer.append(Util.dk("rowB"), cfamily);
cfamily.clear();
writer.closeAndOpenReader();
@@ -98,28 +93,21 @@ public class SSTableExportTest extends S
File tempSS = tempSSTableFile("Keyspace1", "Standard1");
ColumnFamily cfamily = ColumnFamily.create("Keyspace1", "Standard1");
IPartitioner<?> partitioner = DatabaseDescriptor.getPartitioner();
- DataOutputBuffer dob = new DataOutputBuffer();
SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2, partitioner);
// Add rowA
cfamily.addColumn(new QueryPath("Standard1", null, "colA".getBytes()), "valA".getBytes(), new TimestampClock(1));
- ColumnFamily.serializer().serializeWithIndexes(cfamily, dob);
- writer.append(Util.dk("rowA"), dob);
- dob.reset();
+ writer.append(Util.dk("rowA"), cfamily);
cfamily.clear();
// Add rowB
cfamily.addColumn(new QueryPath("Standard1", null, "colB".getBytes()), "valB".getBytes(), new TimestampClock(1));
- ColumnFamily.serializer().serializeWithIndexes(cfamily, dob);
- writer.append(Util.dk("rowB"), dob);
- dob.reset();
+ writer.append(Util.dk("rowB"), cfamily);
cfamily.clear();
// Add rowExclude
cfamily.addColumn(new QueryPath("Standard1", null, "colX".getBytes()), "valX".getBytes(), new TimestampClock(1));
- ColumnFamily.serializer().serializeWithIndexes(cfamily, dob);
- writer.append(Util.dk("rowExclude"), dob);
- dob.reset();
+ writer.append(Util.dk("rowExclude"), cfamily);
cfamily.clear();
SSTableReader reader = writer.closeAndOpenReader();
@@ -148,28 +136,21 @@ public class SSTableExportTest extends S
File tempSS = tempSSTableFile("Keyspace1", "Super4");
ColumnFamily cfamily = ColumnFamily.create("Keyspace1", "Super4");
IPartitioner<?> partitioner = DatabaseDescriptor.getPartitioner();
- DataOutputBuffer dob = new DataOutputBuffer();
SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2, partitioner);
// Add rowA
cfamily.addColumn(new QueryPath("Super4", "superA".getBytes(), "colA".getBytes()), "valA".getBytes(), new TimestampClock(1));
- ColumnFamily.serializer().serializeWithIndexes(cfamily, dob);
- writer.append(Util.dk("rowA"), dob);
- dob.reset();
+ writer.append(Util.dk("rowA"), cfamily);
cfamily.clear();
// Add rowB
cfamily.addColumn(new QueryPath("Super4", "superB".getBytes(), "colB".getBytes()), "valB".getBytes(), new TimestampClock(1));
- ColumnFamily.serializer().serializeWithIndexes(cfamily, dob);
- writer.append(Util.dk("rowB"), dob);
- dob.reset();
+ writer.append(Util.dk("rowB"), cfamily);
cfamily.clear();
// Add rowExclude
cfamily.addColumn(new QueryPath("Super4", "superX".getBytes(), "colX".getBytes()), "valX".getBytes(), new TimestampClock(1));
- ColumnFamily.serializer().serializeWithIndexes(cfamily, dob);
- writer.append(Util.dk("rowExclude"), dob);
- dob.reset();
+ writer.append(Util.dk("rowExclude"), cfamily);
cfamily.clear();
SSTableReader reader = writer.closeAndOpenReader();
@@ -196,21 +177,16 @@ public class SSTableExportTest extends S
File tempSS = tempSSTableFile("Keyspace1", "Standard1");
ColumnFamily cfamily = ColumnFamily.create("Keyspace1", "Standard1");
IPartitioner<?> partitioner = DatabaseDescriptor.getPartitioner();
- DataOutputBuffer dob = new DataOutputBuffer();
SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2, partitioner);
// Add rowA
cfamily.addColumn(new QueryPath("Standard1", null, "name".getBytes()), "val".getBytes(), new TimestampClock(1));
- ColumnFamily.serializer().serializeWithIndexes(cfamily, dob);
- writer.append(Util.dk("rowA"), dob);
- dob.reset();
+ writer.append(Util.dk("rowA"), cfamily);
cfamily.clear();
// Add rowExclude
cfamily.addColumn(new QueryPath("Standard1", null, "name".getBytes()), "val".getBytes(), new TimestampClock(1));
- ColumnFamily.serializer().serializeWithIndexes(cfamily, dob);
- writer.append(Util.dk("rowExclude"), dob);
- dob.reset();
+ writer.append(Util.dk("rowExclude"), cfamily);
cfamily.clear();
SSTableReader reader = writer.closeAndOpenReader();