You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2010/10/04 18:10:11 UTC
svn commit: r1004303 - in /cassandra/branches/cassandra-0.6: CHANGES.txt
src/java/org/apache/cassandra/io/IndexSummary.java
src/java/org/apache/cassandra/io/SSTableReader.java
Author: jbellis
Date: Mon Oct 4 16:10:11 2010
New Revision: 1004303
URL: http://svn.apache.org/viewvc?rev=1004303&view=rev
Log:
optimize SSTableReader.loadIndexFile to reduce re-reading of index data during startup. patch by jbellis and Stu Hood for CASSANDRA-1526
Modified:
cassandra/branches/cassandra-0.6/CHANGES.txt
cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/IndexSummary.java
cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/SSTableReader.java
Modified: cassandra/branches/cassandra-0.6/CHANGES.txt
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.6/CHANGES.txt?rev=1004303&r1=1004302&r2=1004303&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.6/CHANGES.txt (original)
+++ cassandra/branches/cassandra-0.6/CHANGES.txt Mon Oct 4 16:10:11 2010
@@ -39,6 +39,8 @@
* allow nodes to change IPs between restarts (CASSANDRA-1518)
* remove assertion causing rare (and harmless) error messages in
commitlog (CASSANDRA-1330)
+ * optimize SSTableReader.loadIndexFile to reduce re-reading of index
+ data during server startup (CASSANDRA-1526)
0.6.5
Modified: cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/IndexSummary.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/IndexSummary.java?rev=1004303&r1=1004302&r2=1004303&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/IndexSummary.java (original)
+++ cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/IndexSummary.java Mon Oct 4 16:10:11 2010
@@ -31,7 +31,7 @@ import org.apache.cassandra.db.Decorated
public class IndexSummary
{
- private ArrayList<KeyPosition> indexPositions;
+ private ArrayList<KeyPosition> indexPositions = new ArrayList<KeyPosition>();
private Map<KeyPosition, SSTable.PositionSize> spannedIndexDataPositions;
private Map<Long, KeyPosition> spannedIndexPositions;
private int keysWritten = 0;
@@ -43,10 +43,6 @@ public class IndexSummary
&& SSTableReader.bufferIndex(indexPosition) != SSTableReader.bufferIndex(nextIndexPosition);
if ((keysWritten++ % DatabaseDescriptor.getIndexInterval() == 0) || spannedIndexEntry)
{
- if (indexPositions == null)
- {
- indexPositions = new ArrayList<KeyPosition>();
- }
KeyPosition info = new KeyPosition(decoratedKey, indexPosition);
indexPositions.add(info);
Modified: cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/SSTableReader.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/SSTableReader.java?rev=1004303&r1=1004302&r2=1004303&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/SSTableReader.java (original)
+++ cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/SSTableReader.java Mon Oct 4 16:10:11 2010
@@ -19,26 +19,29 @@
package org.apache.cassandra.io;
import java.io.*;
-import java.util.*;
-import java.lang.ref.ReferenceQueue;
import java.lang.ref.Reference;
-import java.nio.channels.FileChannel;
+import java.lang.ref.ReferenceQueue;
import java.nio.MappedByteBuffer;
+import java.nio.channels.FileChannel;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
import org.apache.log4j.Logger;
import org.apache.cassandra.cache.InstrumentedCache;
-import org.apache.cassandra.dht.IPartitioner;
-import org.apache.cassandra.utils.BloomFilter;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.Pair;
-import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.io.util.BufferedRandomAccessFile;
import org.apache.cassandra.io.util.FileDataInput;
import org.apache.cassandra.io.util.MappedFileDataInput;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.BloomFilter;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.Pair;
/**
* SSTableReaders are open()ed by Table.onStart; after that they are created by SSTableWriter.renameAndOpen.
@@ -244,30 +247,36 @@ public class SSTableReader extends SSTab
try
{
long indexSize = input.length();
+ // we need to know both the current index entry and its data position, as well as the
+ // next such pair, in order to compute tne mmap-spanning entries. since seeking
+ // backwards in a 0.6 BRAF is expensive, we make one pass through by reading the "next"
+ // entry in each loop through, then summarizing the previous one.
+ IndexSummary.KeyPosition thisEntry = null, nextEntry = null;
+ long thisDataPos = -1, nextDataPos = -1;
while (true)
{
long indexPosition = input.getFilePointer();
if (indexPosition == indexSize)
- {
break;
- }
- DecoratedKey decoratedKey = partitioner.convertFromDiskFormat(input.readUTF());
+
+ DecoratedKey key = partitioner.convertFromDiskFormat(input.readUTF());
long dataPosition = input.readLong();
- long nextIndexPosition = input.getFilePointer();
- // read the next index entry to see how big the row is
- long nextDataPosition;
- if (input.isEOF())
+ if (thisEntry == null)
{
- nextDataPosition = length();
- }
- else
- {
- input.readUTF();
- nextDataPosition = input.readLong();
- input.seek(nextIndexPosition);
+ thisEntry = new IndexSummary.KeyPosition(key, indexPosition);
+ thisDataPos = dataPosition;
+ continue;
}
- indexSummary.maybeAddEntry(decoratedKey, dataPosition, nextDataPosition - dataPosition, indexPosition, nextIndexPosition);
+
+ nextEntry = new IndexSummary.KeyPosition(key, indexPosition);
+ nextDataPos = dataPosition;
+ indexSummary.maybeAddEntry(thisEntry.key, thisDataPos, nextDataPos - thisDataPos, thisEntry.indexPosition, nextEntry.indexPosition);
+
+ thisEntry = nextEntry;
+ thisDataPos = nextDataPos;
}
+ assert thisEntry != null; // should not have any zero-row sstables
+ indexSummary.maybeAddEntry(thisEntry.key, thisDataPos, length() - thisDataPos, thisEntry.indexPosition, input.length());
indexSummary.complete();
}
finally