You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2009/08/07 18:24:26 UTC
svn commit: r802075 - in
/incubator/cassandra/trunk/src/java/org/apache/cassandra:
db/ColumnFamily.java db/filter/SSTableNamesIterator.java
db/filter/SSTableSliceIterator.java io/IndexHelper.java
service/CassandraServer.java
Author: jbellis
Date: Fri Aug 7 16:24:25 2009
New Revision: 802075
URL: http://svn.apache.org/viewvc?rev=802075&view=rev
Log:
final cleanup of SSTableSliceIterator, now with less unnecessary copying of Columns around
patch by jbellis; reviewed by Jun Rao for CASSANDRA-332
Modified:
incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamily.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableNamesIterator.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableSliceIterator.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/io/IndexHelper.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/service/CassandraServer.java
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamily.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamily.java?rev=802075&r1=802074&r2=802075&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamily.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamily.java Fri Aug 7 16:24:25 2009
@@ -481,8 +481,6 @@
*/
public void serialize(ColumnFamily columnFamily, DataOutput dos) throws IOException
{
- // TODO whenever we change this we need to change the code in SSTableSliceIterator to match.
- // This SUCKS and is inefficient to boot. let's fix this ASAP.
Collection<IColumn> columns = columnFamily.getSortedColumns();
dos.writeUTF(columnFamily.name());
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableNamesIterator.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableNamesIterator.java?rev=802075&r1=802074&r2=802075&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableNamesIterator.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableNamesIterator.java Fri Aug 7 16:24:25 2009
@@ -49,7 +49,7 @@
SortedSet<IndexHelper.IndexInfo> ranges = new TreeSet<IndexHelper.IndexInfo>(IndexHelper.getComparator(comparator));
for (byte[] name : columns)
{
- ranges.add(indexList.get(IndexHelper.indexFor(name, indexList, comparator)));
+ ranges.add(indexList.get(IndexHelper.indexFor(name, indexList, comparator, false)));
}
/* seek to the correct offset to the data */
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableSliceIterator.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableSliceIterator.java?rev=802075&r1=802074&r2=802075&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableSliceIterator.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableSliceIterator.java Fri Aug 7 16:24:25 2009
@@ -14,16 +14,15 @@
*/
class SSTableSliceIterator extends AbstractIterator<IColumn> implements ColumnIterator
{
- protected boolean isAscending;
- private byte[] startColumn;
- private int curColumnIndex;
- private ArrayList<IColumn> curColumns = new ArrayList<IColumn>();
+ private final boolean isAscending;
+ private final byte[] startColumn;
+ private final AbstractType comparator;
private ColumnGroupReader reader;
- private AbstractType comparator;
public SSTableSliceIterator(String filename, String key, AbstractType comparator, byte[] startColumn, boolean isAscending)
throws IOException
{
+ // TODO push finishColumn down here too, so we can tell when we're done and optimize away the slice when the index + start/stop shows there's nothing to scan for
this.isAscending = isAscending;
SSTableReader ssTable = SSTableReader.open(filename);
@@ -34,45 +33,13 @@
this.startColumn = startColumn;
if (position >= 0)
reader = new ColumnGroupReader(filename, decoratedKey, position);
- curColumnIndex = isAscending ? 0 : -1;
}
private boolean isColumnNeeded(IColumn column)
{
- if (isAscending)
- {
- return comparator.compare(column.name(), startColumn) >= 0;
- }
- else
- {
- if (startColumn.length == 0)
- {
- /* assuming scanning from the largest column in descending order */
- return true;
- }
- else
- {
- return comparator.compare(column.name(), startColumn) <= 0;
- }
- }
- }
-
- private void getColumnsFromBuffer() throws IOException
- {
- curColumns.clear();
- while (true)
- {
- IColumn column = reader.pollColumn();
- if (column == null)
- break;
- if (isColumnNeeded(column))
- curColumns.add(column);
- }
-
- if (isAscending)
- curColumnIndex = 0;
- else
- curColumnIndex = curColumns.size() - 1;
+ return isAscending
+ ? comparator.compare(column.name(), startColumn) >= 0
+ : startColumn.length == 0 || comparator.compare(column.name(), startColumn) <= 0;
}
public ColumnFamily getColumnFamily()
@@ -87,31 +54,11 @@
while (true)
{
- if (isAscending)
- {
- if (curColumnIndex < curColumns.size())
- {
- return curColumns.get(curColumnIndex++);
- }
- }
- else
- {
- if (curColumnIndex >= 0)
- {
- return curColumns.get(curColumnIndex--);
- }
- }
-
- try
- {
- if (!reader.getNextBlock())
- return endOfData();
- getColumnsFromBuffer();
- }
- catch (IOException e)
- {
- throw new RuntimeException(e);
- }
+ IColumn column = reader.pollColumn();
+ if (column == null)
+ return endOfData();
+ if (isColumnNeeded(column))
+ return column;
}
}
@@ -128,13 +75,14 @@
*/
class ColumnGroupReader
{
- private ColumnFamily emptyColumnFamily;
+ private final ColumnFamily emptyColumnFamily;
+
+ private final List<IndexHelper.IndexInfo> indexes;
+ private final long columnStartPosition;
+ private final BufferedRandomAccessFile file;
- private List<IndexHelper.IndexInfo> indexes;
- private long columnStartPosition;
private int curRangeIndex;
- private BufferedRandomAccessFile file;
- private Queue<IColumn> blockColumns = new ArrayDeque<IColumn>();
+ private Deque<IColumn> blockColumns = new ArrayDeque<IColumn>();
public ColumnGroupReader(String filename, String key, long position) throws IOException
{
@@ -148,23 +96,11 @@
IndexHelper.skipBloomFilter(file);
indexes = IndexHelper.deserializeIndex(file);
- /* need to do two things here.
- * 1. move the file pointer to the beginning of the list of stored columns
- * 2. calculate the size of all columns */
emptyColumnFamily = ColumnFamily.serializer().deserializeEmpty(file);
file.readInt(); // column count
columnStartPosition = file.getFilePointer();
-
- if (startColumn.length == 0 && !isAscending)
- {
- /* in this case, we assume that we want to scan from the largest column in descending order. */
- curRangeIndex = indexes.size() - 1;
- }
- else
- {
- curRangeIndex = IndexHelper.indexFor(startColumn, indexes, comparator);
- }
+ curRangeIndex = IndexHelper.indexFor(startColumn, indexes, comparator, isAscending);
}
public ColumnFamily getEmptyColumnFamily()
@@ -174,7 +110,20 @@
public IColumn pollColumn()
{
- return blockColumns.poll();
+ IColumn column = blockColumns.poll();
+ if (column == null)
+ {
+ try
+ {
+ if (getNextBlock())
+ column = blockColumns.poll();
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+ return column;
}
public boolean getNextBlock() throws IOException
@@ -187,7 +136,11 @@
file.seek(columnStartPosition + curColPostion.offset);
while (file.getFilePointer() < columnStartPosition + curColPostion.offset + curColPostion.width)
{
- blockColumns.add(emptyColumnFamily.getColumnSerializer().deserialize(file));
+ IColumn column = emptyColumnFamily.getColumnSerializer().deserialize(file);
+ if (isAscending)
+ blockColumns.addLast(column);
+ else
+ blockColumns.addFirst(column);
}
if (isAscending)
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/io/IndexHelper.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/io/IndexHelper.java?rev=802075&r1=802074&r2=802075&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/io/IndexHelper.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/io/IndexHelper.java Fri Aug 7 16:24:25 2009
@@ -120,8 +120,10 @@
return BloomFilter.serializer().deserialize(bufIn);
}
- public static int indexFor(byte[] name, List<IndexInfo> indexList, AbstractType comparator)
+ public static int indexFor(byte[] name, List<IndexInfo> indexList, AbstractType comparator, boolean ascending)
{
+ if (name.length == 0 && !ascending)
+ return indexList.size() - 1;
IndexInfo target = new IndexInfo(name, name, 0, 0);
int index = Collections.binarySearch(indexList, target, getComparator(comparator));
return index < 0 ? -1 * (index + 1) : index;
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/service/CassandraServer.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/service/CassandraServer.java?rev=802075&r1=802074&r2=802075&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/service/CassandraServer.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/service/CassandraServer.java Fri Aug 7 16:24:25 2009
@@ -132,6 +132,8 @@
thriftColumns.add(thrift_column);
}
+ // we have to do the reversing here, since internally we pass results around in ColumnFamily
+ // objects, which always sort their columns in the "natural" order
if (reverseOrder)
Collections.reverse(thriftColumns);
return thriftColumns;