You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2010/05/10 16:59:14 UTC
svn commit: r942768 - in /cassandra/trunk: ./
interface/thrift/gen-java/org/apache/cassandra/thrift/
src/java/org/apache/cassandra/io/sstable/
test/unit/org/apache/cassandra/io/sstable/
Author: jbellis
Date: Mon May 10 14:59:13 2010
New Revision: 942768
URL: http://svn.apache.org/viewvc?rev=942768&view=rev
Log:
merge from 0.6
Modified:
cassandra/trunk/ (props changed)
cassandra/trunk/CHANGES.txt
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java (props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java (props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java (props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java (props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java (props changed)
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/IndexSummary.java
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
Propchange: cassandra/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 10 14:59:13 2010
@@ -1,4 +1,4 @@
-/cassandra/branches/cassandra-0.6:922689-942206
+/cassandra/branches/cassandra-0.6:922689-942757
/incubator/cassandra/branches/cassandra-0.3:774578-796573
/incubator/cassandra/branches/cassandra-0.4:810145-834239,834349-834350
/incubator/cassandra/branches/cassandra-0.5:888872-915439
Modified: cassandra/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/cassandra/trunk/CHANGES.txt?rev=942768&r1=942767&r2=942768&view=diff
==============================================================================
--- cassandra/trunk/CHANGES.txt (original)
+++ cassandra/trunk/CHANGES.txt Mon May 10 14:59:13 2010
@@ -36,6 +36,7 @@ dev
(CASSANDRA-1060)
* disable GCInspector on non-Sun JVMs (CASSANDRA-1061)
* fix tombstone handling in sstable rows with no other data (CASSANDRA-1063)
+ * fix size of row in spanned index entries (CASSANDRA-1056)
0.6.1
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 10 14:59:13 2010
@@ -1,4 +1,4 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-942206
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-942757
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/Cassandra.java:774578-796573
/incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/Cassandra.java:810145-834239,834349-834350
/incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/Cassandra.java:888872-903502
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 10 14:59:13 2010
@@ -1,4 +1,4 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-942206
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-942757
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/column_t.java:774578-792198
/incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/Column.java:810145-834239,834349-834350
/incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/Column.java:888872-903502
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 10 14:59:13 2010
@@ -1,4 +1,4 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-942206
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-942757
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:774578-796573
/incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:810145-834239,834349-834350
/incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:888872-903502
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 10 14:59:13 2010
@@ -1,4 +1,4 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:922689-942206
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:922689-942757
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:774578-796573
/incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:810145-834239,834349-834350
/incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:888872-903502
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 10 14:59:13 2010
@@ -1,4 +1,4 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:922689-942206
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:922689-942757
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/superColumn_t.java:774578-792198
/incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/SuperColumn.java:810145-834239,834349-834350
/incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/SuperColumn.java:888872-903502
Modified: cassandra/trunk/src/java/org/apache/cassandra/io/sstable/IndexSummary.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/IndexSummary.java?rev=942768&r1=942767&r2=942768&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/IndexSummary.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/IndexSummary.java Mon May 10 14:59:13 2010
@@ -41,11 +41,11 @@ public class IndexSummary
private int keysWritten = 0;
private long lastIndexPosition;
- public void maybeAddEntry(DecoratedKey decoratedKey, long dataPosition, long dataSize, long indexPosition, long nextIndexPosition)
+ public void maybeAddEntry(DecoratedKey decoratedKey, long dataPosition, long rowSize, long indexPosition, long nextIndexPosition)
{
boolean spannedIndexEntry = DatabaseDescriptor.getIndexAccessMode() == Config.DiskAccessMode.mmap
&& RowIndexedReader.bufferIndex(indexPosition) != RowIndexedReader.bufferIndex(nextIndexPosition);
- if (keysWritten++ % INDEX_INTERVAL == 0 || spannedIndexEntry)
+ if ((keysWritten++ % INDEX_INTERVAL == 0) || spannedIndexEntry)
{
if (indexPositions == null)
{
@@ -61,13 +61,18 @@ public class IndexSummary
spannedIndexDataPositions = new HashMap<KeyPosition, SSTable.PositionSize>();
spannedIndexPositions = new HashMap<Long, KeyPosition>();
}
- spannedIndexDataPositions.put(info, new SSTable.PositionSize(dataPosition, dataSize));
+ spannedIndexDataPositions.put(info, new SSTable.PositionSize(dataPosition, rowSize));
spannedIndexPositions.put(info.indexPosition, info);
}
}
lastIndexPosition = indexPosition;
}
+ public Map<KeyPosition, SSTable.PositionSize> getSpannedIndexDataPositions()
+ {
+ return spannedIndexDataPositions;
+ }
+
public List<KeyPosition> getIndexPositions()
{
return indexPositions;
Modified: cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java?rev=942768&r1=942767&r2=942768&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java Mon May 10 14:59:13 2010
@@ -59,7 +59,6 @@ public class SSTableWriter extends SSTab
{
private static Logger logger = LoggerFactory.getLogger(SSTableWriter.class);
- private long keysWritten;
private BufferedRandomAccessFile dataFile;
private BufferedRandomAccessFile indexFile;
private DecoratedKey lastWrittenKey;
@@ -90,7 +89,7 @@ public class SSTableWriter extends SSTab
return (lastWrittenKey == null) ? 0 : dataFile.getFilePointer();
}
- private void afterAppend(DecoratedKey decoratedKey, long dataPosition, int dataSize) throws IOException
+ private void afterAppend(DecoratedKey decoratedKey, long dataPosition) throws IOException
{
byte[] diskKey = partitioner.convertToDiskFormat(decoratedKey);
bf.add(diskKey);
@@ -103,7 +102,8 @@ public class SSTableWriter extends SSTab
if (logger.isTraceEnabled())
logger.trace("wrote index of " + decoratedKey + " at " + indexPosition);
- indexSummary.maybeAddEntry(decoratedKey, dataPosition, dataSize, indexPosition, indexFile.getFilePointer());
+ int rowSize = (int)(dataFile.getFilePointer() - dataPosition);
+ indexSummary.maybeAddEntry(decoratedKey, dataPosition, rowSize, indexPosition, indexFile.getFilePointer());
}
// TODO make this take a DataOutputStream and wrap the byte[] version to combine them
@@ -115,7 +115,7 @@ public class SSTableWriter extends SSTab
assert length > 0;
dataFile.writeInt(length);
dataFile.write(buffer.getData(), 0, length);
- afterAppend(decoratedKey, currentPosition, length);
+ afterAppend(decoratedKey, currentPosition);
}
public void append(DecoratedKey decoratedKey, byte[] value) throws IOException
@@ -125,7 +125,7 @@ public class SSTableWriter extends SSTab
assert value.length > 0;
dataFile.writeInt(value.length);
dataFile.write(value);
- afterAppend(decoratedKey, currentPosition, value.length);
+ afterAppend(decoratedKey, currentPosition);
}
public SSTableReader closeAndOpenReader() throws IOException
Modified: cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java?rev=942768&r1=942767&r2=942768&view=diff
==============================================================================
--- cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java (original)
+++ cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java Mon May 10 14:59:13 2010
@@ -2,6 +2,7 @@ package org.apache.cassandra.io.sstable;
import java.io.IOException;
import java.util.concurrent.ExecutionException;
+import java.util.Map;
import org.junit.Test;
@@ -9,22 +10,26 @@ import org.apache.cassandra.CleanupHelpe
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.QueryPath;
+import org.apache.cassandra.io.util.BufferedRandomAccessFile;
import org.apache.cassandra.io.util.FileDataInput;
+import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.Util;
+import static org.junit.Assert.assertEquals;
+
public class SSTableReaderTest extends CleanupHelper
{
@Test
public void testSpannedIndexPositions() throws IOException, ExecutionException, InterruptedException
{
- RowIndexedReader.BUFFER_SIZE = 40;
+ RowIndexedReader.BUFFER_SIZE = 40; // each index entry is ~11 bytes, so this will generate lots of spanned entries
Table table = Table.open("Keyspace1");
ColumnFamilyStore store = table.getColumnFamilyStore("Standard1");
- // insert a bunch of data
+ // insert a bunch of data and compact to a single sstable
CompactionManager.instance.disableAutoCompaction();
for (int j = 0; j < 100; j += 2)
{
@@ -52,5 +57,22 @@ public class SSTableReaderTest extends C
DecoratedKey dk = Util.dk(String.valueOf(j));
assert sstable.getPosition(dk) == null;
}
+
+ // check positionsize information
+ assert sstable.indexSummary.getSpannedIndexDataPositions().entrySet().size() > 0;
+ for (Map.Entry<IndexSummary.KeyPosition, SSTable.PositionSize> entry : sstable.indexSummary.getSpannedIndexDataPositions().entrySet())
+ {
+ IndexSummary.KeyPosition kp = entry.getKey();
+ SSTable.PositionSize info = entry.getValue();
+
+ long nextIndexPosition = kp.indexPosition + 2 + StorageService.getPartitioner().convertToDiskFormat(kp.key).length + 8;
+ BufferedRandomAccessFile indexFile = new BufferedRandomAccessFile(sstable.indexFilename(), "r");
+ indexFile.seek(nextIndexPosition);
+ String nextKey = indexFile.readUTF();
+
+ BufferedRandomAccessFile file = new BufferedRandomAccessFile(sstable.getFilename(), "r");
+ file.seek(info.position + info.size);
+ assertEquals(nextKey, file.readUTF());
+ }
}
}