You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by sl...@apache.org on 2013/10/03 11:07:12 UTC
git commit: Fix skipping columns with multiple slices
Updated Branches:
refs/heads/cassandra-1.2 c8f0e3a9f -> 20a805023
Fix skipping columns with multiple slices
patch by frousseau & slebresne; reviewed by frousseau & slebresne for CASSANDRA-6119
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/20a80502
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/20a80502
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/20a80502
Branch: refs/heads/cassandra-1.2
Commit: 20a805023fa26ab1b2f70b574b35357df9652cd3
Parents: c8f0e3a
Author: Sylvain Lebresne <sy...@datastax.com>
Authored: Thu Oct 3 11:06:12 2013 +0200
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Thu Oct 3 11:06:12 2013 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../db/columniterator/IndexedSliceReader.java | 17 +-
.../cassandra/db/ColumnFamilyStoreTest.java | 198 +++++++++++++++++++
3 files changed, 213 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/20a80502/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index cc04eca..c1d1991 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -12,6 +12,7 @@
* Add tombstone debug threshold and histogram (CASSANDRA-6042, 6057)
* Fix fat client schema pull NPE (CASSANDRA-6089)
* Fix memtable flushing for indexed tables (CASSANDRA-6112)
+ * Fix skipping columns with multiple slices (CASSANDRA-6119)
1.2.10
http://git-wip-us.apache.org/repos/asf/cassandra/blob/20a80502/src/java/org/apache/cassandra/db/columniterator/IndexedSliceReader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/columniterator/IndexedSliceReader.java b/src/java/org/apache/cassandra/db/columniterator/IndexedSliceReader.java
index 472ecfc..df916b2 100644
--- a/src/java/org/apache/cassandra/db/columniterator/IndexedSliceReader.java
+++ b/src/java/org/apache/cassandra/db/columniterator/IndexedSliceReader.java
@@ -391,7 +391,7 @@ class IndexedSliceReader extends AbstractIterator<OnDiskAtom> implements OnDiskA
// scan from index start
OnDiskAtom column = null;
- while (file.bytesPastMark(mark) < currentIndex.width)
+ while (file.bytesPastMark(mark) < currentIndex.width || column != null)
{
// Only fetch a new column if we haven't dealt with the previous one.
if (column == null)
@@ -467,20 +467,31 @@ class IndexedSliceReader extends AbstractIterator<OnDiskAtom> implements OnDiskA
OnDiskAtom.Serializer atomSerializer = emptyColumnFamily.getOnDiskSerializer();
int columns = file.readInt();
- for (int i = 0; i < columns; i++)
+ OnDiskAtom column = null;
+ int i = 0;
+ while (i < columns || column != null)
{
- OnDiskAtom column = atomSerializer.deserializeFromSSTable(file, sstable.descriptor.version);
+ // Only fetch a new column if we haven't dealt with the previous one.
+ if (column == null)
+ {
+ column = atomSerializer.deserializeFromSSTable(file, sstable.descriptor.version);
+ i++;
+ }
// col is before slice
// (If in slice, don't bother checking that until we change slice)
if (!inSlice && isColumnBeforeSliceStart(column))
+ {
+ column = null;
continue;
+ }
// col is within slice
if (isColumnBeforeSliceFinish(column))
{
inSlice = true;
addColumn(column);
+ column = null;
}
// col is after slice. more slices?
else
http://git-wip-us.apache.org/repos/asf/cassandra/blob/20a80502/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
index abe3f05..cd30297 100644
--- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
+++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
@@ -1160,6 +1160,204 @@ public class ColumnFamilyStoreTest extends SchemaLoader
@SuppressWarnings("unchecked")
@Test
+ public void testMultiRangeSomeEmptyNoIndex() throws Throwable
+ {
+ // in order not to change thrift interfaces at this stage we build SliceQueryFilter
+ // directly instead of using QueryFilter to build it for us
+ ColumnSlice[] ranges = new ColumnSlice[] {
+ new ColumnSlice(ByteBuffer.wrap(EMPTY_BYTE_ARRAY), bytes("colA")),
+ new ColumnSlice(bytes("colC"), bytes("colE")),
+ new ColumnSlice(bytes("colF"), bytes("colF")),
+ new ColumnSlice(bytes("colG"), bytes("colG")),
+ new ColumnSlice(bytes("colI"), ByteBuffer.wrap(EMPTY_BYTE_ARRAY)) };
+
+ ColumnSlice[] rangesReversed = new ColumnSlice[] {
+ new ColumnSlice(ByteBuffer.wrap(EMPTY_BYTE_ARRAY), bytes("colI")),
+ new ColumnSlice(bytes("colG"), bytes("colG")),
+ new ColumnSlice(bytes("colF"), bytes("colF")),
+ new ColumnSlice(bytes("colE"), bytes("colC")),
+ new ColumnSlice(bytes("colA"), ByteBuffer.wrap(EMPTY_BYTE_ARRAY)) };
+
+ String tableName = "Keyspace1";
+ String cfName = "Standard1";
+ Table table = Table.open(tableName);
+ ColumnFamilyStore cfs = table.getColumnFamilyStore(cfName);
+ cfs.clearUnsafe();
+
+ String[] letters = new String[] { "a", "b", "c", "d", "i" };
+ Column[] cols = new Column[letters.length];
+ for (int i = 0; i < cols.length; i++)
+ {
+ cols[i] = new Column(ByteBufferUtil.bytes("col" + letters[i].toUpperCase()),
+ ByteBuffer.wrap(new byte[1]), 1);
+ }
+
+ putColsStandard(cfs, dk("a"), cols);
+
+ cfs.forceBlockingFlush();
+
+ SliceQueryFilter multiRangeForward = new SliceQueryFilter(ranges, false, 100);
+ SliceQueryFilter multiRangeForwardWithCounting = new SliceQueryFilter(ranges, false, 3);
+ SliceQueryFilter multiRangeReverse = new SliceQueryFilter(rangesReversed, true, 100);
+ SliceQueryFilter multiRangeReverseWithCounting = new SliceQueryFilter(rangesReversed, true, 3);
+
+ findRowGetSlicesAndAssertColsFound(cfs, multiRangeForward, "a", "colA", "colC", "colD", "colI");
+ findRowGetSlicesAndAssertColsFound(cfs, multiRangeForwardWithCounting, "a", "colA", "colC", "colD");
+ findRowGetSlicesAndAssertColsFound(cfs, multiRangeReverse, "a", "colI", "colD", "colC", "colA");
+ findRowGetSlicesAndAssertColsFound(cfs, multiRangeReverseWithCounting, "a", "colI", "colD", "colC");
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testMultiRangeSomeEmptyIndexed() throws Throwable
+ {
+ // in order not to change thrift interfaces at this stage we build SliceQueryFilter
+ // directly instead of using QueryFilter to build it for us
+ ColumnSlice[] ranges = new ColumnSlice[] {
+ new ColumnSlice(ByteBuffer.wrap(EMPTY_BYTE_ARRAY), bytes("colA")),
+ new ColumnSlice(bytes("colC"), bytes("colE")),
+ new ColumnSlice(bytes("colF"), bytes("colF")),
+ new ColumnSlice(bytes("colG"), bytes("colG")),
+ new ColumnSlice(bytes("colI"), ByteBuffer.wrap(EMPTY_BYTE_ARRAY)) };
+
+ ColumnSlice[] rangesReversed = new ColumnSlice[] {
+ new ColumnSlice(ByteBuffer.wrap(EMPTY_BYTE_ARRAY), bytes("colI")),
+ new ColumnSlice(bytes("colG"), bytes("colG")),
+ new ColumnSlice(bytes("colF"), bytes("colF")),
+ new ColumnSlice(bytes("colE"), bytes("colC")),
+ new ColumnSlice(bytes("colA"), ByteBuffer.wrap(EMPTY_BYTE_ARRAY)) };
+
+ String tableName = "Keyspace1";
+ String cfName = "Standard1";
+ Table table = Table.open(tableName);
+ ColumnFamilyStore cfs = table.getColumnFamilyStore(cfName);
+ cfs.clearUnsafe();
+
+ String[] letters = new String[] { "a", "b", "c", "d", "i" };
+ Column[] cols = new Column[letters.length];
+ for (int i = 0; i < cols.length; i++)
+ {
+ cols[i] = new Column(ByteBufferUtil.bytes("col" + letters[i].toUpperCase()),
+ ByteBuffer.wrap(new byte[1366]), 1);
+ }
+
+ putColsStandard(cfs, dk("a"), cols);
+
+ cfs.forceBlockingFlush();
+
+ SliceQueryFilter multiRangeForward = new SliceQueryFilter(ranges, false, 100);
+ SliceQueryFilter multiRangeForwardWithCounting = new SliceQueryFilter(ranges, false, 3);
+ SliceQueryFilter multiRangeReverse = new SliceQueryFilter(rangesReversed, true, 100);
+ SliceQueryFilter multiRangeReverseWithCounting = new SliceQueryFilter(rangesReversed, true, 3);
+
+ findRowGetSlicesAndAssertColsFound(cfs, multiRangeForward, "a", "colA", "colC", "colD", "colI");
+ findRowGetSlicesAndAssertColsFound(cfs, multiRangeForwardWithCounting, "a", "colA", "colC", "colD");
+ findRowGetSlicesAndAssertColsFound(cfs, multiRangeReverse, "a", "colI", "colD", "colC", "colA");
+ findRowGetSlicesAndAssertColsFound(cfs, multiRangeReverseWithCounting, "a", "colI", "colD", "colC");
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testMultiRangeContiguousNoIndex() throws Throwable
+ {
+ // in order not to change thrift interfaces at this stage we build SliceQueryFilter
+ // directly instead of using QueryFilter to build it for us
+ ColumnSlice[] ranges = new ColumnSlice[] {
+ new ColumnSlice(ByteBuffer.wrap(EMPTY_BYTE_ARRAY), bytes("colA")),
+ new ColumnSlice(bytes("colC"), bytes("colE")),
+ new ColumnSlice(bytes("colF"), bytes("colF")),
+ new ColumnSlice(bytes("colG"), bytes("colG")),
+ new ColumnSlice(bytes("colI"), ByteBuffer.wrap(EMPTY_BYTE_ARRAY)) };
+
+ ColumnSlice[] rangesReversed = new ColumnSlice[] {
+ new ColumnSlice(ByteBuffer.wrap(EMPTY_BYTE_ARRAY), bytes("colI")),
+ new ColumnSlice(bytes("colG"), bytes("colG")),
+ new ColumnSlice(bytes("colF"), bytes("colF")),
+ new ColumnSlice(bytes("colE"), bytes("colC")),
+ new ColumnSlice(bytes("colA"), ByteBuffer.wrap(EMPTY_BYTE_ARRAY)) };
+
+ String tableName = "Keyspace1";
+ String cfName = "Standard1";
+ Table table = Table.open(tableName);
+ ColumnFamilyStore cfs = table.getColumnFamilyStore(cfName);
+ cfs.clearUnsafe();
+
+ String[] letters = new String[] { "a", "b", "c", "d", "e", "f", "g", "h", "i" };
+ Column[] cols = new Column[letters.length];
+ for (int i = 0; i < cols.length; i++)
+ {
+ cols[i] = new Column(ByteBufferUtil.bytes("col" + letters[i].toUpperCase()),
+ ByteBuffer.wrap(new byte[1]), 1);
+ }
+
+ putColsStandard(cfs, dk("a"), cols);
+
+ cfs.forceBlockingFlush();
+
+ SliceQueryFilter multiRangeForward = new SliceQueryFilter(ranges, false, 100);
+ SliceQueryFilter multiRangeForwardWithCounting = new SliceQueryFilter(ranges, false, 3);
+ SliceQueryFilter multiRangeReverse = new SliceQueryFilter(rangesReversed, true, 100);
+ SliceQueryFilter multiRangeReverseWithCounting = new SliceQueryFilter(rangesReversed, true, 3);
+
+ findRowGetSlicesAndAssertColsFound(cfs, multiRangeForward, "a", "colA", "colC", "colD", "colE", "colF", "colG", "colI");
+ findRowGetSlicesAndAssertColsFound(cfs, multiRangeForwardWithCounting, "a", "colA", "colC", "colD");
+ findRowGetSlicesAndAssertColsFound(cfs, multiRangeReverse, "a", "colI", "colG", "colF", "colE", "colD", "colC", "colA");
+ findRowGetSlicesAndAssertColsFound(cfs, multiRangeReverseWithCounting, "a", "colI", "colG", "colF");
+
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testMultiRangeContiguousIndexed() throws Throwable
+ {
+ // in order not to change thrift interfaces at this stage we build SliceQueryFilter
+ // directly instead of using QueryFilter to build it for us
+ ColumnSlice[] ranges = new ColumnSlice[] {
+ new ColumnSlice(ByteBuffer.wrap(EMPTY_BYTE_ARRAY), bytes("colA")),
+ new ColumnSlice(bytes("colC"), bytes("colE")),
+ new ColumnSlice(bytes("colF"), bytes("colF")),
+ new ColumnSlice(bytes("colG"), bytes("colG")),
+ new ColumnSlice(bytes("colI"), ByteBuffer.wrap(EMPTY_BYTE_ARRAY)) };
+
+ ColumnSlice[] rangesReversed = new ColumnSlice[] {
+ new ColumnSlice(ByteBuffer.wrap(EMPTY_BYTE_ARRAY), bytes("colI")),
+ new ColumnSlice(bytes("colG"), bytes("colG")),
+ new ColumnSlice(bytes("colF"), bytes("colF")),
+ new ColumnSlice(bytes("colE"), bytes("colC")),
+ new ColumnSlice(bytes("colA"), ByteBuffer.wrap(EMPTY_BYTE_ARRAY)) };
+
+ String tableName = "Keyspace1";
+ String cfName = "Standard1";
+ Table table = Table.open(tableName);
+ ColumnFamilyStore cfs = table.getColumnFamilyStore(cfName);
+ cfs.clearUnsafe();
+
+ String[] letters = new String[] { "a", "b", "c", "d", "e", "f", "g", "h", "i" };
+ Column[] cols = new Column[letters.length];
+ for (int i = 0; i < cols.length; i++)
+ {
+ cols[i] = new Column(ByteBufferUtil.bytes("col" + letters[i].toUpperCase()),
+ ByteBuffer.wrap(new byte[1366]), 1);
+ }
+
+ putColsStandard(cfs, dk("a"), cols);
+
+ cfs.forceBlockingFlush();
+
+ SliceQueryFilter multiRangeForward = new SliceQueryFilter(ranges, false, 100);
+ SliceQueryFilter multiRangeForwardWithCounting = new SliceQueryFilter(ranges, false, 3);
+ SliceQueryFilter multiRangeReverse = new SliceQueryFilter(rangesReversed, true, 100);
+ SliceQueryFilter multiRangeReverseWithCounting = new SliceQueryFilter(rangesReversed, true, 3);
+
+ findRowGetSlicesAndAssertColsFound(cfs, multiRangeForward, "a", "colA", "colC", "colD", "colE", "colF", "colG", "colI");
+ findRowGetSlicesAndAssertColsFound(cfs, multiRangeForwardWithCounting, "a", "colA", "colC", "colD");
+ findRowGetSlicesAndAssertColsFound(cfs, multiRangeReverse, "a", "colI", "colG", "colF", "colE", "colD", "colC", "colA");
+ findRowGetSlicesAndAssertColsFound(cfs, multiRangeReverseWithCounting, "a", "colI", "colG", "colF");
+
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
public void testMultiRangeIndexed() throws Throwable
{
// in order not to change thrift interfaces at this stage we build SliceQueryFilter