You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2013/09/20 00:45:54 UTC
[2/6] git commit: Revert CASSANDRA-5577 to fix CASSANDRA-6025 and add
test patch by jbellis
Revert CASSANDRA-5577 to fix CASSANDRA-6025 and add test
patch by jbellis
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/e5cbde9d
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/e5cbde9d
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/e5cbde9d
Branch: refs/heads/trunk
Commit: e5cbde9d62d7cb3a4755099750107aea77b157be
Parents: 023b55f
Author: Jonathan Ellis <jb...@apache.org>
Authored: Thu Sep 19 17:43:21 2013 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Thu Sep 19 17:43:21 2013 -0500
----------------------------------------------------------------------
.../cassandra/db/CollationController.java | 32 ++-------
.../unit/org/apache/cassandra/SchemaLoader.java | 1 +
.../cassandra/db/CollationControllerTest.java | 73 +++++++++++++++-----
3 files changed, 63 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e5cbde9d/src/java/org/apache/cassandra/db/CollationController.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/CollationController.java b/src/java/org/apache/cassandra/db/CollationController.java
index 4ae27a2..859135d 100644
--- a/src/java/org/apache/cassandra/db/CollationController.java
+++ b/src/java/org/apache/cassandra/db/CollationController.java
@@ -85,11 +85,7 @@ public class CollationController
iterators.add(iter);
temp.delete(iter.getColumnFamily());
while (iter.hasNext())
- {
- OnDiskAtom atom = iter.next();
- if (atom.getLocalDeletionTime() >= gcBefore)
- temp.addAtom(atom);
- }
+ temp.addAtom(iter.next());
}
container.addAll(temp, HeapAllocator.instance);
@@ -131,11 +127,7 @@ public class CollationController
temp.delete(cf);
sstablesIterated++;
while (iter.hasNext())
- {
- OnDiskAtom atom = iter.next();
- if (atom.getLocalDeletionTime() >= gcBefore)
- temp.addAtom(atom);
- }
+ temp.addAtom(iter.next());
}
container.addAll(temp, HeapAllocator.instance);
@@ -147,22 +139,10 @@ public class CollationController
if (iterators.isEmpty())
return null;
- // We may have added columns that are shadowed by range or row tombstones, since we don't know what
- // tombstones we may encounter in older sstables (and we don't know how many older sstables we'll have
- // to open, without processing newer ones first). So, make one more pass if necessary to clean those out.
- ColumnFamily returnCF;
- if (container.isMarkedForDelete())
- {
- returnCF = container.cloneMeShallow();
- Tracing.trace("Removing shadowed cells");
- filter.collateOnDiskAtom(returnCF, container.iterator(), gcBefore);
- }
- else
- {
- // skipping the collate is safe because we only do this time-ordered path for NameQueryFilter;
- // for SQF, the collate is also what limits us to the requested number of columns.
- returnCF = container;
- }
+ // do a final collate. toCollate is boilerplate required to provide a CloseableIterator
+ ColumnFamily returnCF = container.cloneMeShallow();
+ Tracing.trace("Collating all results");
+ filter.collateOnDiskAtom(returnCF, container.iterator(), gcBefore);
// "hoist up" the requested data into a more recent sstable
if (sstablesIterated > cfs.getMinimumCompactionThreshold()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e5cbde9d/test/unit/org/apache/cassandra/SchemaLoader.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/SchemaLoader.java b/test/unit/org/apache/cassandra/SchemaLoader.java
index 1bd192b..f933b6f 100644
--- a/test/unit/org/apache/cassandra/SchemaLoader.java
+++ b/test/unit/org/apache/cassandra/SchemaLoader.java
@@ -149,6 +149,7 @@ public class SchemaLoader
standardCFMD(ks1, "Standard2"),
standardCFMD(ks1, "Standard3"),
standardCFMD(ks1, "Standard4"),
+ standardCFMD(ks1, "StandardGCGS0").gcGraceSeconds(0),
standardCFMD(ks1, "StandardLong1"),
standardCFMD(ks1, "StandardLong2"),
new CFMetaData(ks1,
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e5cbde9d/test/unit/org/apache/cassandra/db/CollationControllerTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/CollationControllerTest.java b/test/unit/org/apache/cassandra/db/CollationControllerTest.java
index 0c23272..8763a41 100644
--- a/test/unit/org/apache/cassandra/db/CollationControllerTest.java
+++ b/test/unit/org/apache/cassandra/db/CollationControllerTest.java
@@ -21,6 +21,7 @@ package org.apache.cassandra.db;
import static org.junit.Assert.assertEquals;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.util.concurrent.ExecutionException;
import org.apache.cassandra.SchemaLoader;
@@ -35,51 +36,89 @@ public class CollationControllerTest extends SchemaLoader
{
@Test
public void getTopLevelColumnsSkipsSSTablesModifiedBeforeRowDelete()
- throws IOException, ExecutionException, InterruptedException
+ throws IOException, ExecutionException, InterruptedException
{
Keyspace keyspace = Keyspace.open("Keyspace1");
- ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard1");
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Standard1");
RowMutation rm;
DecoratedKey dk = Util.dk("key1");
// add data
- rm = new RowMutation("Keyspace1", dk.key);
- rm.add("Standard1", ByteBufferUtil.bytes("Column1"), ByteBufferUtil.bytes("asdf"), 0);
+ rm = new RowMutation(keyspace.getName(), dk.key);
+ rm.add(cfs.name, ByteBufferUtil.bytes("Column1"), ByteBufferUtil.bytes("asdf"), 0);
rm.apply();
- store.forceBlockingFlush();
+ cfs.forceBlockingFlush();
// remove
- rm = new RowMutation("Keyspace1", dk.key);
- rm.delete("Standard1", 10);
+ rm = new RowMutation(keyspace.getName(), dk.key);
+ rm.delete(cfs.name, 10);
rm.apply();
// add another mutation because sstable maxtimestamp isn't set
// correctly during flush if the most recent mutation is a row delete
- rm = new RowMutation("Keyspace1", Util.dk("key2").key);
- rm.add("Standard1", ByteBufferUtil.bytes("Column1"), ByteBufferUtil.bytes("zxcv"), 20);
+ rm = new RowMutation(keyspace.getName(), Util.dk("key2").key);
+ rm.add(cfs.name, ByteBufferUtil.bytes("Column1"), ByteBufferUtil.bytes("zxcv"), 20);
rm.apply();
- store.forceBlockingFlush();
+ cfs.forceBlockingFlush();
// add yet one more mutation
- rm = new RowMutation("Keyspace1", dk.key);
- rm.add("Standard1", ByteBufferUtil.bytes("Column1"), ByteBufferUtil.bytes("foobar"), 30);
+ rm = new RowMutation(keyspace.getName(), dk.key);
+ rm.add(cfs.name, ByteBufferUtil.bytes("Column1"), ByteBufferUtil.bytes("foobar"), 30);
rm.apply();
- store.forceBlockingFlush();
+ cfs.forceBlockingFlush();
// A NamesQueryFilter goes down one code path (through collectTimeOrderedData())
// It should only iterate the last flushed sstable, since it probably contains the most recent value for Column1
- QueryFilter filter = QueryFilter.getNamesFilter(dk, "Standard1", ByteBufferUtil.bytes("Column1"), System.currentTimeMillis());
- CollationController controller = new CollationController(store, filter, Integer.MIN_VALUE);
+ QueryFilter filter = QueryFilter.getNamesFilter(dk, cfs.name, ByteBufferUtil.bytes("Column1"), System.currentTimeMillis());
+ CollationController controller = new CollationController(cfs, filter, Integer.MIN_VALUE);
controller.getTopLevelColumns();
assertEquals(1, controller.getSstablesIterated());
// SliceQueryFilter goes down another path (through collectAllData())
// We will read "only" the last sstable in that case, but because the 2nd sstable has a tombstone that is more
// recent than the maxTimestamp of the very first sstable we flushed, we should only read the 2 first sstables.
- filter = QueryFilter.getIdentityFilter(dk, "Standard1", System.currentTimeMillis());
- controller = new CollationController(store, filter, Integer.MIN_VALUE);
+ filter = QueryFilter.getIdentityFilter(dk, cfs.name, System.currentTimeMillis());
+ controller = new CollationController(cfs, filter, Integer.MIN_VALUE);
controller.getTopLevelColumns();
assertEquals(2, controller.getSstablesIterated());
}
+
+ @Test
+ public void ensureTombstonesAppliedAfterGCGS()
+ throws IOException, ExecutionException, InterruptedException
+ {
+ Keyspace keyspace = Keyspace.open("Keyspace1");
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("StandardGCGS0");
+ cfs.disableAutoCompaction();
+
+ RowMutation rm;
+ DecoratedKey dk = Util.dk("key1");
+ ByteBuffer cellName = ByteBufferUtil.bytes("Column1");
+
+ // add data
+ rm = new RowMutation(keyspace.getName(), dk.key);
+ rm.add(cfs.name, cellName, ByteBufferUtil.bytes("asdf"), 0);
+ rm.apply();
+ cfs.forceBlockingFlush();
+
+ // remove
+ rm = new RowMutation(keyspace.getName(), dk.key);
+ rm.delete(cfs.name, cellName, 0);
+ rm.apply();
+ cfs.forceBlockingFlush();
+
+ // use "realistic" query times since we'll compare these numbers to the local deletion time of the tombstone
+ QueryFilter filter;
+ long queryAt = System.currentTimeMillis() + 1000;
+ int gcBefore = cfs.gcBefore(queryAt);
+
+ filter = QueryFilter.getNamesFilter(dk, cfs.name, cellName, queryAt);
+ CollationController controller = new CollationController(cfs, filter, gcBefore);
+ assert ColumnFamilyStore.removeDeleted(controller.getTopLevelColumns(), gcBefore) == null;
+
+ filter = QueryFilter.getIdentityFilter(dk, cfs.name, queryAt);
+ controller = new CollationController(cfs, filter, gcBefore);
+ assert ColumnFamilyStore.removeDeleted(controller.getTopLevelColumns(), gcBefore) == null;
+ }
}