You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by br...@apache.org on 2013/10/30 20:11:44 UTC
[1/6] git commit: Use non-pooled SegmentedFile builders when bulk
loading Patch by Sam Tunnicliffe,
reviewed by brandonwilliams for CASSANDRA-6272
Updated Branches:
refs/heads/cassandra-1.2 620bb80ec -> 18f793537
refs/heads/cassandra-2.0 4169aa3d4 -> 89911c82a
refs/heads/trunk 45bb92467 -> 58e2706d3
Use non-pooled SegmentedFile builders when bulk loading
Patch by Sam Tunnicliffe, reviewed by brandonwilliams for CASSANDRA-6272
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/18f79353
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/18f79353
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/18f79353
Branch: refs/heads/cassandra-1.2
Commit: 18f7935370b1628cba8fa35819c05f235ba918aa
Parents: 620bb80
Author: Brandon Williams <br...@apache.org>
Authored: Wed Oct 30 14:01:44 2013 -0500
Committer: Brandon Williams <br...@apache.org>
Committed: Wed Oct 30 14:03:08 2013 -0500
----------------------------------------------------------------------
.../cassandra/io/sstable/SSTableReader.java | 44 ++++++--------------
.../cassandra/io/sstable/SSTableReaderTest.java | 38 +++++++++++++++++
2 files changed, 50 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/18f79353/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
index ed221d9..4af1cdc 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
@@ -166,8 +166,19 @@ public class SSTableReader extends SSTable implements Closeable
partitioner,
System.currentTimeMillis(),
sstableMetadata);
+
+ // special implementation of load to use non-pooled SegmentedFile builders
+ SegmentedFile.Builder ibuilder = new BufferedSegmentedFile.Builder();
+ SegmentedFile.Builder dbuilder = sstable.compression
+ ? new CompressedSegmentedFile.Builder()
+ : new BufferedSegmentedFile.Builder();
+
+ if (!loadSummary(sstable, ibuilder, dbuilder))
+ sstable.buildSummary(false, ibuilder, dbuilder, false);
+ sstable.ifile = ibuilder.complete(sstable.descriptor.filenameFor(Component.PRIMARY_INDEX));
+ sstable.dfile = dbuilder.complete(sstable.descriptor.filenameFor(Component.DATA));
+
sstable.bf = new AlwaysPresentFilter();
- sstable.loadForBatch();
return sstable;
}
@@ -409,37 +420,6 @@ public class SSTableReader extends SSTable implements Closeable
saveSummary(this, ibuilder, dbuilder);
}
- /**
- * A simplified load that creates a minimal partition index
- */
- private void loadForBatch() throws IOException
- {
- SegmentedFile.Builder ibuilder = new BufferedSegmentedFile.Builder();
- SegmentedFile.Builder dbuilder = compression
- ? new CompressedSegmentedFile.Builder()
- : new BufferedSegmentedFile.Builder();
-
- // build a bare-bones IndexSummary
- IndexSummaryBuilder summaryBuilder = new IndexSummaryBuilder(1);
- RandomAccessReader in = RandomAccessReader.open(new File(descriptor.filenameFor(Component.PRIMARY_INDEX)), true);
- try
- {
- ByteBuffer key = ByteBufferUtil.readWithShortLength(in);
- first = decodeKey(partitioner, descriptor, key);
- summaryBuilder.maybeAddEntry(first, 0);
- indexSummary = summaryBuilder.build(partitioner);
- }
- finally
- {
- FileUtils.closeQuietly(in);
- }
-
- last = null; // shouldn't need this for batch operations
-
- ifile = ibuilder.complete(descriptor.filenameFor(Component.PRIMARY_INDEX));
- dfile = dbuilder.complete(descriptor.filenameFor(Component.DATA));
- }
-
private void buildSummary(boolean recreatebloom, SegmentedFile.Builder ibuilder, SegmentedFile.Builder dbuilder, boolean summaryLoaded) throws IOException
{
// we read the positions in a BRAF so we don't have to worry about an entry spanning a mmap boundary.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/18f79353/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
index 02b6855..fef27f1 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
@@ -27,8 +27,10 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.Set;
import java.util.concurrent.ExecutionException;
+import com.google.common.collect.Sets;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -315,6 +317,42 @@ public class SSTableReaderTest extends SchemaLoader
assert reopened.first.token instanceof LocalToken;
}
+ @Test
+ public void testGetPositionsForRangesFromTableOpenedForBulkLoading() throws IOException, ExecutionException, InterruptedException
+ {
+ Table table = Table.open("Keyspace1");
+ ColumnFamilyStore store = table.getColumnFamilyStore("Standard2");
+
+ // insert data and compact to a single sstable. The
+ // number of keys inserted is greater than index_interval
+ // to ensure multiple segments in the index file
+ CompactionManager.instance.disableAutoCompaction();
+ for (int j = 0; j < 130; j++)
+ {
+ ByteBuffer key = ByteBufferUtil.bytes(String.valueOf(j));
+ RowMutation rm = new RowMutation("Keyspace1", key);
+ rm.add(new QueryPath("Standard2", null, ByteBufferUtil.bytes("0")), ByteBufferUtil.EMPTY_BYTE_BUFFER, j);
+ rm.apply();
+ }
+ store.forceBlockingFlush();
+ CompactionManager.instance.performMaximal(store);
+
+ // construct a range which is present in the sstable, but whose
+ // keys are not found in the first segment of the index.
+ List<Range<Token>> ranges = new ArrayList<Range<Token>>();
+ ranges.add(new Range<Token>(t(98), t(99)));
+
+ SSTableReader sstable = store.getSSTables().iterator().next();
+ List<Pair<Long,Long>> sections = sstable.getPositionsForRanges(ranges);
+ assert sections.size() == 1 : "Expected to find range in sstable" ;
+
+ // re-open the same sstable as it would be during bulk loading
+ Set<Component> components = Sets.newHashSet(Component.DATA, Component.PRIMARY_INDEX);
+ SSTableReader bulkLoaded = SSTableReader.openForBatch(sstable.descriptor, components, sstable.partitioner);
+ sections = bulkLoaded.getPositionsForRanges(ranges);
+ assert sections.size() == 1 : "Expected to find range in sstable opened for bulk loading";
+ }
+
private void assertIndexQueryWorks(ColumnFamilyStore indexedCFS) throws IOException
{
assert "Indexed1".equals(indexedCFS.getColumnFamilyName());
[5/6] git commit: Merge branch 'cassandra-1.2' into cassandra-2.0
Posted by br...@apache.org.
Merge branch 'cassandra-1.2' into cassandra-2.0
Conflicts:
src/java/org/apache/cassandra/io/sstable/SSTableReader.java
test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/89911c82
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/89911c82
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/89911c82
Branch: refs/heads/cassandra-2.0
Commit: 89911c82ac7bddb32ccac4adc224c764d87b5cf3
Parents: 4169aa3 18f7935
Author: Brandon Williams <br...@apache.org>
Authored: Wed Oct 30 14:09:47 2013 -0500
Committer: Brandon Williams <br...@apache.org>
Committed: Wed Oct 30 14:09:47 2013 -0500
----------------------------------------------------------------------
.../cassandra/io/sstable/SSTableReaderTest.java | 37 ++++++++++++++++++++
1 file changed, 37 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/89911c82/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
index eeb22e6,fef27f1..8a9619d
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
@@@ -26,9 -27,10 +26,11 @@@ import java.nio.ByteBuffer
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+ import java.util.Set;
import java.util.concurrent.ExecutionException;
+import org.junit.Assert;
+ import com.google.common.collect.Sets;
import org.junit.Test;
import org.junit.runner.RunWith;
@@@ -284,25 -317,40 +286,60 @@@ public class SSTableReaderTest extends
assert reopened.first.token instanceof LocalToken;
}
+ /** see CASSANDRA-5407 */
+ @Test
+ public void testGetScannerForNoIntersectingRanges()
+ {
+ Keyspace keyspace = Keyspace.open("Keyspace1");
+ ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard1");
+ ByteBuffer key = ByteBufferUtil.bytes(String.valueOf("k1"));
+ RowMutation rm = new RowMutation("Keyspace1", key);
+ rm.add("Standard1", ByteBufferUtil.bytes("xyz"), ByteBufferUtil.bytes("abc"), 0);
+ rm.apply();
+ store.forceBlockingFlush();
+ boolean foundScanner = false;
+ for (SSTableReader s : store.getSSTables())
+ {
+ ICompactionScanner scanner = s.getScanner(new Range<Token>(t(0), t(1), s.partitioner), null);
+ scanner.next(); // throws exception pre 5407
+ foundScanner = true;
+ }
+ assertTrue(foundScanner);
++
+ @Test
+ public void testGetPositionsForRangesFromTableOpenedForBulkLoading() throws IOException, ExecutionException, InterruptedException
+ {
+ Table table = Table.open("Keyspace1");
+ ColumnFamilyStore store = table.getColumnFamilyStore("Standard2");
+
+ // insert data and compact to a single sstable. The
+ // number of keys inserted is greater than index_interval
+ // to ensure multiple segments in the index file
+ CompactionManager.instance.disableAutoCompaction();
+ for (int j = 0; j < 130; j++)
+ {
+ ByteBuffer key = ByteBufferUtil.bytes(String.valueOf(j));
+ RowMutation rm = new RowMutation("Keyspace1", key);
+ rm.add(new QueryPath("Standard2", null, ByteBufferUtil.bytes("0")), ByteBufferUtil.EMPTY_BYTE_BUFFER, j);
+ rm.apply();
+ }
+ store.forceBlockingFlush();
+ CompactionManager.instance.performMaximal(store);
+
+ // construct a range which is present in the sstable, but whose
+ // keys are not found in the first segment of the index.
+ List<Range<Token>> ranges = new ArrayList<Range<Token>>();
+ ranges.add(new Range<Token>(t(98), t(99)));
+
+ SSTableReader sstable = store.getSSTables().iterator().next();
+ List<Pair<Long,Long>> sections = sstable.getPositionsForRanges(ranges);
+ assert sections.size() == 1 : "Expected to find range in sstable" ;
+
+ // re-open the same sstable as it would be during bulk loading
+ Set<Component> components = Sets.newHashSet(Component.DATA, Component.PRIMARY_INDEX);
+ SSTableReader bulkLoaded = SSTableReader.openForBatch(sstable.descriptor, components, sstable.partitioner);
+ sections = bulkLoaded.getPositionsForRanges(ranges);
+ assert sections.size() == 1 : "Expected to find range in sstable opened for bulk loading";
}
private void assertIndexQueryWorks(ColumnFamilyStore indexedCFS) throws IOException
[4/6] git commit: Merge branch 'cassandra-1.2' into cassandra-2.0
Posted by br...@apache.org.
Merge branch 'cassandra-1.2' into cassandra-2.0
Conflicts:
src/java/org/apache/cassandra/io/sstable/SSTableReader.java
test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/89911c82
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/89911c82
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/89911c82
Branch: refs/heads/trunk
Commit: 89911c82ac7bddb32ccac4adc224c764d87b5cf3
Parents: 4169aa3 18f7935
Author: Brandon Williams <br...@apache.org>
Authored: Wed Oct 30 14:09:47 2013 -0500
Committer: Brandon Williams <br...@apache.org>
Committed: Wed Oct 30 14:09:47 2013 -0500
----------------------------------------------------------------------
.../cassandra/io/sstable/SSTableReaderTest.java | 37 ++++++++++++++++++++
1 file changed, 37 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/89911c82/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
index eeb22e6,fef27f1..8a9619d
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
@@@ -26,9 -27,10 +26,11 @@@ import java.nio.ByteBuffer
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+ import java.util.Set;
import java.util.concurrent.ExecutionException;
+import org.junit.Assert;
+ import com.google.common.collect.Sets;
import org.junit.Test;
import org.junit.runner.RunWith;
@@@ -284,25 -317,40 +286,60 @@@ public class SSTableReaderTest extends
assert reopened.first.token instanceof LocalToken;
}
+ /** see CASSANDRA-5407 */
+ @Test
+ public void testGetScannerForNoIntersectingRanges()
+ {
+ Keyspace keyspace = Keyspace.open("Keyspace1");
+ ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard1");
+ ByteBuffer key = ByteBufferUtil.bytes(String.valueOf("k1"));
+ RowMutation rm = new RowMutation("Keyspace1", key);
+ rm.add("Standard1", ByteBufferUtil.bytes("xyz"), ByteBufferUtil.bytes("abc"), 0);
+ rm.apply();
+ store.forceBlockingFlush();
+ boolean foundScanner = false;
+ for (SSTableReader s : store.getSSTables())
+ {
+ ICompactionScanner scanner = s.getScanner(new Range<Token>(t(0), t(1), s.partitioner), null);
+ scanner.next(); // throws exception pre 5407
+ foundScanner = true;
+ }
+ assertTrue(foundScanner);
++
+ @Test
+ public void testGetPositionsForRangesFromTableOpenedForBulkLoading() throws IOException, ExecutionException, InterruptedException
+ {
+ Table table = Table.open("Keyspace1");
+ ColumnFamilyStore store = table.getColumnFamilyStore("Standard2");
+
+ // insert data and compact to a single sstable. The
+ // number of keys inserted is greater than index_interval
+ // to ensure multiple segments in the index file
+ CompactionManager.instance.disableAutoCompaction();
+ for (int j = 0; j < 130; j++)
+ {
+ ByteBuffer key = ByteBufferUtil.bytes(String.valueOf(j));
+ RowMutation rm = new RowMutation("Keyspace1", key);
+ rm.add(new QueryPath("Standard2", null, ByteBufferUtil.bytes("0")), ByteBufferUtil.EMPTY_BYTE_BUFFER, j);
+ rm.apply();
+ }
+ store.forceBlockingFlush();
+ CompactionManager.instance.performMaximal(store);
+
+ // construct a range which is present in the sstable, but whose
+ // keys are not found in the first segment of the index.
+ List<Range<Token>> ranges = new ArrayList<Range<Token>>();
+ ranges.add(new Range<Token>(t(98), t(99)));
+
+ SSTableReader sstable = store.getSSTables().iterator().next();
+ List<Pair<Long,Long>> sections = sstable.getPositionsForRanges(ranges);
+ assert sections.size() == 1 : "Expected to find range in sstable" ;
+
+ // re-open the same sstable as it would be during bulk loading
+ Set<Component> components = Sets.newHashSet(Component.DATA, Component.PRIMARY_INDEX);
+ SSTableReader bulkLoaded = SSTableReader.openForBatch(sstable.descriptor, components, sstable.partitioner);
+ sections = bulkLoaded.getPositionsForRanges(ranges);
+ assert sections.size() == 1 : "Expected to find range in sstable opened for bulk loading";
}
private void assertIndexQueryWorks(ColumnFamilyStore indexedCFS) throws IOException
[2/6] git commit: Use non-pooled SegmentedFile builders when bulk
loading Patch by Sam Tunnicliffe,
reviewed by brandonwilliams for CASSANDRA-6272
Posted by br...@apache.org.
Use non-pooled SegmentedFile builders when bulk loading
Patch by Sam Tunnicliffe, reviewed by brandonwilliams for CASSANDRA-6272
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/18f79353
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/18f79353
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/18f79353
Branch: refs/heads/cassandra-2.0
Commit: 18f7935370b1628cba8fa35819c05f235ba918aa
Parents: 620bb80
Author: Brandon Williams <br...@apache.org>
Authored: Wed Oct 30 14:01:44 2013 -0500
Committer: Brandon Williams <br...@apache.org>
Committed: Wed Oct 30 14:03:08 2013 -0500
----------------------------------------------------------------------
.../cassandra/io/sstable/SSTableReader.java | 44 ++++++--------------
.../cassandra/io/sstable/SSTableReaderTest.java | 38 +++++++++++++++++
2 files changed, 50 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/18f79353/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
index ed221d9..4af1cdc 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
@@ -166,8 +166,19 @@ public class SSTableReader extends SSTable implements Closeable
partitioner,
System.currentTimeMillis(),
sstableMetadata);
+
+ // special implementation of load to use non-pooled SegmentedFile builders
+ SegmentedFile.Builder ibuilder = new BufferedSegmentedFile.Builder();
+ SegmentedFile.Builder dbuilder = sstable.compression
+ ? new CompressedSegmentedFile.Builder()
+ : new BufferedSegmentedFile.Builder();
+
+ if (!loadSummary(sstable, ibuilder, dbuilder))
+ sstable.buildSummary(false, ibuilder, dbuilder, false);
+ sstable.ifile = ibuilder.complete(sstable.descriptor.filenameFor(Component.PRIMARY_INDEX));
+ sstable.dfile = dbuilder.complete(sstable.descriptor.filenameFor(Component.DATA));
+
sstable.bf = new AlwaysPresentFilter();
- sstable.loadForBatch();
return sstable;
}
@@ -409,37 +420,6 @@ public class SSTableReader extends SSTable implements Closeable
saveSummary(this, ibuilder, dbuilder);
}
- /**
- * A simplified load that creates a minimal partition index
- */
- private void loadForBatch() throws IOException
- {
- SegmentedFile.Builder ibuilder = new BufferedSegmentedFile.Builder();
- SegmentedFile.Builder dbuilder = compression
- ? new CompressedSegmentedFile.Builder()
- : new BufferedSegmentedFile.Builder();
-
- // build a bare-bones IndexSummary
- IndexSummaryBuilder summaryBuilder = new IndexSummaryBuilder(1);
- RandomAccessReader in = RandomAccessReader.open(new File(descriptor.filenameFor(Component.PRIMARY_INDEX)), true);
- try
- {
- ByteBuffer key = ByteBufferUtil.readWithShortLength(in);
- first = decodeKey(partitioner, descriptor, key);
- summaryBuilder.maybeAddEntry(first, 0);
- indexSummary = summaryBuilder.build(partitioner);
- }
- finally
- {
- FileUtils.closeQuietly(in);
- }
-
- last = null; // shouldn't need this for batch operations
-
- ifile = ibuilder.complete(descriptor.filenameFor(Component.PRIMARY_INDEX));
- dfile = dbuilder.complete(descriptor.filenameFor(Component.DATA));
- }
-
private void buildSummary(boolean recreatebloom, SegmentedFile.Builder ibuilder, SegmentedFile.Builder dbuilder, boolean summaryLoaded) throws IOException
{
// we read the positions in a BRAF so we don't have to worry about an entry spanning a mmap boundary.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/18f79353/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
index 02b6855..fef27f1 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
@@ -27,8 +27,10 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.Set;
import java.util.concurrent.ExecutionException;
+import com.google.common.collect.Sets;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -315,6 +317,42 @@ public class SSTableReaderTest extends SchemaLoader
assert reopened.first.token instanceof LocalToken;
}
+ @Test
+ public void testGetPositionsForRangesFromTableOpenedForBulkLoading() throws IOException, ExecutionException, InterruptedException
+ {
+ Table table = Table.open("Keyspace1");
+ ColumnFamilyStore store = table.getColumnFamilyStore("Standard2");
+
+ // insert data and compact to a single sstable. The
+ // number of keys inserted is greater than index_interval
+ // to ensure multiple segments in the index file
+ CompactionManager.instance.disableAutoCompaction();
+ for (int j = 0; j < 130; j++)
+ {
+ ByteBuffer key = ByteBufferUtil.bytes(String.valueOf(j));
+ RowMutation rm = new RowMutation("Keyspace1", key);
+ rm.add(new QueryPath("Standard2", null, ByteBufferUtil.bytes("0")), ByteBufferUtil.EMPTY_BYTE_BUFFER, j);
+ rm.apply();
+ }
+ store.forceBlockingFlush();
+ CompactionManager.instance.performMaximal(store);
+
+ // construct a range which is present in the sstable, but whose
+ // keys are not found in the first segment of the index.
+ List<Range<Token>> ranges = new ArrayList<Range<Token>>();
+ ranges.add(new Range<Token>(t(98), t(99)));
+
+ SSTableReader sstable = store.getSSTables().iterator().next();
+ List<Pair<Long,Long>> sections = sstable.getPositionsForRanges(ranges);
+ assert sections.size() == 1 : "Expected to find range in sstable" ;
+
+ // re-open the same sstable as it would be during bulk loading
+ Set<Component> components = Sets.newHashSet(Component.DATA, Component.PRIMARY_INDEX);
+ SSTableReader bulkLoaded = SSTableReader.openForBatch(sstable.descriptor, components, sstable.partitioner);
+ sections = bulkLoaded.getPositionsForRanges(ranges);
+ assert sections.size() == 1 : "Expected to find range in sstable opened for bulk loading";
+ }
+
private void assertIndexQueryWorks(ColumnFamilyStore indexedCFS) throws IOException
{
assert "Indexed1".equals(indexedCFS.getColumnFamilyName());
[6/6] git commit: Merge branch 'cassandra-2.0' into trunk
Posted by br...@apache.org.
Merge branch 'cassandra-2.0' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/58e2706d
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/58e2706d
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/58e2706d
Branch: refs/heads/trunk
Commit: 58e2706d3d66095802c3a03dee0bdfae0045b324
Parents: 45bb924 89911c8
Author: Brandon Williams <br...@apache.org>
Authored: Wed Oct 30 14:10:00 2013 -0500
Committer: Brandon Williams <br...@apache.org>
Committed: Wed Oct 30 14:10:00 2013 -0500
----------------------------------------------------------------------
.../cassandra/io/sstable/SSTableReaderTest.java | 37 ++++++++++++++++++++
1 file changed, 37 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/58e2706d/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
----------------------------------------------------------------------
[3/6] git commit: Use non-pooled SegmentedFile builders when bulk
loading Patch by Sam Tunnicliffe,
reviewed by brandonwilliams for CASSANDRA-6272
Posted by br...@apache.org.
Use non-pooled SegmentedFile builders when bulk loading
Patch by Sam Tunnicliffe, reviewed by brandonwilliams for CASSANDRA-6272
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/18f79353
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/18f79353
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/18f79353
Branch: refs/heads/trunk
Commit: 18f7935370b1628cba8fa35819c05f235ba918aa
Parents: 620bb80
Author: Brandon Williams <br...@apache.org>
Authored: Wed Oct 30 14:01:44 2013 -0500
Committer: Brandon Williams <br...@apache.org>
Committed: Wed Oct 30 14:03:08 2013 -0500
----------------------------------------------------------------------
.../cassandra/io/sstable/SSTableReader.java | 44 ++++++--------------
.../cassandra/io/sstable/SSTableReaderTest.java | 38 +++++++++++++++++
2 files changed, 50 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/18f79353/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
index ed221d9..4af1cdc 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
@@ -166,8 +166,19 @@ public class SSTableReader extends SSTable implements Closeable
partitioner,
System.currentTimeMillis(),
sstableMetadata);
+
+ // special implementation of load to use non-pooled SegmentedFile builders
+ SegmentedFile.Builder ibuilder = new BufferedSegmentedFile.Builder();
+ SegmentedFile.Builder dbuilder = sstable.compression
+ ? new CompressedSegmentedFile.Builder()
+ : new BufferedSegmentedFile.Builder();
+
+ if (!loadSummary(sstable, ibuilder, dbuilder))
+ sstable.buildSummary(false, ibuilder, dbuilder, false);
+ sstable.ifile = ibuilder.complete(sstable.descriptor.filenameFor(Component.PRIMARY_INDEX));
+ sstable.dfile = dbuilder.complete(sstable.descriptor.filenameFor(Component.DATA));
+
sstable.bf = new AlwaysPresentFilter();
- sstable.loadForBatch();
return sstable;
}
@@ -409,37 +420,6 @@ public class SSTableReader extends SSTable implements Closeable
saveSummary(this, ibuilder, dbuilder);
}
- /**
- * A simplified load that creates a minimal partition index
- */
- private void loadForBatch() throws IOException
- {
- SegmentedFile.Builder ibuilder = new BufferedSegmentedFile.Builder();
- SegmentedFile.Builder dbuilder = compression
- ? new CompressedSegmentedFile.Builder()
- : new BufferedSegmentedFile.Builder();
-
- // build a bare-bones IndexSummary
- IndexSummaryBuilder summaryBuilder = new IndexSummaryBuilder(1);
- RandomAccessReader in = RandomAccessReader.open(new File(descriptor.filenameFor(Component.PRIMARY_INDEX)), true);
- try
- {
- ByteBuffer key = ByteBufferUtil.readWithShortLength(in);
- first = decodeKey(partitioner, descriptor, key);
- summaryBuilder.maybeAddEntry(first, 0);
- indexSummary = summaryBuilder.build(partitioner);
- }
- finally
- {
- FileUtils.closeQuietly(in);
- }
-
- last = null; // shouldn't need this for batch operations
-
- ifile = ibuilder.complete(descriptor.filenameFor(Component.PRIMARY_INDEX));
- dfile = dbuilder.complete(descriptor.filenameFor(Component.DATA));
- }
-
private void buildSummary(boolean recreatebloom, SegmentedFile.Builder ibuilder, SegmentedFile.Builder dbuilder, boolean summaryLoaded) throws IOException
{
// we read the positions in a BRAF so we don't have to worry about an entry spanning a mmap boundary.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/18f79353/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
index 02b6855..fef27f1 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
@@ -27,8 +27,10 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.Set;
import java.util.concurrent.ExecutionException;
+import com.google.common.collect.Sets;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -315,6 +317,42 @@ public class SSTableReaderTest extends SchemaLoader
assert reopened.first.token instanceof LocalToken;
}
+ @Test
+ public void testGetPositionsForRangesFromTableOpenedForBulkLoading() throws IOException, ExecutionException, InterruptedException
+ {
+ Table table = Table.open("Keyspace1");
+ ColumnFamilyStore store = table.getColumnFamilyStore("Standard2");
+
+ // insert data and compact to a single sstable. The
+ // number of keys inserted is greater than index_interval
+ // to ensure multiple segments in the index file
+ CompactionManager.instance.disableAutoCompaction();
+ for (int j = 0; j < 130; j++)
+ {
+ ByteBuffer key = ByteBufferUtil.bytes(String.valueOf(j));
+ RowMutation rm = new RowMutation("Keyspace1", key);
+ rm.add(new QueryPath("Standard2", null, ByteBufferUtil.bytes("0")), ByteBufferUtil.EMPTY_BYTE_BUFFER, j);
+ rm.apply();
+ }
+ store.forceBlockingFlush();
+ CompactionManager.instance.performMaximal(store);
+
+ // construct a range which is present in the sstable, but whose
+ // keys are not found in the first segment of the index.
+ List<Range<Token>> ranges = new ArrayList<Range<Token>>();
+ ranges.add(new Range<Token>(t(98), t(99)));
+
+ SSTableReader sstable = store.getSSTables().iterator().next();
+ List<Pair<Long,Long>> sections = sstable.getPositionsForRanges(ranges);
+ assert sections.size() == 1 : "Expected to find range in sstable" ;
+
+ // re-open the same sstable as it would be during bulk loading
+ Set<Component> components = Sets.newHashSet(Component.DATA, Component.PRIMARY_INDEX);
+ SSTableReader bulkLoaded = SSTableReader.openForBatch(sstable.descriptor, components, sstable.partitioner);
+ sections = bulkLoaded.getPositionsForRanges(ranges);
+ assert sections.size() == 1 : "Expected to find range in sstable opened for bulk loading";
+ }
+
private void assertIndexQueryWorks(ColumnFamilyStore indexedCFS) throws IOException
{
assert "Indexed1".equals(indexedCFS.getColumnFamilyName());