You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by vi...@apache.org on 2013/08/22 04:24:36 UTC
[1/3] git commit: additional check for offheap bloom filter size
patch by Vijay; reviewed by jbellis for CASSANDRA-5903
Updated Branches:
refs/heads/cassandra-2.0 f3bb8237a -> 7b9ae6a03
additional check for offheap bloom filter size
patch by Vijay; reviewed by jbellis for CASSANDRA-5903
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/9bb4d93e
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/9bb4d93e
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/9bb4d93e
Branch: refs/heads/cassandra-2.0
Commit: 9bb4d93e3c8471012a39cbb0a1be5b082115a42b
Parents: 4bc8c89
Author: Vijay Parthasarathy <vi...@gmail.com>
Authored: Wed Aug 21 19:21:28 2013 -0700
Committer: Vijay Parthasarathy <vi...@gmail.com>
Committed: Wed Aug 21 19:21:28 2013 -0700
----------------------------------------------------------------------
.../cassandra/utils/obs/OffHeapBitSet.java | 5 ++-
.../apache/cassandra/utils/BloomFilterTest.java | 36 ++++++++++++++++++++
2 files changed, 40 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/9bb4d93e/src/java/org/apache/cassandra/utils/obs/OffHeapBitSet.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/utils/obs/OffHeapBitSet.java b/src/java/org/apache/cassandra/utils/obs/OffHeapBitSet.java
index fb32043..c103835 100644
--- a/src/java/org/apache/cassandra/utils/obs/OffHeapBitSet.java
+++ b/src/java/org/apache/cassandra/utils/obs/OffHeapBitSet.java
@@ -36,9 +36,12 @@ public class OffHeapBitSet implements IBitSet
public OffHeapBitSet(long numBits)
{
// OpenBitSet.bits2words calculation is there for backward compatibility.
- long byteCount = OpenBitSet.bits2words(numBits) * 8L;
+ long wordCount = OpenBitSet.bits2words(numBits);
+ if (wordCount > Integer.MAX_VALUE)
+ throw new UnsupportedOperationException("Bloom filter size is > 16GB, reduce the bloom_filter_fp_chance");
try
{
+ long byteCount = wordCount * 8L;
bytes = RefCountedMemory.allocate(byteCount);
}
catch (OutOfMemoryError e)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/9bb4d93e/test/unit/org/apache/cassandra/utils/BloomFilterTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/utils/BloomFilterTest.java b/test/unit/org/apache/cassandra/utils/BloomFilterTest.java
index bb0865f..562e3ea 100644
--- a/test/unit/org/apache/cassandra/utils/BloomFilterTest.java
+++ b/test/unit/org/apache/cassandra/utils/BloomFilterTest.java
@@ -20,16 +20,25 @@ package org.apache.cassandra.utils;
import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;
+import junit.framework.Assert;
+
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Test;
import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.FilterFactory.Type;
public class BloomFilterTest
{
@@ -138,4 +147,31 @@ public class BloomFilterTest
{
testManyHashes(FilterTestHelper.randomKeys());
}
+
+ @Test(expected = UnsupportedOperationException.class)
+ public void testOffHeapException()
+ {
+ long numKeys = (Integer.MAX_VALUE * 64) + 1; // approx 128 Billion
+ FilterFactory.getFilter(numKeys, 0.01d, true);
+ }
+
+ @Test
+ @Ignore
+ public void testHugeBFSerialization() throws IOException
+ {
+ ByteBuffer test = ByteBuffer.wrap(new byte[] {0, 1});
+
+ File file = FileUtils.createTempFile("bloomFilterTest-", ".dat");
+ BloomFilter filter = (BloomFilter) FilterFactory.getFilter(((long)Integer.MAX_VALUE / 8) + 1, 0.01d, true);
+ filter.add(test);
+ DataOutputStream out = new DataOutputStream(new FileOutputStream(file));
+ FilterFactory.serialize(filter, out);
+ filter.bitset.serialize(out);
+ out.close();
+
+ DataInputStream in = new DataInputStream(new FileInputStream(file));
+ BloomFilter filter2 = (BloomFilter) FilterFactory.deserialize(in, Type.MURMUR3, true);
+ Assert.assertTrue(filter2.isPresent(test));
+ FileUtils.closeQuietly(in);
+ }
}
[3/3] git commit: Merge branch 'cassandra-2.0.0' into cassandra-2.0
Posted by vi...@apache.org.
Merge branch 'cassandra-2.0.0' into cassandra-2.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/7b9ae6a0
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/7b9ae6a0
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/7b9ae6a0
Branch: refs/heads/cassandra-2.0
Commit: 7b9ae6a0375d52d18944ad9274f4c5f01ca5a42a
Parents: f3bb823 ed6a5b9
Author: Vijay Parthasarathy <vi...@gmail.com>
Authored: Wed Aug 21 19:24:28 2013 -0700
Committer: Vijay Parthasarathy <vi...@gmail.com>
Committed: Wed Aug 21 19:24:28 2013 -0700
----------------------------------------------------------------------
.../cassandra/utils/obs/OffHeapBitSet.java | 5 ++-
.../apache/cassandra/utils/BloomFilterTest.java | 36 ++++++++++++++++++++
2 files changed, 40 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
[2/3] git commit: Merge branch 'cassandra-1.2' into cassandra-2.0.0
Posted by vi...@apache.org.
Merge branch 'cassandra-1.2' into cassandra-2.0.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/ed6a5b93
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/ed6a5b93
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/ed6a5b93
Branch: refs/heads/cassandra-2.0
Commit: ed6a5b935d164bb1e0a106886f09dca47517f556
Parents: 372af08 9bb4d93
Author: Vijay Parthasarathy <vi...@gmail.com>
Authored: Wed Aug 21 19:23:53 2013 -0700
Committer: Vijay Parthasarathy <vi...@gmail.com>
Committed: Wed Aug 21 19:23:53 2013 -0700
----------------------------------------------------------------------
.../cassandra/utils/obs/OffHeapBitSet.java | 5 ++-
.../apache/cassandra/utils/BloomFilterTest.java | 36 ++++++++++++++++++++
2 files changed, 40 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ed6a5b93/src/java/org/apache/cassandra/utils/obs/OffHeapBitSet.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ed6a5b93/test/unit/org/apache/cassandra/utils/BloomFilterTest.java
----------------------------------------------------------------------