You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ja...@apache.org on 2018/01/19 14:42:46 UTC
cassandra git commit: New Bloomfilter format without changing the
byte ordering
Repository: cassandra
Updated Branches:
refs/heads/trunk 716d08f90 -> d6cc59432
New Bloomfilter format without changing the byte ordering
patch by Jay Zhuang; reviewed by jasobrown for CASSANDRA-9067
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/d6cc5943
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/d6cc5943
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/d6cc5943
Branch: refs/heads/trunk
Commit: d6cc5943250a7c19adb42ef86dc9a186d4e52166
Parents: 716d08f
Author: Jay Zhuang <ja...@yahoo.com>
Authored: Sun Jan 14 20:42:57 2018 -0800
Committer: Jason Brown <ja...@gmail.com>
Committed: Fri Jan 19 06:40:50 2018 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../io/sstable/format/SSTableReader.java | 8 +-
.../cassandra/io/sstable/format/Version.java | 7 ++
.../io/sstable/format/big/BigFormat.java | 14 ++-
.../io/sstable/format/big/BigTableWriter.java | 2 +-
.../cassandra/utils/BloomFilterSerializer.java | 8 +-
.../apache/cassandra/utils/FilterFactory.java | 14 ---
.../org/apache/cassandra/utils/obs/IBitSet.java | 4 +-
.../cassandra/utils/obs/OffHeapBitSet.java | 60 +++++++----
.../legacy_na_clust/na-1-big-CompressionInfo.db | Bin 87 -> 87 bytes
.../legacy_na_clust/na-1-big-Data.db | Bin 5215 -> 5259 bytes
.../legacy_na_clust/na-1-big-Digest.crc32 | 2 +-
.../legacy_na_clust/na-1-big-Filter.db | Bin 24 -> 24 bytes
.../legacy_na_clust/na-1-big-Index.db | Bin 157553 -> 157553 bytes
.../legacy_na_clust/na-1-big-Statistics.db | Bin 7095 -> 7095 bytes
.../legacy_na_clust/na-1-big-TOC.txt | 6 +-
.../na-1-big-CompressionInfo.db | Bin 79 -> 79 bytes
.../legacy_na_clust_counter/na-1-big-Data.db | Bin 5781 -> 5888 bytes
.../na-1-big-Digest.crc32 | 2 +-
.../legacy_na_clust_counter/na-1-big-Filter.db | Bin 24 -> 24 bytes
.../legacy_na_clust_counter/na-1-big-Index.db | Bin 157553 -> 157553 bytes
.../na-1-big-Statistics.db | Bin 7104 -> 7104 bytes
.../legacy_na_clust_counter/na-1-big-TOC.txt | 6 +-
.../legacy_na_simple/na-1-big-Data.db | Bin 89 -> 89 bytes
.../legacy_na_simple/na-1-big-Digest.crc32 | 2 +-
.../legacy_na_simple/na-1-big-Filter.db | Bin 24 -> 24 bytes
.../legacy_na_simple/na-1-big-Statistics.db | Bin 4648 -> 4648 bytes
.../legacy_na_simple/na-1-big-TOC.txt | 6 +-
.../legacy_na_simple_counter/na-1-big-Data.db | Bin 146 -> 140 bytes
.../na-1-big-Digest.crc32 | 2 +-
.../legacy_na_simple_counter/na-1-big-Filter.db | Bin 24 -> 24 bytes
.../na-1-big-Statistics.db | Bin 4657 -> 4657 bytes
.../legacy_na_simple_counter/na-1-big-TOC.txt | 6 +-
.../serialization/4.0/utils.BloomFilter1000.bin | Bin 0 -> 2500016 bytes
.../microbench/BloomFilterSerializerBench.java | 99 +++++++++++++++++++
.../cassandra/io/sstable/LegacySSTableTest.java | 3 +-
.../apache/cassandra/utils/BloomFilterTest.java | 24 +++--
.../cassandra/utils/SerializationsTest.java | 49 +++++++--
.../cassandra/utils/obs/OffHeapBitSetTest.java | 21 ++--
39 files changed, 257 insertions(+), 89 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 65a7a40..fd0acb9 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
4.0
+ * BloomFilter serialization format should not change byte ordering (CASSANDRA-9067)
* Remove unused on-heap BloomFilter implementation (CASSANDRA-14152)
* Delete temp test files on exit (CASSANDRA-14153)
* Make PartitionUpdate and Mutation immutable (CASSANDRA-13867)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
index 6722e1b..757b9a8 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
@@ -75,6 +75,7 @@ import org.apache.cassandra.utils.*;
import org.apache.cassandra.utils.concurrent.OpOrder;
import org.apache.cassandra.utils.concurrent.Ref;
import org.apache.cassandra.utils.concurrent.SelfRefCounted;
+import org.apache.cassandra.utils.BloomFilterSerializer;
import static org.apache.cassandra.db.Directories.SECONDARY_INDEX_NAME_SEPARATOR;
@@ -728,7 +729,7 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
{
// bf is enabled and fp chance matches the currently configured value.
load(false, true);
- loadBloomFilter();
+ loadBloomFilter(descriptor.version.hasOldBfFormat());
}
}
@@ -736,12 +737,13 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
* Load bloom filter from Filter.db file.
*
* @throws IOException
+ * @param oldBfFormat
*/
- private void loadBloomFilter() throws IOException
+ private void loadBloomFilter(boolean oldBfFormat) throws IOException
{
try (DataInputStream stream = new DataInputStream(new BufferedInputStream(Files.newInputStream(Paths.get(descriptor.filenameFor(Component.FILTER))))))
{
- bf = FilterFactory.deserialize(stream);
+ bf = BloomFilterSerializer.deserialize(stream, oldBfFormat);
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/src/java/org/apache/cassandra/io/sstable/format/Version.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/Version.java b/src/java/org/apache/cassandra/io/sstable/format/Version.java
index e8721a6..1d965ce 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/Version.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/Version.java
@@ -57,6 +57,13 @@ public abstract class Version
public abstract boolean hasMetadataChecksum();
+ /**
+ * The old bloomfilter format serializes the data as BIG_ENDIAN long's, the new one uses the
+ * same format as in memory (serializes as bytes).
+ * @return True if the bloomfilter file is old serialization format
+ */
+ public abstract boolean hasOldBfFormat();
+
public String getVersion()
{
return version;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
index 5f6dd53..db73b4f 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
@@ -120,7 +120,7 @@ public class BigFormat implements SSTableFormat
// mb (3.0.7, 3.7): commit log lower bound included
// mc (3.0.8, 3.9): commit log intervals included
- // na (4.0.0): uncompressed chunks, pending repair session, checksummed sstable metadata file
+ // na (4.0.0): uncompressed chunks, pending repair session, checksummed sstable metadata file, new Bloomfilter format
//
// NOTE: when adding a new version, please add that to LegacySSTableTest, too.
@@ -131,6 +131,11 @@ public class BigFormat implements SSTableFormat
public final boolean hasMaxCompressedLength;
private final boolean hasPendingRepair;
private final boolean hasMetadataChecksum;
+ /**
+ * CASSANDRA-9067: 4.0 bloom filter representation changed (two longs just swapped)
+ * have no 'static' bits caused by using the same upper bits for both bloom filter and token distribution.
+ */
+ private final boolean hasOldBfFormat;
BigVersion(String version)
{
@@ -144,6 +149,7 @@ public class BigFormat implements SSTableFormat
hasMaxCompressedLength = version.compareTo("na") >= 0;
hasPendingRepair = version.compareTo("na") >= 0;
hasMetadataChecksum = version.compareTo("na") >= 0;
+ hasOldBfFormat = version.compareTo("na") < 0;
}
@Override
@@ -197,5 +203,11 @@ public class BigFormat implements SSTableFormat
{
return hasMaxCompressedLength;
}
+
+ @Override
+ public boolean hasOldBfFormat()
+ {
+ return hasOldBfFormat;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
index 04c7bbf..b5488ed 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
@@ -484,7 +484,7 @@ public class BigTableWriter extends SSTableWriter
DataOutputStreamPlus stream = new BufferedDataOutputStreamPlus(fos))
{
// bloom filter
- FilterFactory.serialize(bf, stream);
+ BloomFilterSerializer.serialize((BloomFilter) bf, stream);
stream.flush();
SyncUtil.sync(fos);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/src/java/org/apache/cassandra/utils/BloomFilterSerializer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/utils/BloomFilterSerializer.java b/src/java/org/apache/cassandra/utils/BloomFilterSerializer.java
index 9e9d15a..d3c08b5 100644
--- a/src/java/org/apache/cassandra/utils/BloomFilterSerializer.java
+++ b/src/java/org/apache/cassandra/utils/BloomFilterSerializer.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.utils;
-import java.io.DataInput;
+import java.io.DataInputStream;
import java.io.IOException;
import org.apache.cassandra.db.TypeSizes;
@@ -25,7 +25,7 @@ import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.utils.obs.IBitSet;
import org.apache.cassandra.utils.obs.OffHeapBitSet;
-final class BloomFilterSerializer
+public final class BloomFilterSerializer
{
private BloomFilterSerializer()
{
@@ -38,10 +38,10 @@ final class BloomFilterSerializer
}
@SuppressWarnings("resource")
- public static BloomFilter deserialize(DataInput in) throws IOException
+ public static BloomFilter deserialize(DataInputStream in, boolean oldBfFormat) throws IOException
{
int hashes = in.readInt();
- IBitSet bs = OffHeapBitSet.deserialize(in);
+ IBitSet bs = OffHeapBitSet.deserialize(in, oldBfFormat);
return new BloomFilter(hashes, bs);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/src/java/org/apache/cassandra/utils/FilterFactory.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/utils/FilterFactory.java b/src/java/org/apache/cassandra/utils/FilterFactory.java
index 9479452..4cf0cbf 100644
--- a/src/java/org/apache/cassandra/utils/FilterFactory.java
+++ b/src/java/org/apache/cassandra/utils/FilterFactory.java
@@ -17,13 +17,9 @@
*/
package org.apache.cassandra.utils;
-import java.io.DataInput;
-import java.io.IOException;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.utils.obs.IBitSet;
import org.apache.cassandra.utils.obs.OffHeapBitSet;
@@ -34,16 +30,6 @@ public class FilterFactory
private static final Logger logger = LoggerFactory.getLogger(FilterFactory.class);
private static final long BITSET_EXCESS = 20;
- public static void serialize(IFilter bf, DataOutputPlus output) throws IOException
- {
- BloomFilterSerializer.serialize((BloomFilter) bf, output);
- }
-
- public static IFilter deserialize(DataInput input) throws IOException
- {
- return BloomFilterSerializer.deserialize(input);
- }
-
/**
* @return A BloomFilter with the lowest practical false positive
* probability for the given number of elements.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/src/java/org/apache/cassandra/utils/obs/IBitSet.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/utils/obs/IBitSet.java b/src/java/org/apache/cassandra/utils/obs/IBitSet.java
index 15ff361..b262cf5 100644
--- a/src/java/org/apache/cassandra/utils/obs/IBitSet.java
+++ b/src/java/org/apache/cassandra/utils/obs/IBitSet.java
@@ -18,9 +18,9 @@
package org.apache.cassandra.utils.obs;
import java.io.Closeable;
-import java.io.DataOutput;
import java.io.IOException;
+import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.utils.concurrent.Ref;
public interface IBitSet extends Closeable
@@ -44,7 +44,7 @@ public interface IBitSet extends Closeable
*/
public void clear(long index);
- public void serialize(DataOutput out) throws IOException;
+ public void serialize(DataOutputPlus out) throws IOException;
public long serializedSize();
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/src/java/org/apache/cassandra/utils/obs/OffHeapBitSet.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/utils/obs/OffHeapBitSet.java b/src/java/org/apache/cassandra/utils/obs/OffHeapBitSet.java
index d2c15ca..486ec38 100644
--- a/src/java/org/apache/cassandra/utils/obs/OffHeapBitSet.java
+++ b/src/java/org/apache/cassandra/utils/obs/OffHeapBitSet.java
@@ -18,11 +18,17 @@
package org.apache.cassandra.utils.obs;
import java.io.DataInput;
+import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.IOException;
+import com.google.common.annotations.VisibleForTesting;
+
import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.io.util.Memory;
+import org.apache.cassandra.io.util.MemoryOutputStream;
+import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.concurrent.Ref;
/**
@@ -109,19 +115,26 @@ public class OffHeapBitSet implements IBitSet
bytes.setMemory(0, bytes.size(), (byte) 0);
}
- public void serialize(DataOutput out) throws IOException
+ public void serialize(DataOutputPlus out) throws IOException
{
out.writeInt((int) (bytes.size() / 8));
- for (long i = 0; i < bytes.size();)
+ out.write(bytes, 0, bytes.size());
+ }
+
+ @VisibleForTesting
+ public void serializeOldBfFormat(DataOutputPlus out) throws IOException
+ {
+ out.writeInt((int) (bytes.size() / 8));
+ for (long i = 0; i < bytes.size(); )
{
long value = ((bytes.getByte(i++) & 0xff) << 0)
- + ((bytes.getByte(i++) & 0xff) << 8)
- + ((bytes.getByte(i++) & 0xff) << 16)
- + ((long) (bytes.getByte(i++) & 0xff) << 24)
- + ((long) (bytes.getByte(i++) & 0xff) << 32)
- + ((long) (bytes.getByte(i++) & 0xff) << 40)
- + ((long) (bytes.getByte(i++) & 0xff) << 48)
- + ((long) bytes.getByte(i++) << 56);
+ + ((bytes.getByte(i++) & 0xff) << 8)
+ + ((bytes.getByte(i++) & 0xff) << 16)
+ + ((long) (bytes.getByte(i++) & 0xff) << 24)
+ + ((long) (bytes.getByte(i++) & 0xff) << 32)
+ + ((long) (bytes.getByte(i++) & 0xff) << 40)
+ + ((long) (bytes.getByte(i++) & 0xff) << 48)
+ + ((long) bytes.getByte(i++) << 56);
out.writeLong(value);
}
}
@@ -132,21 +145,28 @@ public class OffHeapBitSet implements IBitSet
}
@SuppressWarnings("resource")
- public static OffHeapBitSet deserialize(DataInput in) throws IOException
+ public static OffHeapBitSet deserialize(DataInputStream in, boolean oldBfFormat) throws IOException
{
long byteCount = in.readInt() * 8L;
Memory memory = Memory.allocate(byteCount);
- for (long i = 0; i < byteCount;)
+ if (oldBfFormat)
+ {
+ for (long i = 0; i < byteCount; )
+ {
+ long v = in.readLong();
+ memory.setByte(i++, (byte) (v >>> 0));
+ memory.setByte(i++, (byte) (v >>> 8));
+ memory.setByte(i++, (byte) (v >>> 16));
+ memory.setByte(i++, (byte) (v >>> 24));
+ memory.setByte(i++, (byte) (v >>> 32));
+ memory.setByte(i++, (byte) (v >>> 40));
+ memory.setByte(i++, (byte) (v >>> 48));
+ memory.setByte(i++, (byte) (v >>> 56));
+ }
+ }
+ else
{
- long v = in.readLong();
- memory.setByte(i++, (byte) (v >>> 0));
- memory.setByte(i++, (byte) (v >>> 8));
- memory.setByte(i++, (byte) (v >>> 16));
- memory.setByte(i++, (byte) (v >>> 24));
- memory.setByte(i++, (byte) (v >>> 32));
- memory.setByte(i++, (byte) (v >>> 40));
- memory.setByte(i++, (byte) (v >>> 48));
- memory.setByte(i++, (byte) (v >>> 56));
+ FBUtilities.copy(in, new MemoryOutputStream(memory), byteCount);
}
return new OffHeapBitSet(memory);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-CompressionInfo.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-CompressionInfo.db
index 935d863..ceaa5a3 100644
Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-CompressionInfo.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-CompressionInfo.db differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Data.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Data.db
index 9982578..6968720 100644
Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Data.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Data.db differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Digest.crc32
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Digest.crc32
index 1e6cbfc..f1c192b 100644
--- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Digest.crc32
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Digest.crc32
@@ -1 +1 @@
-4285275084
\ No newline at end of file
+4004129384
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Filter.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Filter.db
index 2e1d5d2..8868e5c 100644
Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Filter.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Filter.db differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Index.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Index.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Index.db
index 25b063c..af16195 100644
Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Index.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Index.db differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Statistics.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Statistics.db
index 59969c2..970e385 100644
Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Statistics.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Statistics.db differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-TOC.txt
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-TOC.txt
index 734a80d..bb800f8 100644
--- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-TOC.txt
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-TOC.txt
@@ -1,8 +1,8 @@
-Index.db
-TOC.txt
+Digest.crc32
Filter.db
CompressionInfo.db
+Index.db
Summary.db
Data.db
+TOC.txt
Statistics.db
-Digest.crc32
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-CompressionInfo.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-CompressionInfo.db
index d6a1ff8..f5ad4d0 100644
Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-CompressionInfo.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-CompressionInfo.db differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Data.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Data.db
index 03babd4..7217716 100644
Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Data.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Data.db differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Digest.crc32
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Digest.crc32
index 7bd9e39..4f1391a 100644
--- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Digest.crc32
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Digest.crc32
@@ -1 +1 @@
-1093063834
\ No newline at end of file
+4072239034
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Filter.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Filter.db
index 2e1d5d2..8868e5c 100644
Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Filter.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Filter.db differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Index.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Index.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Index.db
index 5ad5400..6dd3da6 100644
Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Index.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Index.db differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Statistics.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Statistics.db
index 9347a2c..3a0e63f 100644
Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Statistics.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Statistics.db differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-TOC.txt
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-TOC.txt
index 734a80d..bb800f8 100644
--- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-TOC.txt
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-TOC.txt
@@ -1,8 +1,8 @@
-Index.db
-TOC.txt
+Digest.crc32
Filter.db
CompressionInfo.db
+Index.db
Summary.db
Data.db
+TOC.txt
Statistics.db
-Digest.crc32
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Data.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Data.db
index 414ff07..c665dfb 100644
Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Data.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Data.db differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Digest.crc32
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Digest.crc32
index 1dfec4f..c6c24a7 100644
--- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Digest.crc32
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Digest.crc32
@@ -1 +1 @@
-2354437953
\ No newline at end of file
+3772296151
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Filter.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Filter.db
index 2e1d5d2..8868e5c 100644
Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Filter.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Filter.db differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Statistics.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Statistics.db
index 33f0516..6741430 100644
Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Statistics.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Statistics.db differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-TOC.txt
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-TOC.txt
index 734a80d..bb800f8 100644
--- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-TOC.txt
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-TOC.txt
@@ -1,8 +1,8 @@
-Index.db
-TOC.txt
+Digest.crc32
Filter.db
CompressionInfo.db
+Index.db
Summary.db
Data.db
+TOC.txt
Statistics.db
-Digest.crc32
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Data.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Data.db
index ed0dadb..d9fe576 100644
Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Data.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Data.db differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Digest.crc32
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Digest.crc32
index 1480f5d..de7baed 100644
--- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Digest.crc32
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Digest.crc32
@@ -1 +1 @@
-2023874595
\ No newline at end of file
+4035692752
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Filter.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Filter.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Filter.db
index 2e1d5d2..8868e5c 100644
Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Filter.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Filter.db differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Statistics.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Statistics.db
index 124dacd..e9556d1 100644
Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Statistics.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Statistics.db differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-TOC.txt
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-TOC.txt
index 734a80d..bb800f8 100644
--- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-TOC.txt
+++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-TOC.txt
@@ -1,8 +1,8 @@
-Index.db
-TOC.txt
+Digest.crc32
Filter.db
CompressionInfo.db
+Index.db
Summary.db
Data.db
+TOC.txt
Statistics.db
-Digest.crc32
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/data/serialization/4.0/utils.BloomFilter1000.bin
----------------------------------------------------------------------
diff --git a/test/data/serialization/4.0/utils.BloomFilter1000.bin b/test/data/serialization/4.0/utils.BloomFilter1000.bin
new file mode 100644
index 0000000..b1bfe99
Binary files /dev/null and b/test/data/serialization/4.0/utils.BloomFilter1000.bin differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/microbench/org/apache/cassandra/test/microbench/BloomFilterSerializerBench.java
----------------------------------------------------------------------
diff --git a/test/microbench/org/apache/cassandra/test/microbench/BloomFilterSerializerBench.java b/test/microbench/org/apache/cassandra/test/microbench/BloomFilterSerializerBench.java
new file mode 100644
index 0000000..9222811
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/BloomFilterSerializerBench.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench;
+
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.db.BufferDecoratedKey;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.io.util.BufferedDataOutputStreamPlus;
+import org.apache.cassandra.io.util.DataOutputStreamPlus;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.BloomFilter;
+import org.apache.cassandra.utils.BloomFilterSerializer;
+import org.apache.cassandra.utils.FilterFactory;
+import org.apache.cassandra.utils.IFilter;
+import org.apache.cassandra.utils.SerializationsTest;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MICROSECONDS)
+@Warmup(iterations = 1, time = 1, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 2, time = 4, timeUnit = TimeUnit.SECONDS)
+@Fork(value = 2)
+@State(Scope.Benchmark)
+public class BloomFilterSerializerBench
+{
+
+ @Param({"1", "10", "100", "1024"})
+ private long numElemsInK;
+
+ @Param({"true", "false"})
+ public boolean oldBfFormat;
+
+ static final IFilter.FilterKey wrap(ByteBuffer buf)
+ {
+ return new BufferDecoratedKey(new Murmur3Partitioner.LongToken(0L), buf);
+ }
+
+ private ByteBuffer testVal = ByteBuffer.wrap(new byte[] { 0, 1});
+
+ @Benchmark
+ public void serializationTest() throws IOException
+ {
+ File file = FileUtils.createTempFile("bloomFilterTest-", ".dat");
+ try
+ {
+ BloomFilter filter = (BloomFilter) FilterFactory.getFilter(numElemsInK * 1024, 0.01d);
+ filter.add(wrap(testVal));
+ DataOutputStreamPlus out = new BufferedDataOutputStreamPlus(new FileOutputStream(file));
+ if (oldBfFormat)
+ SerializationsTest.serializeOldBfFormat(filter, out);
+ else
+ BloomFilterSerializer.serialize(filter, out);
+ out.close();
+ filter.close();
+
+ DataInputStream in = new DataInputStream(new FileInputStream(file));
+ BloomFilter filter2 = BloomFilterSerializer.deserialize(in, oldBfFormat);
+ FileUtils.closeQuietly(in);
+ filter2.close();
+ }
+ finally
+ {
+ file.delete();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
index 4892184..6e7d173 100644
--- a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
@@ -94,7 +94,7 @@ public class LegacySSTableTest
{
String scp = System.getProperty(LEGACY_SSTABLE_PROP);
Assert.assertNotNull("System property " + LEGACY_SSTABLE_ROOT + " not set", scp);
-
+
LEGACY_SSTABLE_ROOT = new File(scp).getAbsoluteFile();
Assert.assertTrue("System property " + LEGACY_SSTABLE_ROOT + " does not specify a directory", LEGACY_SSTABLE_ROOT.isDirectory());
@@ -270,7 +270,6 @@ public class LegacySSTableTest
logger.debug("for pk={} ck={}", pk, ck);
String pkValue = Integer.toString(pk);
- UntypedResultSet rs;
if (ck == 0)
{
readSimpleTable(legacyVersion, pkValue);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/unit/org/apache/cassandra/utils/BloomFilterTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/utils/BloomFilterTest.java b/test/unit/org/apache/cassandra/utils/BloomFilterTest.java
index 581248c..1c3afff 100644
--- a/test/unit/org/apache/cassandra/utils/BloomFilterTest.java
+++ b/test/unit/org/apache/cassandra/utils/BloomFilterTest.java
@@ -43,19 +43,23 @@ public class BloomFilterTest
{
public IFilter bfInvHashes;
- public BloomFilterTest()
- {
- }
- public static IFilter testSerialize(IFilter f) throws IOException
+ public static IFilter testSerialize(IFilter f, boolean oldBfFormat) throws IOException
{
f.add(FilterTestHelper.bytes("a"));
DataOutputBuffer out = new DataOutputBuffer();
- FilterFactory.serialize(f, out);
+ if (oldBfFormat)
+ {
+ SerializationsTest.serializeOldBfFormat((BloomFilter) f, out);
+ }
+ else
+ {
+ BloomFilterSerializer.serialize((BloomFilter) f, out);
+ }
ByteArrayInputStream in = new ByteArrayInputStream(out.getData(), 0, out.getLength());
- IFilter f2 = FilterFactory.deserialize(new DataInputStream(in));
+ IFilter f2 = BloomFilterSerializer.deserialize(new DataInputStream(in), oldBfFormat);
assert f2.isPresent(FilterTestHelper.bytes("a"));
assert !f2.isPresent(FilterTestHelper.bytes("b"));
@@ -132,7 +136,8 @@ public class BloomFilterTest
@Test
public void testSerialize() throws IOException
{
- BloomFilterTest.testSerialize(bfInvHashes).close();
+ BloomFilterTest.testSerialize(bfInvHashes, true).close();
+ BloomFilterTest.testSerialize(bfInvHashes, false).close();
}
@Test
@@ -206,13 +211,12 @@ public class BloomFilterTest
BloomFilter filter = (BloomFilter) FilterFactory.getFilter(((long) Integer.MAX_VALUE / 8) + 1, 0.01d);
filter.add(FilterTestHelper.wrap(test));
DataOutputStreamPlus out = new BufferedDataOutputStreamPlus(new FileOutputStream(file));
- FilterFactory.serialize(filter, out);
- filter.bitset.serialize(out);
+ BloomFilterSerializer.serialize(filter, out);
out.close();
filter.close();
DataInputStream in = new DataInputStream(new FileInputStream(file));
- BloomFilter filter2 = (BloomFilter) FilterFactory.deserialize(in);
+ BloomFilter filter2 = BloomFilterSerializer.deserialize(in, false);
Assert.assertTrue(filter2.isPresent(FilterTestHelper.wrap(test)));
FileUtils.closeQuietly(in);
filter2.close();
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/unit/org/apache/cassandra/utils/SerializationsTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/utils/SerializationsTest.java b/test/unit/org/apache/cassandra/utils/SerializationsTest.java
index 8da4a92..f260428 100644
--- a/test/unit/org/apache/cassandra/utils/SerializationsTest.java
+++ b/test/unit/org/apache/cassandra/utils/SerializationsTest.java
@@ -31,30 +31,42 @@ import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.io.util.DataOutputStreamPlus;
-import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.utils.obs.OffHeapBitSet;
import java.io.File;
import java.io.FileInputStream;
public class SerializationsTest extends AbstractSerializationsTester
{
+ // Helper function to serialize old Bloomfilter format, should be removed once the old format is not supported
+ public static void serializeOldBfFormat(BloomFilter bf, DataOutputPlus out) throws IOException
+ {
+ out.writeInt(bf.hashCount);
+ Assert.assertTrue(bf.bitset instanceof OffHeapBitSet);
+ ((OffHeapBitSet) bf.bitset).serializeOldBfFormat(out);
+ }
+
@BeforeClass
public static void initDD()
{
DatabaseDescriptor.daemonInitialization();
}
- private static void testBloomFilterWrite1000() throws IOException
+ private static void testBloomFilterWrite1000(boolean oldBfFormat) throws IOException
{
try (IFilter bf = FilterFactory.getFilter(1000000, 0.0001))
{
for (int i = 0; i < 1000; i++)
bf.add(Util.dk(Int32Type.instance.decompose(i)));
- try (DataOutputStreamPlus out = getOutput("3.0", "utils.BloomFilter1000.bin"))
+ try (DataOutputStreamPlus out = getOutput(oldBfFormat ? "3.0" : "4.0", "utils.BloomFilter1000.bin"))
{
- FilterFactory.serialize(bf, out);
+ if (oldBfFormat)
+ serializeOldBfFormat((BloomFilter) bf, out);
+ else
+ BloomFilterSerializer.serialize((BloomFilter) bf, out);
}
}
}
@@ -63,10 +75,29 @@ public class SerializationsTest extends AbstractSerializationsTester
public void testBloomFilterRead1000() throws IOException
{
if (EXECUTE_WRITES)
- testBloomFilterWrite1000();
+ {
+ testBloomFilterWrite1000(false);
+ testBloomFilterWrite1000(true);
+ }
+
+ try (DataInputStream in = getInput("4.0", "utils.BloomFilter1000.bin");
+ IFilter filter = BloomFilterSerializer.deserialize(in, false))
+ {
+ boolean present;
+ for (int i = 0 ; i < 1000 ; i++)
+ {
+ present = filter.isPresent(Util.dk(Int32Type.instance.decompose(i)));
+ Assert.assertTrue(present);
+ }
+ for (int i = 1000 ; i < 2000 ; i++)
+ {
+ present = filter.isPresent(Util.dk(Int32Type.instance.decompose(i)));
+ Assert.assertFalse(present);
+ }
+ }
try (DataInputStream in = getInput("3.0", "utils.BloomFilter1000.bin");
- IFilter filter = FilterFactory.deserialize(in))
+ IFilter filter = BloomFilterSerializer.deserialize(in, true))
{
boolean present;
for (int i = 0 ; i < 1000 ; i++)
@@ -85,15 +116,15 @@ public class SerializationsTest extends AbstractSerializationsTester
@Test
public void testBloomFilterTable() throws Exception
{
- testBloomFilterTable("test/data/bloom-filter/la/foo/la-1-big-Filter.db");
+ testBloomFilterTable("test/data/bloom-filter/la/foo/la-1-big-Filter.db", true);
}
- private static void testBloomFilterTable(String file) throws Exception
+ private static void testBloomFilterTable(String file, boolean oldBfFormat) throws Exception
{
Murmur3Partitioner partitioner = new Murmur3Partitioner();
try (DataInputStream in = new DataInputStream(new FileInputStream(new File(file)));
- IFilter filter = FilterFactory.deserialize(in))
+ IFilter filter = BloomFilterSerializer.deserialize(in, oldBfFormat))
{
for (int i = 1; i <= 10; i++)
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6cc5943/test/unit/org/apache/cassandra/utils/obs/OffHeapBitSetTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/utils/obs/OffHeapBitSetTest.java b/test/unit/org/apache/cassandra/utils/obs/OffHeapBitSetTest.java
index f0325da..49b4c94 100644
--- a/test/unit/org/apache/cassandra/utils/obs/OffHeapBitSetTest.java
+++ b/test/unit/org/apache/cassandra/utils/obs/OffHeapBitSetTest.java
@@ -25,11 +25,9 @@ import java.util.List;
import java.util.Random;
import com.google.common.collect.Lists;
+import org.apache.cassandra.io.util.DataOutputBuffer;
import org.junit.Assert;
import org.junit.Test;
-import org.junit.rules.ExpectedException;
-
-import org.apache.cassandra.io.util.DataOutputBuffer;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -46,8 +44,7 @@ public class OffHeapBitSetTest
Assert.assertEquals(bs.get(i), newbs.get(i));
}
- @Test
- public void testOffHeapSerialization() throws IOException
+ private void testOffHeapSerialization(boolean oldBfFormat) throws IOException
{
try (OffHeapBitSet bs = new OffHeapBitSet(100000))
{
@@ -56,10 +53,13 @@ public class OffHeapBitSetTest
bs.set(i);
DataOutputBuffer out = new DataOutputBuffer();
- bs.serialize(out);
+ if (oldBfFormat)
+ bs.serializeOldBfFormat(out);
+ else
+ bs.serialize(out);
DataInputStream in = new DataInputStream(new ByteArrayInputStream(out.getData()));
- try (OffHeapBitSet newbs = OffHeapBitSet.deserialize(in))
+ try (OffHeapBitSet newbs = OffHeapBitSet.deserialize(in, oldBfFormat))
{
compare(bs, newbs);
}
@@ -67,6 +67,13 @@ public class OffHeapBitSetTest
}
@Test
+ public void testSerialization() throws IOException
+ {
+ testOffHeapSerialization(true);
+ testOffHeapSerialization(false);
+ }
+
+ @Test
public void testBitSetGetClear()
{
int size = Integer.MAX_VALUE / 4000;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org