You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ns...@apache.org on 2011/10/24 22:44:46 UTC
svn commit: r1188367 - in /hbase/branches/0.89-fb/src:
main/java/org/apache/hadoop/hbase/
main/java/org/apache/hadoop/hbase/io/hfile/
main/java/org/apache/hadoop/hbase/regionserver/
main/java/org/apache/hadoop/hbase/util/ test/java/org/apache/hadoop/hb...
Author: nspiegelberg
Date: Mon Oct 24 20:44:45 2011
New Revision: 1188367
URL: http://svn.apache.org/viewvc?rev=1188367&view=rev
Log:
HBASE-4532 Avoid top row seek by dedicated bloom filter for delete
family bloom filter
Modified:
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/KeyValue.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockType.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/BloomFilterFactory.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/KeyValue.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/KeyValue.java?rev=1188367&r1=1188366&r2=1188367&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/KeyValue.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/KeyValue.java Mon Oct 24 20:44:45 2011
@@ -1648,6 +1648,21 @@ public class KeyValue implements Writabl
}
/**
+ * Create a Delete Family KeyValue for the specified row and family that would
+ * be smaller than all other possible Delete Family KeyValues that have the
+ * same row and family.
+ * Used for seeking.
+ * @param row - row key (arbitrary byte array)
+ * @param family - family name
+ * @return First Delete Family possible key on passed <code>row</code>.
+ */
+ public static KeyValue createFirstDeleteFamilyOnRow(final byte [] row,
+ final byte [] family) {
+ return new KeyValue(row, family, null, HConstants.LATEST_TIMESTAMP,
+ Type.DeleteFamily);
+ }
+
+ /**
* @param row - row key (arbitrary byte array)
* @param f - family name
* @param q - column qualifier
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockType.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockType.java?rev=1188367&r1=1188366&r2=1188367&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockType.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockType.java Mon Oct 24 20:44:45 2011
@@ -61,8 +61,11 @@ public enum BlockType {
/** File info, version 2 */
FILE_INFO("FILEINF2", BlockCategory.META),
- /** Bloom filter metadata, version 2 */
- BLOOM_META("BLMFMET2", BlockCategory.BLOOM),
+ /** General Bloom filter metadata, version 2 */
+ GENERAL_BLOOM_META("BLMFMET2", BlockCategory.BLOOM),
+
+ /** Delete Family Bloom filter metadata, version 2 */
+ DELETE_FAMILY_BLOOM_META("DFBLMET2", BlockCategory.BLOOM),
// Trailer
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java?rev=1188367&r1=1188366&r2=1188367&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java Mon Oct 24 20:44:45 2011
@@ -224,11 +224,17 @@ public class HFile {
void addInlineBlockWriter(InlineBlockWriter bloomWriter);
/**
- * Store Bloom filter in the file. This does not deal with Bloom filter
+ * Store general Bloom filter in the file. This does not deal with Bloom filter
* internals but is necessary, since Bloom filters are stored differently
* in HFile version 1 and version 2.
*/
- void addBloomFilter(BloomFilterWriter bfw);
+ void addGeneralBloomFilter(BloomFilterWriter bfw);
+
+ /**
+ * Store delete family Bloom filter in the file, which is only supported in
+ * HFile V2.
+ */
+ void addDeleteFamilyBloomFilter(BloomFilterWriter bfw) throws IOException;
}
/**
@@ -355,10 +361,18 @@ public class HFile {
Compression.Algorithm getCompressionAlgorithm();
/**
- * Retrieves Bloom filter metadata as appropriate for each {@link HFile}
- * version. Knows nothing about how that metadata is structured.
+ * Retrieves general Bloom filter metadata as appropriate for each
+ * {@link HFile} version.
+ * Knows nothing about how that metadata is structured.
+ */
+ DataInput getGeneralBloomFilterMetadata() throws IOException;
+
+ /**
+ * Retrieves delete family Bloom filter metadata as appropriate for each
+ * {@link HFile} version.
+ * Knows nothing about how that metadata is structured.
*/
- DataInput getBloomFilterMetadata() throws IOException;
+ DataInput getDeleteBloomFilterMetadata() throws IOException;
Path getPath();
}
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java?rev=1188367&r1=1188366&r2=1188367&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java Mon Oct 24 20:44:45 2011
@@ -322,8 +322,8 @@ public class HFilePrettyPrinter {
}
System.out.println("Mid-key: " + Bytes.toStringBinary(reader.midkey()));
- // Printing bloom information
- DataInput bloomMeta = reader.getBloomFilterMetadata();
+ // Printing general bloom information
+ DataInput bloomMeta = reader.getGeneralBloomFilterMetadata();
BloomFilter bloomFilter = null;
if (bloomMeta != null)
bloomFilter = BloomFilterFactory.createFromMeta(bloomMeta, reader);
@@ -336,5 +336,20 @@ public class HFilePrettyPrinter {
} else {
System.out.println(FOUR_SPACES + "Not present");
}
+
+ // Printing delete bloom information
+ bloomMeta = reader.getDeleteBloomFilterMetadata();
+ bloomFilter = null;
+ if (bloomMeta != null)
+ bloomFilter = BloomFilterFactory.createFromMeta(bloomMeta, reader);
+
+ System.out.println("Delete Family Bloom filter:");
+ if (bloomFilter != null) {
+ System.out.println(FOUR_SPACES
+ + bloomFilter.toString().replaceAll(ByteBloomFilter.STATS_RECORD_SEP,
+ "\n" + FOUR_SPACES));
+ } else {
+ System.out.println(FOUR_SPACES + "Not present");
+ }
}
}
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java?rev=1188367&r1=1188366&r2=1188367&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java Mon Oct 24 20:44:45 2011
@@ -663,7 +663,7 @@ public class HFileReaderV1 extends Abstr
}
@Override
- public DataInput getBloomFilterMetadata() throws IOException {
+ public DataInput getGeneralBloomFilterMetadata() throws IOException {
ByteBuffer buf = getMetaBlock(HFileWriterV1.BLOOM_FILTER_META_KEY, false);
if (buf == null)
return null;
@@ -673,6 +673,11 @@ public class HFileReaderV1 extends Abstr
}
@Override
+ public DataInput getDeleteBloomFilterMetadata() throws IOException {
+ return null;
+ }
+
+ @Override
public boolean isFileInfoLoaded() {
return fileInfoLoaded;
}
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java?rev=1188367&r1=1188366&r2=1188367&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java Mon Oct 24 20:44:45 2011
@@ -709,9 +709,25 @@ public class HFileReaderV2 extends Abstr
* ownership of the buffer.
*/
@Override
- public DataInput getBloomFilterMetadata() throws IOException {
+ public DataInput getGeneralBloomFilterMetadata() throws IOException {
+ return this.getBloomFilterMetadata(BlockType.GENERAL_BLOOM_META);
+ }
+
+ @Override
+ public DataInput getDeleteBloomFilterMetadata() throws IOException {
+ return this.getBloomFilterMetadata(BlockType.DELETE_FAMILY_BLOOM_META);
+ }
+
+ private DataInput getBloomFilterMetadata(BlockType blockType)
+ throws IOException {
+ if (blockType != BlockType.GENERAL_BLOOM_META &&
+ blockType != BlockType.DELETE_FAMILY_BLOOM_META) {
+ throw new RuntimeException("Block Type: " + blockType.toString() +
+ " is not supported") ;
+ }
+
for (HFileBlock b : loadOnOpenBlocks)
- if (b.getBlockType() == BlockType.BLOOM_META)
+ if (b.getBlockType() == blockType)
return b.getByteStream();
return null;
}
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java?rev=1188367&r1=1188366&r2=1188367&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java Mon Oct 24 20:44:45 2011
@@ -437,11 +437,11 @@ public class HFileWriterV1 extends Abstr
}
/**
- * Version 1 Bloom filters are stored in two meta blocks with two different
+ * Version 1 general Bloom filters are stored in two meta blocks with two different
* keys.
*/
@Override
- public void addBloomFilter(BloomFilterWriter bfw) {
+ public void addGeneralBloomFilter(BloomFilterWriter bfw) {
appendMetaBlock(BLOOM_FILTER_META_KEY,
bfw.getMetaWriter());
Writable dataWriter = bfw.getDataWriter();
@@ -450,6 +450,12 @@ public class HFileWriterV1 extends Abstr
}
}
+ @Override
+ public void addDeleteFamilyBloomFilter(BloomFilterWriter bfw)
+ throws IOException {
+ throw new IOException("Delete Bloom filter is not supported in HFile V1");
+ }
+
/**
* Write out the index in the version 1 format. This conforms to the legacy
* version 1 format, but can still be read by
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java?rev=1188367&r1=1188366&r2=1188367&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java Mon Oct 24 20:44:45 2011
@@ -434,14 +434,29 @@ public class HFileWriterV2 extends Abstr
}
@Override
- public void addBloomFilter(final BloomFilterWriter bfw) {
+ public void addGeneralBloomFilter(final BloomFilterWriter bfw) {
+ this.addBloomFilter(bfw, BlockType.GENERAL_BLOOM_META);
+ }
+
+ @Override
+ public void addDeleteFamilyBloomFilter(final BloomFilterWriter bfw) {
+ this.addBloomFilter(bfw, BlockType.DELETE_FAMILY_BLOOM_META);
+ }
+
+ private void addBloomFilter(final BloomFilterWriter bfw,
+ final BlockType blockType) {
if (bfw.getKeyCount() <= 0)
return;
+ if (blockType != BlockType.GENERAL_BLOOM_META &&
+ blockType != BlockType.DELETE_FAMILY_BLOOM_META) {
+ throw new RuntimeException("Block Type: " + blockType.toString() +
+ "is not supported");
+ }
additionalLoadOnOpenData.add(new BlockWritable() {
@Override
public BlockType getBlockType() {
- return BlockType.BLOOM_META;
+ return blockType;
}
@Override
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java?rev=1188367&r1=1188366&r2=1188367&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java Mon Oct 24 20:44:45 2011
@@ -65,15 +65,23 @@ public class ScanQueryMatcher {
protected byte [] row;
/**
+ * This variable shows whether there is an null column in the query. There
+ * always exists a null column in the wildcard column query.
+ * There maybe exists a null column in the explicit column query based on the
+ * first column.
+ * */
+ private boolean hasNullColumn = true;
+
+ /**
* Constructs a ScanQueryMatcher for a Scan.
* @param scan
* @param family
- * @param columns
+ * @param columnSet
* @param ttl
* @param rowComparator
*/
public ScanQueryMatcher(Scan scan, byte [] family,
- NavigableSet<byte[]> columns, long ttl,
+ NavigableSet<byte[]> columnSet, long ttl,
KeyValue.KeyComparator rowComparator, int maxVersions,
boolean retainDeletesInOutput) {
this.tr = scan.getTimeRange();
@@ -81,21 +89,36 @@ public class ScanQueryMatcher {
this.rowComparator = rowComparator;
this.deletes = new ScanDeleteTracker();
this.stopRow = scan.getStopRow();
- this.startKey = KeyValue.createFirstOnRow(scan.getStartRow(), family, null);
+ this.startKey = KeyValue.createFirstDeleteFamilyOnRow(scan.getStartRow(),
+ family);
this.filter = scan.getFilter();
this.retainDeletesInOutput = retainDeletesInOutput;
// Single branch to deal with two types of reads (columns vs all in family)
- if (columns == null || columns.size() == 0) {
+ if (columnSet == null || columnSet.size() == 0) {
+ // there is always a null column in the wildcard column query.
+ hasNullColumn = true;
+
// use a specialized scan for wildcard column tracker.
this.columns = new ScanWildcardColumnTracker(maxVersions);
} else {
+ // whether there is null column in the explicit column query
+ hasNullColumn = (columnSet.first().length == 0);
+
// We can share the ExplicitColumnTracker, diff is we reset
// between rows, not between storefiles.
- this.columns = new ExplicitColumnTracker(columns,maxVersions);
+ this.columns = new ExplicitColumnTracker(columnSet,maxVersions);
}
}
+ /**
+ *
+ * @return whether there is an null column in the query
+ */
+ public boolean hasNullColumnInQuery() {
+ return hasNullColumn;
+ }
+
public ScanQueryMatcher(Scan scan, byte [] family,
NavigableSet<byte[]> columns, long ttl,
KeyValue.KeyComparator rowComparator, int maxVersions) {
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java?rev=1188367&r1=1188366&r2=1188367&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java Mon Oct 24 20:44:45 2011
@@ -48,6 +48,7 @@ import org.apache.hadoop.hbase.client.Sc
import org.apache.hadoop.hbase.io.HalfStoreFileReader;
import org.apache.hadoop.hbase.io.Reference;
import org.apache.hadoop.hbase.io.hfile.BlockCache;
+import org.apache.hadoop.hbase.io.hfile.BlockType;
import org.apache.hadoop.hbase.io.hfile.Compression;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.io.hfile.HFileScanner;
@@ -115,6 +116,10 @@ public class StoreFile {
static final byte[] BLOOM_FILTER_TYPE_KEY =
Bytes.toBytes("BLOOM_FILTER_TYPE");
+ /** Delete Family Count in FileInfo */
+ public static final byte[] DELETE_FAMILY_COUNT =
+ Bytes.toBytes("DELETE_FAMILY_COUNT");
+
/** Last Bloom filter key in FileInfo */
private static final byte[] LAST_BLOOM_KEY = Bytes.toBytes("LAST_BLOOM_KEY");
@@ -218,7 +223,7 @@ public class StoreFile {
this.referencePath = getReferredToFile(this.path);
}
- if (BloomFilterFactory.isBloomEnabled(conf)) {
+ if (BloomFilterFactory.isGeneralBloomEnabled(conf)) {
this.cfBloomType = cfBloomType;
} else {
LOG.info("Ignoring bloom filter check for file " + path + ": " +
@@ -451,7 +456,7 @@ public class StoreFile {
BloomType hfileBloomType = reader.getBloomFilterType();
if (cfBloomType != BloomType.NONE) {
- reader.loadBloomfilter();
+ reader.loadBloomfilter(BlockType.GENERAL_BLOOM_META);
if (hfileBloomType != cfBloomType) {
LOG.info("HFile Bloom filter type for "
+ reader.getHFileReader().getName() + ": " + hfileBloomType
@@ -463,6 +468,9 @@ public class StoreFile {
+ reader.getHFileReader().getName());
}
+ // load delete family bloom filter
+ reader.loadBloomfilter(BlockType.DELETE_FAMILY_BLOOM_META);
+
try {
byte [] timerangeBytes = metadataMap.get(TIMERANGE_KEY);
if (timerangeBytes != null) {
@@ -621,7 +629,7 @@ public class StoreFile {
fs.mkdirs(dir);
}
Path path = getUniqueFile(fs, dir);
- if (!BloomFilterFactory.isBloomEnabled(conf)) {
+ if (!BloomFilterFactory.isGeneralBloomEnabled(conf)) {
bloomType = BloomType.NONE;
}
@@ -711,12 +719,15 @@ public class StoreFile {
* local because it is an implementation detail of the HBase regionserver.
*/
public static class Writer {
- private final BloomFilterWriter bloomFilterWriter;
+ private final BloomFilterWriter generalBloomFilterWriter;
+ private final BloomFilterWriter deleteFamilyBloomFilterWriter;
private final BloomType bloomType;
private byte[] lastBloomKey;
private int lastBloomKeyOffset, lastBloomKeyLen;
private KVComparator kvComparator;
private KeyValue lastKv = null;
+ private KeyValue lastDeleteFamilyKV = null;
+ private long deleteFamilyCnt = 0;
TimeRangeTracker timeRangeTracker = new TimeRangeTracker();
/* isTimeRangeTrackerSet keeps track if the timeRange has already been set
@@ -762,18 +773,34 @@ public class StoreFile {
this.kvComparator = comparator;
- bloomFilterWriter = BloomFilterFactory.createBloomAtWrite(conf,
- bloomType, (int) Math.min(maxKeys, Integer.MAX_VALUE), writer, bloomErrorRate);
- if (bloomFilterWriter != null) {
+ generalBloomFilterWriter = BloomFilterFactory.createGeneralBloomAtWrite(
+ conf, bloomType, (int) Math.min(maxKeys, Integer.MAX_VALUE), writer,
+ bloomErrorRate);
+
+ if (generalBloomFilterWriter != null) {
this.bloomType = bloomType;
- LOG.info("Bloom filter type for " + path + ": " + this.bloomType +
- ", "+ bloomFilterWriter.getClass().getSimpleName());
+ LOG.info("Bloom filter type for " + path + ": " + this.bloomType + ", "
+ + generalBloomFilterWriter.getClass().getSimpleName());
} else {
// Not using Bloom filters.
this.bloomType = BloomType.NONE;
}
- }
+ // initialize delete family Bloom filter when there is NO RowCol Bloom
+ // filter
+ if (this.bloomType != BloomType.ROWCOL) {
+ this.deleteFamilyBloomFilterWriter = BloomFilterFactory
+ .createDeleteBloomAtWrite(conf,
+ (int) Math.min(maxKeys, Integer.MAX_VALUE), writer,
+ bloomErrorRate);
+ } else {
+ deleteFamilyBloomFilterWriter = null;
+ }
+ if (deleteFamilyBloomFilterWriter != null) {
+ LOG.info("Delete Family Bloom filter type for " + path + ": "
+ + deleteFamilyBloomFilterWriter.getClass().getSimpleName());
+ }
+ }
/**
* Writes meta data.
@@ -830,8 +857,8 @@ public class StoreFile {
}
}
- public void append(final KeyValue kv) throws IOException {
- if (this.bloomFilterWriter != null) {
+ private void appendGeneralBloomfilter(final KeyValue kv) throws IOException {
+ if (this.generalBloomFilterWriter != null) {
// only add to the bloom filter on a new, unique key
boolean newKey = true;
if (this.lastKv != null) {
@@ -871,7 +898,7 @@ public class StoreFile {
// merge(row, qualifier)
// TODO: could save one buffer copy in case of compound Bloom
// filters when this involves creating a KeyValue
- bloomKey = bloomFilterWriter.createBloomKey(kv.getBuffer(),
+ bloomKey = generalBloomFilterWriter.createBloomKey(kv.getBuffer(),
kv.getRowOffset(), kv.getRowLength(), kv.getBuffer(),
kv.getQualifierOffset(), kv.getQualifierLength());
bloomKeyOffset = 0;
@@ -881,9 +908,9 @@ public class StoreFile {
throw new IOException("Invalid Bloom filter type: " + bloomType +
" (ROW or ROWCOL expected)");
}
- bloomFilterWriter.add(bloomKey, bloomKeyOffset, bloomKeyLen);
+ generalBloomFilterWriter.add(bloomKey, bloomKeyOffset, bloomKeyLen);
if (lastBloomKey != null
- && bloomFilterWriter.getComparator().compare(bloomKey,
+ && generalBloomFilterWriter.getComparator().compare(bloomKey,
bloomKeyOffset, bloomKeyLen, lastBloomKey,
lastBloomKeyOffset, lastBloomKeyLen) <= 0) {
throw new IOException("Non-increasing Bloom keys: "
@@ -898,6 +925,32 @@ public class StoreFile {
this.lastKv = kv;
}
}
+ }
+
+ private void appendDeleteFamilyBloomFilter(final KeyValue kv)
+ throws IOException {
+ if (!kv.isDeleteFamily()) {
+ return;
+ }
+
+ // increase the number of delete family in the store file
+ deleteFamilyCnt++;
+ if (null != this.deleteFamilyBloomFilterWriter) {
+ boolean newKey = true;
+ if (lastDeleteFamilyKV != null) {
+ newKey = !kvComparator.matchingRows(kv, lastDeleteFamilyKV);
+ }
+ if (newKey) {
+ this.deleteFamilyBloomFilterWriter.add(kv.getBuffer(),
+ kv.getRowOffset(), kv.getRowLength());
+ this.lastDeleteFamilyKV = kv;
+ }
+ }
+ }
+
+ public void append(final KeyValue kv) throws IOException {
+ appendGeneralBloomfilter(kv);
+ appendDeleteFamilyBloomFilter(kv);
writer.append(kv);
includeInTimeRangeTracker(kv);
}
@@ -906,25 +959,34 @@ public class StoreFile {
return this.writer.getPath();
}
- boolean hasBloom() {
- return this.bloomFilterWriter != null;
+ boolean hasGeneralBloom() {
+ return this.generalBloomFilterWriter != null;
+
}
/**
* For unit testing only.
+ *
* @return the Bloom filter used by this writer.
*/
- BloomFilterWriter getBloomWriter() {
- return bloomFilterWriter;
+ BloomFilterWriter getGeneralBloomWriter() {
+ return generalBloomFilterWriter;
}
- public void close() throws IOException {
- // Make sure we wrote something to the Bloom filter before adding it.
- boolean haveBloom = bloomFilterWriter != null &&
- bloomFilterWriter.getKeyCount() > 0;
+ private boolean closeBloomFilter(BloomFilterWriter bfw) throws IOException {
+ boolean haveBloom = (bfw != null && bfw.getKeyCount() > 0);
if (haveBloom) {
- bloomFilterWriter.compactBloom();
- writer.addBloomFilter(bloomFilterWriter);
+ bfw.compactBloom();
+ }
+ return haveBloom;
+ }
+
+ private boolean closeGeneralBloomFilter() throws IOException {
+ boolean hasGeneralBloom = closeBloomFilter(generalBloomFilterWriter);
+
+ // add the general Bloom filter writer and append file info
+ if (hasGeneralBloom) {
+ writer.addGeneralBloomFilter(generalBloomFilterWriter);
writer.appendFileInfo(BLOOM_FILTER_TYPE_KEY,
Bytes.toBytes(bloomType.toString()));
if (lastBloomKey != null) {
@@ -933,15 +995,37 @@ public class StoreFile {
+ lastBloomKeyLen));
}
}
+ return hasGeneralBloom;
+ }
+
+ private boolean closeDeleteFamilyBloomFilter() throws IOException {
+ boolean hasDeleteFamilyBloom = closeBloomFilter(deleteFamilyBloomFilterWriter);
+
+ // add the delete family Bloom filter writer
+ if (hasDeleteFamilyBloom) {
+ writer.addDeleteFamilyBloomFilter(deleteFamilyBloomFilterWriter);
+ }
+
+ // append file info about the number of delete family kvs
+ // even if there is no delete family Bloom.
+ writer.appendFileInfo(DELETE_FAMILY_COUNT,
+ Bytes.toBytes(this.deleteFamilyCnt));
+
+ return hasDeleteFamilyBloom;
+ }
+
+ public void close() throws IOException {
+ boolean hasGeneralBloom = this.closeGeneralBloomFilter();
+ boolean hasDeleteFamilyBloom = this.closeDeleteFamilyBloomFilter();
+
writer.close();
// Log final Bloom filter statistics. This needs to be done after close()
// because compound Bloom filters might be finalized as part of closing.
- if (haveBloom && bloomFilterWriter.getMaxKeys() > 0) {
- StoreFile.LOG.info("Bloom added to HFile ("
- + getPath() + "): " +
- bloomFilterWriter.toString().replace("\n", "; "));
- }
+ StoreFile.LOG.info((hasGeneralBloom ? "" : "NO ") + "General Bloom and "
+ + (hasDeleteFamilyBloom ? "" : "NO ") + "DeleteFamily"
+ + " was added to HFile (" + getPath() + ") ");
+
}
public void appendFileInfo(byte[] key, byte[] value) throws IOException {
@@ -960,12 +1044,14 @@ public class StoreFile {
public static class Reader extends SchemaConfigured {
static final Log LOG = LogFactory.getLog(Reader.class.getName());
- protected BloomFilter bloomFilter = null;
+ protected BloomFilter generalBloomFilter = null;
+ protected BloomFilter deleteFamilyBloomFilter = null;
protected BloomType bloomFilterType;
private final HFile.Reader reader;
protected TimeRangeTracker timeRangeTracker = null;
protected long sequenceID = -1;
private byte[] lastBloomKey;
+ private long deleteFamilyCnt = -1;
private Reader(HFile.Reader reader) {
super(reader);
@@ -1076,7 +1162,7 @@ public class StoreFile {
* checks Bloom filters for single-row or single-row-column scans. Bloom
* filter checking for multi-gets is implemented as part of the store
* scanner system (see {@link StoreFileScanner#seekExactly}) and uses
- * the lower-level API {@link #passesBloomFilter(byte[], int, int, byte[],
+ * the lower-level API {@link #passesGeneralBloomFilter(byte[], int, int, byte[],
* int, int)}.
*
* @param scan the scan specification. Used to determine the row, and to
@@ -1095,12 +1181,12 @@ public class StoreFile {
byte[] row = scan.getStartRow();
switch (this.bloomFilterType) {
case ROW:
- return passesBloomFilter(row, 0, row.length, null, 0, 0);
+ return passesGeneralBloomFilter(row, 0, row.length, null, 0, 0);
case ROWCOL:
if (columns != null && columns.size() == 1) {
byte[] column = columns.first();
- return passesBloomFilter(row, 0, row.length, column, 0,
+ return passesGeneralBloomFilter(row, 0, row.length, column, 0,
column.length);
}
@@ -1113,6 +1199,35 @@ public class StoreFile {
}
}
+ public boolean passesDeleteFamilyBloomFilter(byte[] row, int rowOffset,
+ int rowLen) {
+ // Cache Bloom filter as a local variable in case it is set to null by
+ // another thread on an IO error.
+ BloomFilter bloomFilter = this.deleteFamilyBloomFilter;
+
+ // Empty file or there is no delete family at all
+ if (reader.getTrailer().getEntryCount() == 0 || deleteFamilyCnt == 0) {
+ return false;
+ }
+
+ if (bloomFilter == null) {
+ return true;
+ }
+
+ try {
+ if (!bloomFilter.supportsAutoLoading()) {
+ return true;
+ }
+ return bloomFilter.contains(row, rowOffset, rowLen, null);
+ } catch (IllegalArgumentException e) {
+ LOG.error("Bad Delete Family bloom filter data -- proceeding without",
+ e);
+ setDeleteFamilyBloomFilterFaulty();
+ }
+
+ return true;
+ }
+
/**
* A method for checking Bloom filters. Called directly from
* {@link StoreFileScanner} in case of a multi-column query.
@@ -1125,9 +1240,9 @@ public class StoreFile {
* @param colLen
* @return
*/
- public boolean passesBloomFilter(byte[] row, int rowOffset, int rowLen,
- byte[] col, int colOffset, int colLen) {
- if (bloomFilter == null)
+ public boolean passesGeneralBloomFilter(byte[] row, int rowOffset,
+ int rowLen, byte[] col, int colOffset, int colLen) {
+ if (generalBloomFilter == null)
return true;
byte[] key;
@@ -1145,7 +1260,7 @@ public class StoreFile {
break;
case ROWCOL:
- key = bloomFilter.createBloomKey(row, rowOffset, rowLen, col,
+ key = generalBloomFilter.createBloomKey(row, rowOffset, rowLen, col,
colOffset, colLen);
break;
@@ -1155,13 +1270,13 @@ public class StoreFile {
// Cache Bloom filter as a local variable in case it is set to null by
// another thread on an IO error.
- BloomFilter bloomFilter = this.bloomFilter;
+ BloomFilter bloomFilter = this.generalBloomFilter;
if (bloomFilter == null) {
return true;
}
- // Empty file?
+ // Empty file
if (reader.getTrailer().getEntryCount() == 0)
return false;
@@ -1215,10 +1330,10 @@ public class StoreFile {
} catch (IOException e) {
LOG.error("Error reading bloom filter data -- proceeding without",
e);
- setBloomFilterFaulty();
+ setGeneralBloomFilterFaulty();
} catch (IllegalArgumentException e) {
LOG.error("Bad bloom filter data -- proceeding without", e);
- setBloomFilterFaulty();
+ setGeneralBloomFilterFaulty();
}
return true;
@@ -1233,34 +1348,71 @@ public class StoreFile {
}
lastBloomKey = fi.get(LAST_BLOOM_KEY);
+ byte[] cnt = fi.get(DELETE_FAMILY_COUNT);
+ if (cnt != null) {
+ deleteFamilyCnt = Bytes.toLong(cnt);
+ }
return fi;
}
public void loadBloomfilter() {
- if (this.bloomFilter != null) {
- return; // already loaded
- }
+ this.loadBloomfilter(BlockType.GENERAL_BLOOM_META);
+ this.loadBloomfilter(BlockType.DELETE_FAMILY_BLOOM_META);
+ }
+ private void loadBloomfilter(BlockType blockType) {
try {
- DataInput bloomMeta = reader.getBloomFilterMetadata();
- if (bloomMeta != null) {
- if (bloomFilterType == BloomType.NONE) {
- throw new IOException(
- "valid bloom filter type not found in FileInfo");
+ if (blockType == BlockType.GENERAL_BLOOM_META) {
+ if (this.generalBloomFilter != null)
+ return; // Bloom has been loaded
+
+ DataInput bloomMeta = reader.getGeneralBloomFilterMetadata();
+ if (bloomMeta != null) {
+ // sanity check for NONE Bloom filter
+ if (bloomFilterType == BloomType.NONE) {
+ throw new IOException(
+ "valid bloom filter type not found in FileInfo");
+ } else {
+ generalBloomFilter = BloomFilterFactory.createFromMeta(bloomMeta,
+ reader);
+ LOG.info("Loaded " + bloomFilterType.toString() + " ("
+ + generalBloomFilter.getClass().getSimpleName()
+ + ") metadata for " + reader.getName());
+ }
}
-
- bloomFilter = BloomFilterFactory.createFromMeta(bloomMeta, reader);
- LOG.info("Loaded " + bloomFilterType + " " +
- bloomFilter.getClass().getSimpleName() + " metadata for " +
- reader.getName());
+ } else if (blockType == BlockType.DELETE_FAMILY_BLOOM_META) {
+ if (this.deleteFamilyBloomFilter != null)
+ return; // Bloom has been loaded
+
+ DataInput bloomMeta = reader.getDeleteBloomFilterMetadata();
+ if (bloomMeta != null) {
+ deleteFamilyBloomFilter = BloomFilterFactory.createFromMeta(
+ bloomMeta, reader);
+ LOG.info("Loaded Delete Family Bloom ("
+ + deleteFamilyBloomFilter.getClass().getSimpleName()
+ + ") metadata for " + reader.getName());
+ }
+ } else {
+ throw new RuntimeException("Block Type: " + blockType.toString()
+ + "is not supported for Bloom filter");
}
} catch (IOException e) {
- LOG.error("Error reading bloom filter meta -- proceeding without", e);
- this.bloomFilter = null;
+ LOG.error("Error reading bloom filter meta for " + blockType
+ + " -- proceeding without", e);
+ setBloomFilterFaulty(blockType);
} catch (IllegalArgumentException e) {
- LOG.error("Bad bloom filter meta -- proceeding without", e);
- this.bloomFilter = null;
+ LOG.error("Bad bloom filter meta " + blockType
+ + " -- proceeding without", e);
+ setBloomFilterFaulty(blockType);
+ }
+ }
+
+ private void setBloomFilterFaulty(BlockType blockType) {
+ if (blockType == BlockType.GENERAL_BLOOM_META) {
+ setGeneralBloomFilterFaulty();
+ } else if (blockType == BlockType.DELETE_FAMILY_BLOOM_META) {
+ setDeleteFamilyBloomFilterFaulty();
}
}
@@ -1272,12 +1424,16 @@ public class StoreFile {
* @return an estimate of the number of Bloom filter entries in this file
*/
public long getFilterEntries() {
- return bloomFilter != null ? bloomFilter.getKeyCount()
+ return generalBloomFilter != null ? generalBloomFilter.getKeyCount()
: reader.getEntries();
}
- public void setBloomFilterFaulty() {
- bloomFilter = null;
+ public void setGeneralBloomFilterFaulty() {
+ generalBloomFilter = null;
+ }
+
+ public void setDeleteFamilyBloomFilterFaulty() {
+ this.deleteFamilyBloomFilter = null;
}
public byte[] getLastKey() {
@@ -1296,6 +1452,10 @@ public class StoreFile {
return reader.getEntries();
}
+ public long getDeleteFamilyCnt() {
+ return deleteFamilyCnt;
+ }
+
public byte[] getFirstKey() {
return reader.getFirstKey();
}
@@ -1320,8 +1480,8 @@ public class StoreFile {
this.sequenceID = sequenceID;
}
- BloomFilter getBloomFilter() {
- return bloomFilter;
+ BloomFilter getGeneralBloomFilter() {
+ return generalBloomFilter;
}
long getUncompressedDataIndexSize() {
@@ -1329,9 +1489,9 @@ public class StoreFile {
}
public long getTotalBloomSize() {
- if (bloomFilter == null)
+ if (generalBloomFilter == null)
return 0;
- return bloomFilter.getByteSize();
+ return generalBloomFilter.getByteSize();
}
public int getHFileVersion() {
@@ -1343,7 +1503,8 @@ public class StoreFile {
}
void disableBloomFilterForTesting() {
- bloomFilter = null;
+ generalBloomFilter = null;
+ this.deleteFamilyBloomFilter = null;
}
public long getMaxTimestamp() {
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java?rev=1188367&r1=1188366&r2=1188367&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java Mon Oct 24 20:44:45 2011
@@ -26,6 +26,8 @@ import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.hfile.HFileScanner;
import org.apache.hadoop.hbase.regionserver.StoreFile.Reader;
+import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
+import org.apache.hadoop.hbase.util.Bytes;
import java.io.IOException;
import java.util.ArrayList;
@@ -57,6 +59,8 @@ class StoreFileScanner implements KeyVal
private static final AtomicLong seekCount = new AtomicLong();
+ private ScanQueryMatcher matcher;
+
/**
* Implements a {@link KeyValueScanner} on top of the specified {@link HFileScanner}
* @param hfs HFile scanner
@@ -81,19 +85,31 @@ class StoreFileScanner implements KeyVal
}
/**
- * Return an array of scanners corresponding to the given
- * set of store files.
+ * Return an array of scanners corresponding to the given set of store files.
*/
public static List<StoreFileScanner> getScannersForStoreFiles(
- Collection<StoreFile> files,
- boolean cacheBlocks,
- boolean usePread,
+ Collection<StoreFile> files, boolean cacheBlocks, boolean usePread,
boolean isCompaction) throws IOException {
- List<StoreFileScanner> scanners =
- new ArrayList<StoreFileScanner>(files.size());
+ return getScannersForStoreFiles(files, cacheBlocks, usePread, isCompaction,
+ null);
+ }
+
+ /**
+ * Return an array of scanners corresponding to the given set of store files,
+ * And set the ScanQueryMatcher for each store file scanner for further
+ * optimization
+ */
+ public static List<StoreFileScanner> getScannersForStoreFiles(
+ Collection<StoreFile> files, boolean cacheBlocks, boolean usePread,
+ boolean isCompaction, ScanQueryMatcher matcher) throws IOException {
+ List<StoreFileScanner> scanners = new ArrayList<StoreFileScanner>(
+ files.size());
for (StoreFile file : files) {
StoreFile.Reader r = file.createReader();
- scanners.add(r.getStoreFileScanner(cacheBlocks, usePread, isCompaction));
+ StoreFileScanner scanner = r.getStoreFileScanner(cacheBlocks, usePread,
+ isCompaction);
+ scanner.setScanQueryMatcher(matcher);
+ scanners.add(scanner);
}
return scanners;
}
@@ -225,16 +241,24 @@ class StoreFileScanner implements KeyVal
@Override
public boolean requestSeek(KeyValue kv, boolean forward, boolean useBloom)
throws IOException {
- if (reader.getBloomFilterType() != StoreFile.BloomType.ROWCOL ||
- kv.getFamilyLength() == 0) {
+ if (kv.getFamilyLength() == 0) {
useBloom = false;
}
boolean haveToSeek = true;
if (useBloom) {
- haveToSeek = reader.passesBloomFilter(kv.getBuffer(),
- kv.getRowOffset(), kv.getRowLength(), kv.getBuffer(),
- kv.getQualifierOffset(), kv.getQualifierLength());
+ // check ROWCOL Bloom filter first.
+ if (reader.getBloomFilterType() == StoreFile.BloomType.ROWCOL) {
+ haveToSeek = reader.passesGeneralBloomFilter(kv.getBuffer(),
+ kv.getRowOffset(), kv.getRowLength(), kv.getBuffer(),
+ kv.getQualifierOffset(), kv.getQualifierLength());
+ } else if (this.matcher != null && !matcher.hasNullColumnInQuery() &&
+ kv.isDeleteFamily()) {
+ // if there is no such delete family kv in the store file,
+ // then no need to seek.
+ haveToSeek = reader.passesDeleteFamilyBloomFilter(kv.getBuffer(),
+ kv.getRowOffset(), kv.getRowLength());
+ }
}
delayedReseek = forward;
@@ -298,6 +322,10 @@ class StoreFileScanner implements KeyVal
}
}
+ public void setScanQueryMatcher(ScanQueryMatcher matcher) {
+ this.matcher = matcher;
+ }
+
// Test methods
static final long getSeekCount() {
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java?rev=1188367&r1=1188366&r2=1188367&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java Mon Oct 24 20:44:45 2011
@@ -211,7 +211,7 @@ class StoreScanner extends NonLazyKeyVal
// First the store file scanners
List<StoreFileScanner> sfScanners = StoreFileScanner
.getScannersForStoreFiles(store.getStorefiles(), cacheBlocks,
- isGet, false);
+ isGet, false, this.matcher);
List<KeyValueScanner> scanners =
new ArrayList<KeyValueScanner>(sfScanners.size()+1);
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/BloomFilterFactory.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/BloomFilterFactory.java?rev=1188367&r1=1188366&r2=1188367&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/BloomFilterFactory.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/BloomFilterFactory.java Mon Oct 24 20:44:45 2011
@@ -68,6 +68,10 @@ public final class BloomFilterFactory {
public static final String IO_STOREFILE_BLOOM_ENABLED =
"io.storefile.bloom.enabled";
+ /** Master switch to enable Delete Family Bloom filters */
+ public static final String IO_STOREFILE_DELETEFAMILY_BLOOM_ENABLED =
+ "io.storefile.delete.family.bloom.enabled";
+
/**
* Target Bloom block size. Bloom filter blocks of approximately this size
* are interleaved with data blocks.
@@ -115,13 +119,21 @@ public final class BloomFilterFactory {
}
/**
- * @return true if Bloom filters are enabled in the given configuration
+ * @return true if general Bloom (Row or RowCol) filters are enabled in the
+ * given configuration
*/
- public static boolean isBloomEnabled(Configuration conf) {
+ public static boolean isGeneralBloomEnabled(Configuration conf) {
return conf.getBoolean(IO_STOREFILE_BLOOM_ENABLED, true);
}
/**
+ * @return true if Delete Family Bloom filters are enabled in the given configuration
+ */
+ public static boolean isDeleteFamilyBloomEnabled(Configuration conf) {
+ return conf.getBoolean(IO_STOREFILE_DELETEFAMILY_BLOOM_ENABLED, true);
+ }
+
+ /**
* @return the Bloom filter error rate in the given configuration
*/
public static float getErrorRate(Configuration conf) {
@@ -158,7 +170,7 @@ public final class BloomFilterFactory {
* @param toConf conf we will copy to
*/
public static void copyBloomFilterConf(Configuration fromConf, Configuration toConf) {
- toConf.setBoolean(IO_STOREFILE_BLOOM_ENABLED, isBloomEnabled(fromConf));
+ toConf.setBoolean(IO_STOREFILE_BLOOM_ENABLED, isGeneralBloomEnabled(fromConf));
toConf.setFloat(IO_STOREFILE_BLOOM_ERROR_RATE, getErrorRate(fromConf));
toConf.setInt(IO_STOREFILE_BLOOM_MAX_FOLD, getMaxFold(fromConf));
toConf.setInt(IO_STOREFILE_BLOOM_BLOCK_SIZE, getBloomBlockSize(fromConf));
@@ -167,7 +179,7 @@ public final class BloomFilterFactory {
}
/**
- * Creates a new Bloom filter at the time of
+ * Creates a new general (Row or RowCol) Bloom filter at the time of
* {@link org.apache.hadoop.hbase.regionserver.StoreFile} writing.
*
* @param conf
@@ -175,14 +187,14 @@ public final class BloomFilterFactory {
* @param maxKeys an estimate of the number of keys we expect to insert.
* Irrelevant if compound Bloom filters are enabled.
* @param writer the HFile writer
- * @param comparator the comparator to use for compound Bloom filters. This
- * has no effect if creating single-chunk version 1 Bloom filters.
+ * @param bloomErrorRate
* @return the new Bloom filter, or null in case Bloom filters are disabled
* or when failed to create one.
*/
- public static BloomFilterWriter createBloomAtWrite(Configuration conf,
- BloomType bloomType, int maxKeys, HFile.Writer writer, float bloomErrorRate) {
- if (!isBloomEnabled(conf)) {
+ public static BloomFilterWriter createGeneralBloomAtWrite(
+ Configuration conf, BloomType bloomType, int maxKeys, HFile.Writer writer,
+ float bloomErrorRate) {
+ if (!isGeneralBloomEnabled(conf)) {
LOG.info("Bloom filters are disabled by configuration for "
+ writer.getPath()
+ (conf == null ? " (configuration is null)" : ""));
@@ -232,5 +244,39 @@ public final class BloomFilterFactory {
return null;
}
+ /**
+ * Creates a new Delete Family Bloom filter at the time of
+ * {@link org.apache.hadoop.hbase.regionserver.StoreFile} writing.
+ * @param conf
+ * @param maxKeys an estimate of the number of keys we expect to insert.
+ * Irrelevant if compound Bloom filters are enabled.
+ * @param writer the HFile writer
+ * @param bloomErrorRate
+ * @return the new Bloom filter, or null in case Bloom filters are disabled
+ * or when failed to create one.
+ */
+ public static BloomFilterWriter createDeleteBloomAtWrite(
+ Configuration conf, int maxKeys, HFile.Writer writer,
+ float bloomErrorRate) {
+ if (!isDeleteFamilyBloomEnabled(conf)) {
+ LOG.info("Delete Bloom filters are disabled by configuration for "
+ + writer.getPath()
+ + (conf == null ? " (configuration is null)" : ""));
+ return null;
+ }
+ if (HFile.getFormatVersion(conf) > HFile.MIN_FORMAT_VERSION) {
+ int maxFold = getMaxFold(conf);
+ // In case of compound Bloom filters we ignore the maxKeys hint.
+ CompoundBloomFilterWriter bloomWriter = new CompoundBloomFilterWriter(
+ getBloomBlockSize(conf), bloomErrorRate, Hash.getHashType(conf),
+ maxFold,
+ cacheChunksOnWrite(conf), Bytes.BYTES_RAWCOMPARATOR);
+ writer.addInlineBlockWriter(bloomWriter);
+ return bloomWriter;
+ } else {
+ LOG.info("Delete Family Bloom filter is not supported in HFile V1");
+ return null;
+ }
+ }
};
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java?rev=1188367&r1=1188366&r2=1188367&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java Mon Oct 24 20:44:45 2011
@@ -25,6 +25,7 @@ import org.junit.Test;
public class TestBlocksRead extends HBaseTestCase {
static final Log LOG = LogFactory.getLog(TestBlocksRead.class);
+ static final String[] BLOOM_TYPE = new String[] { "ROWCOL", "ROW", "NONE" };
private HBaseConfiguration getConf() {
HBaseConfiguration conf = new HBaseConfiguration();
@@ -35,8 +36,7 @@ public class TestBlocksRead extends HBas
}
HRegion region = null;
- private final String DIR = HBaseTestingUtility.getTestDir() +
- "/TestHRegion/";
+ private final String DIR = HBaseTestingUtility.getTestDir() + "/TestHRegion/";
/**
* @see org.apache.hadoop.hbase.HBaseTestCase#setUp()
@@ -54,31 +54,35 @@ public class TestBlocksRead extends HBas
EnvironmentEdgeManagerTestHelper.reset();
}
- private void initHRegion (byte [] tableName, String callingMethod,
- HBaseConfiguration conf, byte [] ... families)
- throws IOException {
+ private void initHRegion(byte[] tableName, String callingMethod,
+ HBaseConfiguration conf, String family) throws IOException {
HTableDescriptor htd = new HTableDescriptor(tableName);
- for(byte [] family : families) {
- HColumnDescriptor familyDesc = new HColumnDescriptor(
- family,
+ HColumnDescriptor familyDesc;
+ for (int i = 0; i < BLOOM_TYPE.length; i++) {
+ String bloomType = BLOOM_TYPE[i];
+ familyDesc = new HColumnDescriptor(
+ Bytes.toBytes(family + "_" + bloomType),
HColumnDescriptor.DEFAULT_VERSIONS,
HColumnDescriptor.DEFAULT_COMPRESSION,
HColumnDescriptor.DEFAULT_IN_MEMORY,
HColumnDescriptor.DEFAULT_BLOCKCACHE,
1, // small block size deliberate; each kv on its own block
- HColumnDescriptor.DEFAULT_TTL,
- StoreFile.BloomType.ROWCOL.toString(),
+ HColumnDescriptor.DEFAULT_TTL, BLOOM_TYPE[i],
HColumnDescriptor.DEFAULT_REPLICATION_SCOPE);
htd.addFamily(familyDesc);
}
+
HRegionInfo info = new HRegionInfo(htd, null, null, false);
Path path = new Path(DIR + callingMethod);
region = HRegion.createHRegion(info, path, conf);
}
- private void putData(byte[] cf, String row, String col, long version)
- throws IOException {
- putData(cf, row, col, version, version);
+ private void putData(String family, String row, String col, long version)
+ throws IOException {
+ for (int i = 0; i < BLOOM_TYPE.length; i++) {
+ putData(Bytes.toBytes(family + "_" + BLOOM_TYPE[i]), row, col, version,
+ version);
+ }
}
// generates a value to put for a row/col/version.
@@ -86,9 +90,8 @@ public class TestBlocksRead extends HBas
return Bytes.toBytes("Value:" + row + "#" + col + "#" + version);
}
- private void putData(byte[] cf, String row, String col,
- long versionStart, long versionEnd)
- throws IOException {
+ private void putData(byte[] cf, String row, String col, long versionStart,
+ long versionEnd) throws IOException {
byte columnBytes[] = Bytes.toBytes(col);
Put put = new Put(Bytes.toBytes(row));
@@ -98,48 +101,69 @@ public class TestBlocksRead extends HBas
region.put(put);
}
- private KeyValue[] getData(byte[] cf, String row, List<String> columns,
- int expBlocks)
- throws IOException {
-
- long blocksStart = getBlkAccessCount(cf);
- Get get = new Get(Bytes.toBytes(row));
-
- for (String column : columns) {
- get.addColumn(cf, Bytes.toBytes(column));
+ private KeyValue[] getData(String family, String row, List<String> columns,
+ int expBlocks) throws IOException {
+ return getData(family, row, columns, expBlocks, expBlocks, expBlocks);
+ }
+
+ private KeyValue[] getData(String family, String row, List<String> columns,
+ int expBlocksRowCol, int expBlocksRow, int expBlocksNone)
+ throws IOException {
+ int[] expBlocks = new int[] { expBlocksRowCol, expBlocksRow, expBlocksNone };
+ KeyValue[] kvs = null;
+
+ for (int i = 0; i < BLOOM_TYPE.length; i++) {
+ String bloomType = BLOOM_TYPE[i];
+ byte[] cf = Bytes.toBytes(family + "_" + bloomType);
+ long blocksStart = getBlkAccessCount(cf);
+ Get get = new Get(Bytes.toBytes(row));
+
+ for (String column : columns) {
+ get.addColumn(cf, Bytes.toBytes(column));
+ }
+
+ kvs = region.get(get, null).raw();
+ long blocksEnd = getBlkAccessCount(cf);
+ if (expBlocks[i] != -1) {
+ assertEquals("Blocks Read Check for Bloom: " + bloomType, expBlocks[i],
+ blocksEnd - blocksStart);
+ }
+ System.out.println("Blocks Read for Bloom: " + bloomType + " = "
+ + (blocksEnd - blocksStart) + "Expected = " + expBlocks[i]);
}
-
- KeyValue[] kvs = region.get(get, null).raw();
- long blocksEnd = getBlkAccessCount(cf);
- if (expBlocks != -1) {
- assertEquals("Blocks Read Check", expBlocks, blocksEnd - blocksStart);
- }
- System.out.println("Blocks Read = " + (blocksEnd - blocksStart) +
- "Expected = " + expBlocks);
return kvs;
}
- private KeyValue[] getData(byte[] cf, String row, String column,
- int expBlocks)
- throws IOException {
- return getData(cf, row, Arrays.asList(column), expBlocks);
+ private KeyValue[] getData(String family, String row, String column,
+ int expBlocks) throws IOException {
+ return getData(family, row, Arrays.asList(column), expBlocks, expBlocks,
+ expBlocks);
+ }
+
+ private KeyValue[] getData(String family, String row, String column,
+ int expBlocksRowCol, int expBlocksRow, int expBlocksNone)
+ throws IOException {
+ return getData(family, row, Arrays.asList(column), expBlocksRowCol,
+ expBlocksRow, expBlocksNone);
}
- private void deleteFamily(byte[] cf, String row, long version)
- throws IOException {
+ private void deleteFamily(String family, String row, long version)
+ throws IOException {
Delete del = new Delete(Bytes.toBytes(row));
- del.deleteFamily(cf, version);
+ del.deleteFamily(Bytes.toBytes(family + "_ROWCOL"), version);
+ del.deleteFamily(Bytes.toBytes(family + "_ROW"), version);
+ del.deleteFamily(Bytes.toBytes(family + "_NONE"), version);
region.delete(del, null, true);
}
private static void verifyData(KeyValue kv, String expectedRow,
- String expectedCol, long expectedVersion) {
+ String expectedCol, long expectedVersion) {
assertEquals("RowCheck", expectedRow, Bytes.toString(kv.getRow()));
assertEquals("ColumnCheck", expectedCol, Bytes.toString(kv.getQualifier()));
assertEquals("TSCheck", expectedVersion, kv.getTimestamp());
assertEquals("ValueCheck",
- Bytes.toString(genValue(expectedRow, expectedCol, expectedVersion)),
- Bytes.toString(kv.getValue()));
+ Bytes.toString(genValue(expectedRow, expectedCol, expectedVersion)),
+ Bytes.toString(kv.getValue()));
}
private static long getBlkAccessCount(byte[] cf) {
@@ -150,17 +174,16 @@ public class TestBlocksRead extends HBas
/**
* Test # of blocks read for some simple seek cases.
+ *
* @throws Exception
*/
@Test
public void testBlocksRead() throws Exception {
- byte [] TABLE = Bytes.toBytes("testBlocksRead");
- byte [] FAMILY = Bytes.toBytes("cf1");
- byte [][] FAMILIES = new byte[][] { FAMILY };
+ byte[] TABLE = Bytes.toBytes("testBlocksRead");
+ String FAMILY = "cf1";
KeyValue kvs[];
-
HBaseConfiguration conf = getConf();
- initHRegion(TABLE, getName(), conf, FAMILIES);
+ initHRegion(TABLE, getName(), conf, FAMILY);
putData(FAMILY, "row", "col1", 1);
putData(FAMILY, "row", "col2", 2);
@@ -171,44 +194,31 @@ public class TestBlocksRead extends HBas
putData(FAMILY, "row", "col7", 7);
region.flushcache();
- // Expected block reads: 1 (after fixes for HBASE-4433 & HBASE-4434).
- // The top block contains our answer (including DeleteFamily if present).
- // So we should only be reading 1 block
+ // Expected block reads: 1
+ // The top block has the KV we are
+ // interested. So only 1 seek is needed.
kvs = getData(FAMILY, "row", "col1", 1);
assertEquals(1, kvs.length);
verifyData(kvs[0], "row", "col1", 1);
// Expected block reads: 2
- // The first one should be able to process any Delete marker for the
- // row if present, and then get col1. The second one will get col2
+ // The top block and next block has the KVs we are
+ // interested. So only 2 seek is needed.
kvs = getData(FAMILY, "row", Arrays.asList("col1", "col2"), 2);
assertEquals(2, kvs.length);
verifyData(kvs[0], "row", "col1", 1);
verifyData(kvs[1], "row", "col2", 2);
// Expected block reads: 3
- // The first one should be able to process any Delete marker at the top of the
- // row. This will take us to the block containing col1. But that's not a column
- // we are interested in. The two more seeks will be for col2 and col3.
+ // The first 2 seeks is to find out col2. [HBASE-4443]
+ // One additional seek for col3
+ // So 3 seeks are needed.
kvs = getData(FAMILY, "row", Arrays.asList("col2", "col3"), 3);
assertEquals(2, kvs.length);
verifyData(kvs[0], "row", "col2", 2);
verifyData(kvs[1], "row", "col3", 3);
- // Expected block reads: 2
- // Unfortunately, this is a common case when KVs are large, and occupy
- // 1 block each.
- // This issue has been reported as HBASE-4443.
- // Since HBASE-4469 is fixed now, the seek for delete family marker has been
- // optimized.
- // Explanation of 2 seeks:
- // * The 1st one will be block containing col4, because we search for
- // row/col5/TS=Long.MAX_VAL.
- // This will land us in the previous block, and not the block containing
- // row/col5.
- // * The final block we read will be the actual block that contains our data.
- // When HBASE-4443 is fixed, the number of expected blocks here should be
- // dropped to 1.
+ // Expected block reads: 2. [HBASE-4443]
kvs = getData(FAMILY, "row", Arrays.asList("col5"), 2);
assertEquals(1, kvs.length);
verifyData(kvs[0], "row", "col5", 5);
@@ -216,17 +226,16 @@ public class TestBlocksRead extends HBas
/**
* Test # of blocks read (targetted at some of the cases Lazy Seek optimizes).
+ *
* @throws Exception
*/
@Test
public void testLazySeekBlocksRead() throws Exception {
- byte [] TABLE = Bytes.toBytes("testLazySeekBlocksRead");
- byte [] FAMILY = Bytes.toBytes("cf1");
- byte [][] FAMILIES = new byte[][] { FAMILY };
+ byte[] TABLE = Bytes.toBytes("testLazySeekBlocksRead");
+ String FAMILY = "cf1";
KeyValue kvs[];
-
HBaseConfiguration conf = getConf();
- initHRegion(TABLE, getName(), conf, FAMILIES);
+ initHRegion(TABLE, getName(), conf, FAMILY);
// File 1
putData(FAMILY, "row", "col1", 1);
@@ -239,8 +248,6 @@ public class TestBlocksRead extends HBas
region.flushcache();
// Expected blocks read: 1.
- // Since HBASE-4469 is fixed now, the seek for delete family marker has been
- // optimized.
// File 2's top block is also the KV we are
// interested. So only 1 seek is needed.
kvs = getData(FAMILY, "row", Arrays.asList("col1"), 1);
@@ -248,12 +255,9 @@ public class TestBlocksRead extends HBas
verifyData(kvs[0], "row", "col1", 3);
// Expected blocks read: 2
- //
- // Since HBASE-4469 is fixed now, the seek for delete family marker has been
- // optimized. File 2's top block has the "col1" KV we are
+ // File 2's top block has the "col1" KV we are
// interested. We also need "col2" which is in a block
// of its own. So, we need that block as well.
- //
kvs = getData(FAMILY, "row", Arrays.asList("col1", "col2"), 2);
assertEquals(2, kvs.length);
verifyData(kvs[0], "row", "col1", 3);
@@ -264,26 +268,17 @@ public class TestBlocksRead extends HBas
region.flushcache();
// Expected blocks read: 1
- //
- // Since HBASE-4469 is fixed now, the seek for delete family marker has been
- // optimized. File 3's top block has the "col3" KV we are
+ // File 3's top block has the "col3" KV we are
// interested. So only 1 seek is needed.
- //
kvs = getData(FAMILY, "row", "col3", 1);
assertEquals(1, kvs.length);
verifyData(kvs[0], "row", "col3", 5);
// Get a column from older file.
- // Expected blocks read: 3
- //
- // Since HBASE-4469 is fixed now, the seek for delete family marker has been
- // optimized. File 2's top block has the "col1" KV we are
- // interested.
- //
- // Also no need to seek to file 3 since the row-col bloom filter is enabled.
- // Only 1 seek in File 2 is needed.
- //
- kvs = getData(FAMILY, "row", Arrays.asList("col1"), 1);
+ // For ROWCOL Bloom filter: Expected blocks read: 1.
+ // For ROW Bloom filter: Expected blocks read: 2.
+ // For NONE Bloom filter: Expected blocks read: 2.
+ kvs = getData(FAMILY, "row", Arrays.asList("col1"), 1, 2, 2);
assertEquals(1, kvs.length);
verifyData(kvs[0], "row", "col1", 3);
@@ -291,10 +286,12 @@ public class TestBlocksRead extends HBas
deleteFamily(FAMILY, "row", 6);
region.flushcache();
- // Expected blocks read: 2. [HBASE-4585]
- kvs = getData(FAMILY, "row", "col1", 2);
+ // For ROWCOL Bloom filter: Expected blocks read: 2.
+ // For ROW Bloom filter: Expected blocks read: 3.
+ // For NONE Bloom filter: Expected blocks read: 3.
+ kvs = getData(FAMILY, "row", "col1", 2, 3, 3);
assertEquals(0, kvs.length);
- kvs = getData(FAMILY, "row", "col2", 3);
+ kvs = getData(FAMILY, "row", "col2", 3, 4, 4);
assertEquals(0, kvs.length);
kvs = getData(FAMILY, "row", "col3", 2);
assertEquals(0, kvs.length);
@@ -320,6 +317,7 @@ public class TestBlocksRead extends HBas
putData(FAMILY, "row", "col3", 13);
region.flushcache();
+
// Expected blocks read: 5. [HBASE-4585]
kvs = getData(FAMILY, "row", Arrays.asList("col1", "col2", "col3"), 5);
assertEquals(3, kvs.length);
@@ -327,27 +325,25 @@ public class TestBlocksRead extends HBas
verifyData(kvs[1], "row", "col2", 12);
verifyData(kvs[2], "row", "col3", 13);
}
-
+
@Test
public void testLazySeekBlocksReadWithDelete() throws Exception {
- byte [] TABLE = Bytes.toBytes("testLazySeekBlocksReadWithDelete");
- byte [] FAMILY = Bytes.toBytes("cf1");
- byte [][] FAMILIES = new byte[][] { FAMILY };
+ byte[] TABLE = Bytes.toBytes("testLazySeekBlocksReadWithDelete");
+ String FAMILY = "cf1";
KeyValue kvs[];
-
HBaseConfiguration conf = getConf();
- initHRegion(TABLE, getName(), conf, FAMILIES);
-
+ initHRegion(TABLE, getName(), conf, FAMILY);
+
deleteFamily(FAMILY, "row", 200);
- for (int i = 0 ; i < 100; i++) {
- putData(FAMILY, "row", "col"+i, i);
+ for (int i = 0; i < 100; i++) {
+ putData(FAMILY, "row", "col" + i, i);
}
putData(FAMILY, "row", "col99", 201);
region.flushcache();
kvs = getData(FAMILY, "row", Arrays.asList("col0"), 2);
assertEquals(0, kvs.length);
-
+
kvs = getData(FAMILY, "row", Arrays.asList("col99"), 2);
assertEquals(1, kvs.length);
verifyData(kvs[0], "row", "col99", 201);
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java?rev=1188367&r1=1188366&r2=1188367&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java Mon Oct 24 20:44:45 2011
@@ -212,7 +212,7 @@ public class TestCompoundBloomFilter {
try {
String fakeLookupModeStr = ", fake lookup is " + (fakeLookupEnabled ?
"enabled" : "disabled");
- CompoundBloomFilter cbf = (CompoundBloomFilter) r.getBloomFilter();
+ CompoundBloomFilter cbf = (CompoundBloomFilter) r.getGeneralBloomFilter();
cbf.enableTestingStats();
int numFalsePos = 0;
Random rand = new Random(EVALUATION_SEED);
@@ -289,10 +289,10 @@ public class TestCompoundBloomFilter {
HBaseTestingUtility.getTestDir(), BLOCK_SIZES[t], null, null, conf,
bt, 0);
- assertTrue(w.hasBloom());
- assertTrue(w.getBloomWriter() instanceof CompoundBloomFilterWriter);
+ assertTrue(w.hasGeneralBloom());
+ assertTrue(w.getGeneralBloomWriter() instanceof CompoundBloomFilterWriter);
CompoundBloomFilterWriter cbbf =
- (CompoundBloomFilterWriter) w.getBloomWriter();
+ (CompoundBloomFilterWriter) w.getGeneralBloomWriter();
int keyCount = 0;
KeyValue prev = null;
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java?rev=1188367&r1=1188366&r2=1188367&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java Mon Oct 24 20:44:45 2011
@@ -396,6 +396,60 @@ public class TestStoreFile extends HBase
bloomWriteRead(writer, fs);
}
+ public void testDeleteFamilyBloomFilter() throws Exception {
+ FileSystem fs = FileSystem.getLocal(conf);
+ conf.setFloat(BloomFilterFactory.IO_STOREFILE_BLOOM_ERROR_RATE,
+ (float) 0.01);
+ conf.setBoolean(BloomFilterFactory.IO_STOREFILE_BLOOM_ENABLED, true);
+ float err = conf.getFloat(BloomFilterFactory.IO_STOREFILE_BLOOM_ERROR_RATE,
+ 0);
+
+ // write the file
+ Path f = new Path(ROOT_DIR, getName());
+ StoreFile.Writer writer = new StoreFile.Writer(fs, f,
+ StoreFile.DEFAULT_BLOCKSIZE_SMALL, HFile.DEFAULT_COMPRESSION_ALGORITHM,
+ conf, KeyValue.COMPARATOR, StoreFile.BloomType.NONE, 2000);
+
+ // add delete family
+ long now = System.currentTimeMillis();
+ for (int i = 0; i < 2000; i += 2) {
+ String row = String.format(localFormatter, i);
+ KeyValue kv = new KeyValue(row.getBytes(), "family".getBytes(),
+ "col".getBytes(), now, KeyValue.Type.DeleteFamily, "value".getBytes());
+ writer.append(kv);
+ }
+ writer.close();
+
+ StoreFile.Reader reader = new StoreFile.Reader(fs, writer.getPath(), null,
+ false, false);
+ reader.loadFileInfo();
+ reader.loadBloomfilter();
+
+ // check false positives rate
+ int falsePos = 0;
+ int falseNeg = 0;
+ for (int i = 0; i < 2000; i++) {
+ String row = String.format(localFormatter, i);
+ byte[] rowKey = Bytes.toBytes(row);
+ boolean exists = reader.passesDeleteFamilyBloomFilter(rowKey, 0,
+ rowKey.length);
+ if (i % 2 == 0) {
+ if (!exists)
+ falseNeg++;
+ } else {
+ if (exists)
+ falsePos++;
+ }
+ }
+ assertEquals(1000, reader.getDeleteFamilyCnt());
+ reader.close();
+ fs.delete(f, true);
+ assertEquals("False negatives: " + falseNeg, 0, falseNeg);
+ int maxFalsePos = (int) (2 * 2000 * err);
+ assertTrue("Too many false positives: " + falsePos + " (err=" + err
+ + ", expected no more than " + maxFalsePos, falsePos <= maxFalsePos);
+ }
+
public void testBloomTypes() throws Exception {
float err = (float) 0.01;
FileSystem fs = FileSystem.getLocal(conf);
@@ -443,7 +497,7 @@ public class TestStoreFile extends HBase
reader.loadFileInfo();
reader.loadBloomfilter();
StoreFileScanner scanner = reader.getStoreFileScanner(false, false);
- assertEquals(expKeys[x], reader.bloomFilter.getKeyCount());
+ assertEquals(expKeys[x], reader.generalBloomFilter.getKeyCount());
// check false positives rate
int falsePos = 0;
@@ -493,7 +547,7 @@ public class TestStoreFile extends HBase
StoreFile.Writer writer = new StoreFile.Writer(fs, f,
StoreFile.DEFAULT_BLOCKSIZE_SMALL, HFile.DEFAULT_COMPRESSION_ALGORITHM,
conf, KeyValue.COMPARATOR, StoreFile.BloomType.ROW, 2000);
- assertFalse(writer.hasBloom());
+ assertFalse(writer.hasGeneralBloom());
writer.close();
fs.delete(f, true);
@@ -516,7 +570,7 @@ public class TestStoreFile extends HBase
writer = new StoreFile.Writer(fs, f,
StoreFile.DEFAULT_BLOCKSIZE_SMALL, HFile.DEFAULT_COMPRESSION_ALGORITHM,
conf, KeyValue.COMPARATOR, StoreFile.BloomType.ROW, Integer.MAX_VALUE);
- assertFalse(writer.hasBloom());
+ assertFalse(writer.hasGeneralBloom());
writer.close();
fs.delete(f, true);
}