You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2020/10/06 22:41:21 UTC
[incubator-pinot] 01/01: Reduce fpp to 0.001 and rmove the size
limit for bloom filter
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch bloom_filter_test
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 73972a602c4680167963cb5373cc792d992334ec
Author: Xiaotian (Jackie) Jiang <ja...@gmail.com>
AuthorDate: Tue Oct 6 15:40:54 2020 -0700
Reduce fpp to 0.001 and rmove the size limit for bloom filter
---
.../creator/impl/bloom/BloomFilterCreator.java | 20 ++++++++------------
1 file changed, 8 insertions(+), 12 deletions(-)
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/bloom/BloomFilterCreator.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/bloom/BloomFilterCreator.java
index 7c1a6c4..547e478 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/bloom/BloomFilterCreator.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/bloom/BloomFilterCreator.java
@@ -23,7 +23,6 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import org.apache.pinot.core.bloom.BloomFilter;
-import org.apache.pinot.core.bloom.BloomFilterUtil;
import org.apache.pinot.core.bloom.SegmentBloomFilterFactory;
import org.apache.pinot.core.segment.creator.impl.V1Constants;
@@ -39,17 +38,18 @@ import org.apache.pinot.core.segment.creator.impl.V1Constants;
* 0.45 when the filter size is 1MB.
*/
public class BloomFilterCreator implements AutoCloseable {
- private static double DEFAULT_MAX_FALSE_POS_PROBABILITY = 0.05;
- private static int MB_IN_BITS = 8388608;
+ private static final double DEFAULT_MAX_FALSE_POS_PROBABILITY = 0.001;
- private BloomFilter _bloomFilter;
- private File _bloomFilterFile;
+ private final BloomFilter _bloomFilter;
+ private final File _bloomFilterFile;
public BloomFilterCreator(File indexDir, String columnName, int cardinality) {
+ _bloomFilter = SegmentBloomFilterFactory.createSegmentBloomFilter(cardinality, DEFAULT_MAX_FALSE_POS_PROBABILITY);
_bloomFilterFile = new File(indexDir, columnName + V1Constants.Indexes.BLOOM_FILTER_FILE_EXTENSION);
- double maxFalsePosProbability = BloomFilterUtil
- .computeMaxFalsePositiveProbabilityForNumBits(cardinality, MB_IN_BITS, DEFAULT_MAX_FALSE_POS_PROBABILITY);
- _bloomFilter = SegmentBloomFilterFactory.createSegmentBloomFilter(cardinality, maxFalsePosProbability);
+ }
+
+ public void add(Object input) {
+ _bloomFilter.add(input.toString());
}
@Override
@@ -61,8 +61,4 @@ public class BloomFilterCreator implements AutoCloseable {
_bloomFilter.writeTo(outputStream);
}
}
-
- public void add(Object input) {
- _bloomFilter.add(input.toString());
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org