You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2020/10/06 22:41:20 UTC
[incubator-pinot] branch bloom_filter_test created (now 73972a6)
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a change to branch bloom_filter_test
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
at 73972a6 Reduce fpp to 0.001 and rmove the size limit for bloom filter
This branch includes the following new commits:
new 73972a6 Reduce fpp to 0.001 and rmove the size limit for bloom filter
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[incubator-pinot] 01/01: Reduce fpp to 0.001 and rmove the size
limit for bloom filter
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch bloom_filter_test
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 73972a602c4680167963cb5373cc792d992334ec
Author: Xiaotian (Jackie) Jiang <ja...@gmail.com>
AuthorDate: Tue Oct 6 15:40:54 2020 -0700
Reduce fpp to 0.001 and rmove the size limit for bloom filter
---
.../creator/impl/bloom/BloomFilterCreator.java | 20 ++++++++------------
1 file changed, 8 insertions(+), 12 deletions(-)
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/bloom/BloomFilterCreator.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/bloom/BloomFilterCreator.java
index 7c1a6c4..547e478 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/bloom/BloomFilterCreator.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/bloom/BloomFilterCreator.java
@@ -23,7 +23,6 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import org.apache.pinot.core.bloom.BloomFilter;
-import org.apache.pinot.core.bloom.BloomFilterUtil;
import org.apache.pinot.core.bloom.SegmentBloomFilterFactory;
import org.apache.pinot.core.segment.creator.impl.V1Constants;
@@ -39,17 +38,18 @@ import org.apache.pinot.core.segment.creator.impl.V1Constants;
* 0.45 when the filter size is 1MB.
*/
public class BloomFilterCreator implements AutoCloseable {
- private static double DEFAULT_MAX_FALSE_POS_PROBABILITY = 0.05;
- private static int MB_IN_BITS = 8388608;
+ private static final double DEFAULT_MAX_FALSE_POS_PROBABILITY = 0.001;
- private BloomFilter _bloomFilter;
- private File _bloomFilterFile;
+ private final BloomFilter _bloomFilter;
+ private final File _bloomFilterFile;
public BloomFilterCreator(File indexDir, String columnName, int cardinality) {
+ _bloomFilter = SegmentBloomFilterFactory.createSegmentBloomFilter(cardinality, DEFAULT_MAX_FALSE_POS_PROBABILITY);
_bloomFilterFile = new File(indexDir, columnName + V1Constants.Indexes.BLOOM_FILTER_FILE_EXTENSION);
- double maxFalsePosProbability = BloomFilterUtil
- .computeMaxFalsePositiveProbabilityForNumBits(cardinality, MB_IN_BITS, DEFAULT_MAX_FALSE_POS_PROBABILITY);
- _bloomFilter = SegmentBloomFilterFactory.createSegmentBloomFilter(cardinality, maxFalsePosProbability);
+ }
+
+ public void add(Object input) {
+ _bloomFilter.add(input.toString());
}
@Override
@@ -61,8 +61,4 @@ public class BloomFilterCreator implements AutoCloseable {
_bloomFilter.writeTo(outputStream);
}
}
-
- public void add(Object input) {
- _bloomFilter.add(input.toString());
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org