You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@orc.apache.org by om...@apache.org on 2019/08/02 18:57:17 UTC
[orc] branch master updated: ORC-536: "expectedEntries should be >
0" error when bloom filters enabled, but rowIndexStride set to 0
This is an automated email from the ASF dual-hosted git repository.
omalley pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/orc.git
The following commit(s) were added to refs/heads/master by this push:
new e2e6a68 ORC-536: "expectedEntries should be > 0" error when bloom filters enabled, but rowIndexStride set to 0
e2e6a68 is described below
commit e2e6a68d00410435994619225ee2be4b66407c1f
Author: Jason Dere <jd...@cloudera.com>
AuthorDate: Tue Jul 23 15:09:50 2019 -0700
ORC-536: "expectedEntries should be > 0" error when bloom filters enabled,
but rowIndexStride set to 0
Fixes #414
Signed-off-by: Owen O'Malley <om...@apache.org>
---
java/core/src/java/org/apache/orc/impl/WriterImpl.java | 4 ++++
java/core/src/java/org/apache/orc/util/BloomFilter.java | 2 +-
java/core/src/test/org/apache/orc/impl/TestWriterImpl.java | 11 +++++++++++
3 files changed, 16 insertions(+), 1 deletion(-)
diff --git a/java/core/src/java/org/apache/orc/impl/WriterImpl.java b/java/core/src/java/org/apache/orc/impl/WriterImpl.java
index 7f9cb63..b914263 100644
--- a/java/core/src/java/org/apache/orc/impl/WriterImpl.java
+++ b/java/core/src/java/org/apache/orc/impl/WriterImpl.java
@@ -196,8 +196,12 @@ public class WriterImpl implements WriterInternal, MemoryManager.Callback {
" readable by other versions of the software. It is only for" +
" developer testing.");
}
+ boolean buildBloomFilter = buildIndex;
if (version == OrcFile.Version.V_0_11) {
/* do not write bloom filters for ORC v11 */
+ buildBloomFilter = false;
+ }
+ if (!buildBloomFilter) {
this.bloomFilterColumns = new boolean[schema.getMaximumId() + 1];
} else {
this.bloomFilterColumns =
diff --git a/java/core/src/java/org/apache/orc/util/BloomFilter.java b/java/core/src/java/org/apache/orc/util/BloomFilter.java
index c7ed5ee..dd75b9a 100644
--- a/java/core/src/java/org/apache/orc/util/BloomFilter.java
+++ b/java/core/src/java/org/apache/orc/util/BloomFilter.java
@@ -60,7 +60,7 @@ public class BloomFilter {
}
public BloomFilter(long expectedEntries, double fpp) {
- checkArgument(expectedEntries > 0, "expectedEntries should be > 0");
+ expectedEntries = Math.max(expectedEntries, 1);
checkArgument(fpp > 0.0 && fpp < 1.0, "False positive probability should be > 0.0 & < 1.0");
int nb = optimalNumOfBits(expectedEntries, fpp);
// make 'm' multiple of 64
diff --git a/java/core/src/test/org/apache/orc/impl/TestWriterImpl.java b/java/core/src/test/org/apache/orc/impl/TestWriterImpl.java
index 343ce5f..21dd7ed 100644
--- a/java/core/src/test/org/apache/orc/impl/TestWriterImpl.java
+++ b/java/core/src/test/org/apache/orc/impl/TestWriterImpl.java
@@ -70,4 +70,15 @@ public class TestWriterImpl {
Writer w = OrcFile.createWriter(testFilePath, OrcFile.writerOptions(conf).setSchema(schema));
w.close();
}
+
+ @Test
+ public void testNoBFIfNoIndex() throws Exception {
+ // overriding the flag should result in a successful write (no exception)
+ conf.set(OrcConf.OVERWRITE_OUTPUT_FILE.getAttribute(), "true");
+ // Enable bloomfilter, but disable index
+ conf.set(OrcConf.ROW_INDEX_STRIDE.getAttribute(), "0");
+ conf.set(OrcConf.BLOOM_FILTER_COLUMNS.getAttribute(), "*");
+ Writer w = OrcFile.createWriter(testFilePath, OrcFile.writerOptions(conf).setSchema(schema));
+ w.close();
+ }
}