You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@orc.apache.org by om...@apache.org on 2019/08/02 19:13:29 UTC
[orc] branch branch-1.5 updated: ORC-536: "expectedEntries should
be > 0" error when bloom filters enabled, but rowIndexStride set to 0
This is an automated email from the ASF dual-hosted git repository.
omalley pushed a commit to branch branch-1.5
in repository https://gitbox.apache.org/repos/asf/orc.git
The following commit(s) were added to refs/heads/branch-1.5 by this push:
new 378e5e2 ORC-536: "expectedEntries should be > 0" error when bloom filters enabled, but rowIndexStride set to 0
378e5e2 is described below
commit 378e5e20455402ff49356d185a4e4551b8a256fd
Author: Jason Dere <jd...@cloudera.com>
AuthorDate: Tue Jul 23 15:09:50 2019 -0700
ORC-536: "expectedEntries should be > 0" error when bloom filters enabled,
but rowIndexStride set to 0
Fixes #414
Signed-off-by: Owen O'Malley <om...@apache.org>
---
java/core/src/java/org/apache/orc/impl/WriterImpl.java | 4 ++++
java/core/src/java/org/apache/orc/util/BloomFilter.java | 2 +-
java/core/src/test/org/apache/orc/impl/TestWriterImpl.java | 11 +++++++++++
3 files changed, 16 insertions(+), 1 deletion(-)
diff --git a/java/core/src/java/org/apache/orc/impl/WriterImpl.java b/java/core/src/java/org/apache/orc/impl/WriterImpl.java
index 827747e..5ec7d70 100644
--- a/java/core/src/java/org/apache/orc/impl/WriterImpl.java
+++ b/java/core/src/java/org/apache/orc/impl/WriterImpl.java
@@ -166,8 +166,12 @@ public class WriterImpl implements WriterInternal, MemoryManager.Callback {
" readable by other versions of the software. It is only for" +
" developer testing.");
}
+ boolean buildBloomFilter = buildIndex;
if (version == OrcFile.Version.V_0_11) {
/* do not write bloom filters for ORC v11 */
+ buildBloomFilter = false;
+ }
+ if (!buildBloomFilter) {
this.bloomFilterColumns = new boolean[schema.getMaximumId() + 1];
} else {
this.bloomFilterColumns =
diff --git a/java/core/src/java/org/apache/orc/util/BloomFilter.java b/java/core/src/java/org/apache/orc/util/BloomFilter.java
index c7ed5ee..dd75b9a 100644
--- a/java/core/src/java/org/apache/orc/util/BloomFilter.java
+++ b/java/core/src/java/org/apache/orc/util/BloomFilter.java
@@ -60,7 +60,7 @@ public class BloomFilter {
}
public BloomFilter(long expectedEntries, double fpp) {
- checkArgument(expectedEntries > 0, "expectedEntries should be > 0");
+ expectedEntries = Math.max(expectedEntries, 1);
checkArgument(fpp > 0.0 && fpp < 1.0, "False positive probability should be > 0.0 & < 1.0");
int nb = optimalNumOfBits(expectedEntries, fpp);
// make 'm' multiple of 64
diff --git a/java/core/src/test/org/apache/orc/impl/TestWriterImpl.java b/java/core/src/test/org/apache/orc/impl/TestWriterImpl.java
index 343ce5f..21dd7ed 100644
--- a/java/core/src/test/org/apache/orc/impl/TestWriterImpl.java
+++ b/java/core/src/test/org/apache/orc/impl/TestWriterImpl.java
@@ -70,4 +70,15 @@ public class TestWriterImpl {
Writer w = OrcFile.createWriter(testFilePath, OrcFile.writerOptions(conf).setSchema(schema));
w.close();
}
+
+ @Test
+ public void testNoBFIfNoIndex() throws Exception {
+ // overriding the flag should result in a successful write (no exception)
+ conf.set(OrcConf.OVERWRITE_OUTPUT_FILE.getAttribute(), "true");
+ // Enable bloomfilter, but disable index
+ conf.set(OrcConf.ROW_INDEX_STRIDE.getAttribute(), "0");
+ conf.set(OrcConf.BLOOM_FILTER_COLUMNS.getAttribute(), "*");
+ Writer w = OrcFile.createWriter(testFilePath, OrcFile.writerOptions(conf).setSchema(schema));
+ w.close();
+ }
}