[orc] branch master updated: ORC-683: PPD: Make Floating point NaN check more strict (#562)

[do...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/orc.git


The following commit(s) were added to refs/heads/master by this push:
     new a6584ad  ORC-683: PPD: Make Floating point NaN check more strict (#562)
a6584ad is described below

commit a6584ade0ee93985a0e99cc069dbecdcd385f904
Author: Panagiotis Garefalakis <pg...@cloudera.com>
AuthorDate: Thu Nov 5 21:07:59 2020 +0000

    ORC-683: PPD: Make Floating point NaN check more strict (#562)
    
    ### What changes were proposed in this pull request?
    ORC-629 added an extra check (part of  RecordReaderImp) disabling PPD when DoubleColumnStats min, max or sum is a non-Finite number.
    
    However, the check is unnecessarily broad – the way stats are updated, using just sum for the check would suffice.
    https://github.com/apache/orc/blob/master/java/core/src/java/org/apache/orc/impl/ColumnStatisticsImpl.java#L568
    
    
    ### Why are the changes needed?
    Make PPD NaN check for floating point stats more strict
    
    
    ### How was this patch tested?
    TestVectorOrcFile.testPredicatePushdownWithNan()
---
 java/core/src/java/org/apache/orc/impl/RecordReaderImpl.java | 3 +--
 java/core/src/test/org/apache/orc/TestVectorOrcFile.java     | 4 ++--
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/java/core/src/java/org/apache/orc/impl/RecordReaderImpl.java b/java/core/src/java/org/apache/orc/impl/RecordReaderImpl.java
index aba2e4f..e502254 100644
--- a/java/core/src/java/org/apache/orc/impl/RecordReaderImpl.java
+++ b/java/core/src/java/org/apache/orc/impl/RecordReaderImpl.java
@@ -533,8 +533,7 @@ public class RecordReaderImpl implements RecordReader {
       return TruthValue.YES_NO_NULL;
     } else if (category == TypeDescription.Category.DOUBLE) {
       DoubleColumnStatistics dstas = (DoubleColumnStatistics) cs;
-      if (!Double.isFinite(dstas.getMinimum()) || !Double.isFinite(dstas.getMaximum())
-              || !Double.isFinite(dstas.getSum())) {
+      if (!Double.isFinite(dstas.getSum())) {
         LOG.debug("Not using predication pushdown on {} because stats contain NaN values",
                 predicate.getColumnName());
         return dstas.hasNull() ? TruthValue.YES_NO_NULL : TruthValue.YES_NO;
diff --git a/java/core/src/test/org/apache/orc/TestVectorOrcFile.java b/java/core/src/test/org/apache/orc/TestVectorOrcFile.java
index 5c69cce..3b89db9 100644
--- a/java/core/src/test/org/apache/orc/TestVectorOrcFile.java
+++ b/java/core/src/test/org/apache/orc/TestVectorOrcFile.java
@@ -4025,7 +4025,7 @@ public class TestVectorOrcFile {
 
     DoubleColumnVector dbcol = ((DoubleColumnVector) batch.cols[0]);
 
-    // first row NaN (resulting to both min and max columnStats of stride to be NaN)
+    // first row NaN (resulting to min/max and sum columnStats of stride to be NaN)
     // NaN in the middle of a stride causes Sum of last stride to be NaN
     dbcol.vector[0] = Double.NaN;
     for (int i=1; i < 3500; ++i) {
@@ -4051,7 +4051,7 @@ public class TestVectorOrcFile {
     batch = reader.getSchema().createRowBatch(3500);
 
     rows.nextBatch(batch);
-    // First stride should be read as NaN min/max are ignored
+    // First stride should be read as NaN sum is ignored
     assertEquals(1000, batch.size);
 
     rows.nextBatch(batch);