You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@parquet.apache.org by al...@apache.org on 2015/02/03 21:53:47 UTC
incubator-parquet-mr git commit: PARQUET-173: Fixes
`StatisticsFilter` for `And` filter predicate
Repository: incubator-parquet-mr
Updated Branches:
refs/heads/master 3df3372a1 -> 80417356f
PARQUET-173: Fixes `StatisticsFilter` for `And` filter predicate
<!-- Reviewable:start -->
[<img src="https://reviewable.io/review_button.png" height=40 alt="Review on Reviewable"/>](https://reviewable.io/reviews/apache/incubator-parquet-mr/108)
<!-- Reviewable:end -->
Author: Cheng Lian <li...@databricks.com>
Closes #108 from liancheng/PARQUET-173 and squashes the following commits:
d188f0b [Cheng Lian] Fixes test case
be2c8a1 [Cheng Lian] Fixes `StatisticsFilter` for `And` filter predicate
Project: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/commit/80417356
Tree: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/tree/80417356
Diff: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/diff/80417356
Branch: refs/heads/master
Commit: 80417356f04c5ee1cd6f636e9b043db3f2de24f2
Parents: 3df3372
Author: Cheng Lian <li...@databricks.com>
Authored: Tue Feb 3 12:53:37 2015 -0800
Committer: Alex Levenson <al...@twitter.com>
Committed: Tue Feb 3 12:53:37 2015 -0800
----------------------------------------------------------------------
.../java/parquet/filter2/statisticslevel/StatisticsFilter.java | 6 +++++-
.../parquet/filter2/statisticslevel/TestStatisticsFilter.java | 4 ++--
2 files changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/80417356/parquet-hadoop/src/main/java/parquet/filter2/statisticslevel/StatisticsFilter.java
----------------------------------------------------------------------
diff --git a/parquet-hadoop/src/main/java/parquet/filter2/statisticslevel/StatisticsFilter.java b/parquet-hadoop/src/main/java/parquet/filter2/statisticslevel/StatisticsFilter.java
index f7eea41..d6a484c 100644
--- a/parquet-hadoop/src/main/java/parquet/filter2/statisticslevel/StatisticsFilter.java
+++ b/parquet-hadoop/src/main/java/parquet/filter2/statisticslevel/StatisticsFilter.java
@@ -242,7 +242,11 @@ public class StatisticsFilter implements FilterPredicate.Visitor<Boolean> {
@Override
public Boolean visit(And and) {
- return and.getLeft().accept(this) && and.getRight().accept(this);
+ // seems unintuitive to put an || not an && here but we can
+ // drop a chunk of records if we know that either the left or
+ // the right predicate agrees that no matter what we don't
+ // need this chunk.
+ return and.getLeft().accept(this) || and.getRight().accept(this);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/80417356/parquet-hadoop/src/test/java/parquet/filter2/statisticslevel/TestStatisticsFilter.java
----------------------------------------------------------------------
diff --git a/parquet-hadoop/src/test/java/parquet/filter2/statisticslevel/TestStatisticsFilter.java b/parquet-hadoop/src/test/java/parquet/filter2/statisticslevel/TestStatisticsFilter.java
index 03502d2..d558f9b 100644
--- a/parquet-hadoop/src/test/java/parquet/filter2/statisticslevel/TestStatisticsFilter.java
+++ b/parquet-hadoop/src/test/java/parquet/filter2/statisticslevel/TestStatisticsFilter.java
@@ -222,8 +222,8 @@ public class TestStatisticsFilter {
FilterPredicate yes = eq(intColumn, 9);
FilterPredicate no = eq(doubleColumn, 50D);
assertTrue(canDrop(and(yes, yes), columnMetas));
- assertFalse(canDrop(and(yes, no), columnMetas));
- assertFalse(canDrop(and(no, yes), columnMetas));
+ assertTrue(canDrop(and(yes, no), columnMetas));
+ assertTrue(canDrop(and(no, yes), columnMetas));
assertFalse(canDrop(and(no, no), columnMetas));
}