You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by al...@apache.org on 2022/11/07 19:49:00 UTC
[arrow-datafusion] branch master updated: Test page index filtering in parquet integration test (#4062)
This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
The following commit(s) were added to refs/heads/master by this push:
new 532b4a751 Test page index filtering in parquet integration test (#4062)
532b4a751 is described below
commit 532b4a751860111313dba21d3f0a906937d29b8e
Author: Andrew Lamb <an...@nerdnetworks.org>
AuthorDate: Mon Nov 7 14:48:53 2022 -0500
Test page index filtering in parquet integration test (#4062)
---
datafusion/core/tests/parquet_filter_pushdown.rs | 53 ++++++++++++------------
1 file changed, 27 insertions(+), 26 deletions(-)
diff --git a/datafusion/core/tests/parquet_filter_pushdown.rs b/datafusion/core/tests/parquet_filter_pushdown.rs
index cb3f42726..2c0d75cc2 100644
--- a/datafusion/core/tests/parquet_filter_pushdown.rs
+++ b/datafusion/core/tests/parquet_filter_pushdown.rs
@@ -341,32 +341,33 @@ impl<'a> TestCase<'a> {
assert_eq!(no_pushdown, pushdown_and_reordering);
- // page index filtering is not correct:
- // https://github.com/apache/arrow-datafusion/issues/4002
- // when it is fixed we can add these tests too
-
- // let page_index_only = self
- // .read_with_options(
- // ParquetScanOptions {
- // pushdown_filters: false,
- // reorder_filters: false,
- // enable_page_index: true,
- // },
- // )
- // .await;
- //assert_eq!(no_pushdown, page_index_only);
-
- // let pushdown_reordering_and_page_index = self
- // .read_with_options(
- // ParquetScanOptions {
- // pushdown_filters: false,
- // reorder_filters: false,
- // enable_page_index: true,
- // },
- // )
- // .await;
-
- //assert_eq!(no_pushdown, pushdown_reordering_and_page_index);
+ let page_index_only = self
+ .read_with_options(
+ ParquetScanOptions {
+ pushdown_filters: false,
+ reorder_filters: false,
+ enable_page_index: true,
+ },
+ // pushdown is off so no pushdown is expected
+ PushdownExpected::None,
+ filter,
+ )
+ .await;
+ assert_eq!(no_pushdown, page_index_only);
+
+ let pushdown_reordering_and_page_index = self
+ .read_with_options(
+ ParquetScanOptions {
+ pushdown_filters: true,
+ reorder_filters: true,
+ enable_page_index: true,
+ },
+ self.pushdown_expected,
+ filter,
+ )
+ .await;
+
+ assert_eq!(no_pushdown, pushdown_reordering_and_page_index);
}
/// Reads data from a test parquet file using the specified scan options