You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by xu...@apache.org on 2026/04/27 03:25:02 UTC
(datafusion) 01/06: Skip RowFilter and page pruning for fully matched row groups
This is an automated email from the ASF dual-hosted git repository.
xudong963 pushed a commit to branch datafusion/issue-19028-benchmark
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit ee609b60951b877b54ca49e3096ed36d54e30fb7
Author: xudong.w <wx...@gmail.com>
AuthorDate: Wed Apr 15 13:21:01 2026 +0800
Skip RowFilter and page pruning for fully matched row groups
When row group statistics prove that ALL rows satisfy the filter
predicate, skip both RowFilter evaluation (late materialization)
and page index pruning for those row groups. This avoids wasted
work decoding filter columns and evaluating predicates that produce
no useful filtering.
Depends on apache/arrow-rs#9694 for the `with_fully_matched_row_groups()`
builder API.
Co-Authored-By: Claude Opus 4.6 <no...@anthropic.com>
---
Cargo.lock | 288 +++++++++------------
Cargo.toml | 19 ++
datafusion/datasource-parquet/Cargo.toml | 5 +
.../benches/parquet_fully_matched_filter.rs | 275 ++++++++++++++++++++
datafusion/datasource-parquet/src/opener.rs | 11 +-
datafusion/datasource-parquet/src/page_filter.rs | 6 +
.../datasource-parquet/src/row_group_filter.rs | 15 +-
7 files changed, 456 insertions(+), 163 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 5a76c063bb..c9481612cd 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -204,8 +204,7 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
[[package]]
name = "arrow"
version = "58.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d441fdda254b65f3e9025910eb2c2066b6295d9c8ed409522b8d2ace1ff8574c"
+source = "git+https://github.com/xudong963/arrow-rs.git?branch=skip-filter-fully-matched-row-groups#76c028492c13c9319ca189416a84b0dd132e26ca"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -227,8 +226,7 @@ dependencies = [
[[package]]
name = "arrow-arith"
version = "58.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ced5406f8b720cc0bc3aa9cf5758f93e8593cda5490677aa194e4b4b383f9a59"
+source = "git+https://github.com/xudong963/arrow-rs.git?branch=skip-filter-fully-matched-row-groups#76c028492c13c9319ca189416a84b0dd132e26ca"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -241,8 +239,7 @@ dependencies = [
[[package]]
name = "arrow-array"
version = "58.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "772bd34cacdda8baec9418d80d23d0fb4d50ef0735685bd45158b83dfeb6e62d"
+source = "git+https://github.com/xudong963/arrow-rs.git?branch=skip-filter-fully-matched-row-groups#76c028492c13c9319ca189416a84b0dd132e26ca"
dependencies = [
"ahash",
"arrow-buffer",
@@ -251,7 +248,7 @@ dependencies = [
"chrono",
"chrono-tz",
"half",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.0",
"num-complex",
"num-integer",
"num-traits",
@@ -260,8 +257,7 @@ dependencies = [
[[package]]
name = "arrow-avro"
version = "58.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "36a3aadd016f63dfd4941ae8e13539ba98a3c2995adc3c88b9336d2514f6c8a7"
+source = "git+https://github.com/xudong963/arrow-rs.git?branch=skip-filter-fully-matched-row-groups#76c028492c13c9319ca189416a84b0dd132e26ca"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -284,8 +280,7 @@ dependencies = [
[[package]]
name = "arrow-buffer"
version = "58.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "898f4cf1e9598fdb77f356fdf2134feedfd0ee8d5a4e0a5f573e7d0aec16baa4"
+source = "git+https://github.com/xudong963/arrow-rs.git?branch=skip-filter-fully-matched-row-groups#76c028492c13c9319ca189416a84b0dd132e26ca"
dependencies = [
"bytes",
"half",
@@ -296,8 +291,7 @@ dependencies = [
[[package]]
name = "arrow-cast"
version = "58.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0127816c96533d20fc938729f48c52d3e48f99717e7a0b5ade77d742510736d"
+source = "git+https://github.com/xudong963/arrow-rs.git?branch=skip-filter-fully-matched-row-groups#76c028492c13c9319ca189416a84b0dd132e26ca"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -318,8 +312,7 @@ dependencies = [
[[package]]
name = "arrow-csv"
version = "58.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca025bd0f38eeecb57c2153c0123b960494138e6a957bbda10da2b25415209fe"
+source = "git+https://github.com/xudong963/arrow-rs.git?branch=skip-filter-fully-matched-row-groups#76c028492c13c9319ca189416a84b0dd132e26ca"
dependencies = [
"arrow-array",
"arrow-cast",
@@ -333,8 +326,7 @@ dependencies = [
[[package]]
name = "arrow-data"
version = "58.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42d10beeab2b1c3bb0b53a00f7c944a178b622173a5c7bcabc3cb45d90238df4"
+source = "git+https://github.com/xudong963/arrow-rs.git?branch=skip-filter-fully-matched-row-groups#76c028492c13c9319ca189416a84b0dd132e26ca"
dependencies = [
"arrow-buffer",
"arrow-schema",
@@ -346,8 +338,7 @@ dependencies = [
[[package]]
name = "arrow-flight"
version = "58.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "302b2e036335f3f04d65dad3f74ff1f2aae6dc671d6aa04dc6b61193761e16fb"
+source = "git+https://github.com/xudong963/arrow-rs.git?branch=skip-filter-fully-matched-row-groups#76c028492c13c9319ca189416a84b0dd132e26ca"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -374,8 +365,7 @@ dependencies = [
[[package]]
name = "arrow-ipc"
version = "58.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "609a441080e338147a84e8e6904b6da482cefb957c5cdc0f3398872f69a315d0"
+source = "git+https://github.com/xudong963/arrow-rs.git?branch=skip-filter-fully-matched-row-groups#76c028492c13c9319ca189416a84b0dd132e26ca"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -390,8 +380,7 @@ dependencies = [
[[package]]
name = "arrow-json"
version = "58.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ead0914e4861a531be48fe05858265cf854a4880b9ed12618b1d08cba9bebc8"
+source = "git+https://github.com/xudong963/arrow-rs.git?branch=skip-filter-fully-matched-row-groups#76c028492c13c9319ca189416a84b0dd132e26ca"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -414,8 +403,7 @@ dependencies = [
[[package]]
name = "arrow-ord"
version = "58.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "763a7ba279b20b52dad300e68cfc37c17efa65e68623169076855b3a9e941ca5"
+source = "git+https://github.com/xudong963/arrow-rs.git?branch=skip-filter-fully-matched-row-groups#76c028492c13c9319ca189416a84b0dd132e26ca"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -427,8 +415,7 @@ dependencies = [
[[package]]
name = "arrow-row"
version = "58.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e14fe367802f16d7668163ff647830258e6e0aeea9a4d79aaedf273af3bdcd3e"
+source = "git+https://github.com/xudong963/arrow-rs.git?branch=skip-filter-fully-matched-row-groups#76c028492c13c9319ca189416a84b0dd132e26ca"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -440,8 +427,7 @@ dependencies = [
[[package]]
name = "arrow-schema"
version = "58.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c30a1365d7a7dc50cc847e54154e6af49e4c4b0fddc9f607b687f29212082743"
+source = "git+https://github.com/xudong963/arrow-rs.git?branch=skip-filter-fully-matched-row-groups#76c028492c13c9319ca189416a84b0dd132e26ca"
dependencies = [
"bitflags",
"serde",
@@ -452,8 +438,7 @@ dependencies = [
[[package]]
name = "arrow-select"
version = "58.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78694888660a9e8ac949853db393af2a8b8fc82c19ce333132dfa2e72cc1a7fe"
+source = "git+https://github.com/xudong963/arrow-rs.git?branch=skip-filter-fully-matched-row-groups#76c028492c13c9319ca189416a84b0dd132e26ca"
dependencies = [
"ahash",
"arrow-array",
@@ -466,8 +451,7 @@ dependencies = [
[[package]]
name = "arrow-string"
version = "58.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61e04a01f8bb73ce54437514c5fd3ee2aa3e8abe4c777ee5cc55853b1652f79e"
+source = "git+https://github.com/xudong963/arrow-rs.git?branch=skip-filter-fully-matched-row-groups#76c028492c13c9319ca189416a84b0dd132e26ca"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -596,9 +580,9 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
[[package]]
name = "aws-config"
-version = "1.8.16"
+version = "1.8.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50f156acdd2cf55f5aa53ee416c4ac851cf1222694506c0b1f78c85695e9ca9d"
+checksum = "11493b0bad143270fb8ad284a096dd529ba91924c5409adeac856cc1bf047dbc"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -638,9 +622,9 @@ dependencies = [
[[package]]
name = "aws-lc-rs"
-version = "1.16.3"
+version = "1.16.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f"
+checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc"
dependencies = [
"aws-lc-sys",
"zeroize",
@@ -648,9 +632,9 @@ dependencies = [
[[package]]
name = "aws-lc-sys"
-version = "0.40.0"
+version = "0.39.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7"
+checksum = "83a25cf98105baa966497416dbd42565ce3a8cf8dbfd59803ec9ad46f3126399"
dependencies = [
"cc",
"cmake",
@@ -660,9 +644,9 @@ dependencies = [
[[package]]
name = "aws-runtime"
-version = "1.7.3"
+version = "1.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5dcd93c82209ac7413532388067dce79be5a8780c1786e5fae3df22e4dee2864"
+checksum = "5fc0651c57e384202e47153c1260b84a9936e19803d747615edf199dc3b98d17"
dependencies = [
"aws-credential-types",
"aws-sigv4",
@@ -685,9 +669,9 @@ dependencies = [
[[package]]
name = "aws-sdk-sso"
-version = "1.98.0"
+version = "1.97.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d69c77aafa20460c68b6b3213c84f6423b6e76dbf89accd3e1789a686ffd9489"
+checksum = "9aadc669e184501caaa6beafb28c6267fc1baef0810fb58f9b205485ca3f2567"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -709,9 +693,9 @@ dependencies = [
[[package]]
name = "aws-sdk-ssooidc"
-version = "1.100.0"
+version = "1.99.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1c7e7b09346d5ca22a2a08267555843a6a0127fb20d8964cb6ecfb8fdb190225"
+checksum = "1342a7db8f358d3de0aed2007a0b54e875458e39848d54cc1d46700b2bfcb0a8"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -733,9 +717,9 @@ dependencies = [
[[package]]
name = "aws-sdk-sts"
-version = "1.103.0"
+version = "1.101.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c2249b81a2e73a8027c41c378463a81ec39b8510f184f2caab87de912af0f49b"
+checksum = "ab41ad64e4051ecabeea802d6a17845a91e83287e1dd249e6963ea1ba78c428a"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -758,9 +742,9 @@ dependencies = [
[[package]]
name = "aws-sigv4"
-version = "1.4.3"
+version = "1.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68dc0b907359b120170613b5c09ccc61304eac3998ff6274b97d93ee6490115a"
+checksum = "b0b660013a6683ab23797778e21f1f854744fdf05f68204b4cca4c8c04b5d1f4"
dependencies = [
"aws-credential-types",
"aws-smithy-http",
@@ -769,11 +753,11 @@ dependencies = [
"bytes",
"form_urlencoded",
"hex",
- "hmac",
+ "hmac 0.12.1",
"http 0.2.12",
"http 1.4.0",
"percent-encoding",
- "sha2 0.11.0",
+ "sha2 0.10.9",
"time",
"tracing",
]
@@ -864,9 +848,9 @@ dependencies = [
[[package]]
name = "aws-smithy-runtime"
-version = "1.11.1"
+version = "1.10.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0504b1ab12debb5959e5165ee5fe97dd387e7aa7ea6a477bfd7635dfe769a4f5"
+checksum = "028999056d2d2fd58a697232f9eec4a643cf73a71cf327690a7edad1d2af2110"
dependencies = [
"aws-smithy-async",
"aws-smithy-http",
@@ -889,12 +873,11 @@ dependencies = [
[[package]]
name = "aws-smithy-runtime-api"
-version = "1.12.0"
+version = "1.11.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b71a13df6ada0aafbf21a73bdfcdf9324cfa9df77d96b8446045be3cde61b42e"
+checksum = "876ab3c9c29791ba4ba02b780a3049e21ec63dabda09268b175272c3733a79e6"
dependencies = [
"aws-smithy-async",
- "aws-smithy-runtime-api-macros",
"aws-smithy-types",
"bytes",
"http 0.2.12",
@@ -905,17 +888,6 @@ dependencies = [
"zeroize",
]
-[[package]]
-name = "aws-smithy-runtime-api-macros"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d7396fd9500589e62e460e987ecb671bad374934e55ec3b5f498cc7a8a8a7b7"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.117",
-]
-
[[package]]
name = "aws-smithy-types"
version = "1.4.7"
@@ -950,9 +922,9 @@ dependencies = [
[[package]]
name = "aws-types"
-version = "1.3.15"
+version = "1.3.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f4bbcaa9304ea40902d3d5f42a0428d1bd895a2b0f6999436fb279ffddc58ac"
+checksum = "47c8323699dd9b3c8d5b3c13051ae9cdef58fd179957c882f8374dd8725962d9"
dependencies = [
"aws-credential-types",
"aws-smithy-async",
@@ -1323,9 +1295,9 @@ dependencies = [
[[package]]
name = "clap"
-version = "4.6.1"
+version = "4.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51"
+checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351"
dependencies = [
"clap_builder",
"clap_derive",
@@ -1345,9 +1317,9 @@ dependencies = [
[[package]]
name = "clap_derive"
-version = "4.6.1"
+version = "4.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9"
+checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a"
dependencies = [
"heck",
"proc-macro2",
@@ -1757,7 +1729,7 @@ dependencies = [
[[package]]
name = "datafusion"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"arrow",
"arrow-schema",
@@ -1829,7 +1801,7 @@ dependencies = [
[[package]]
name = "datafusion-benchmarks"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"arrow",
"async-trait",
@@ -1856,7 +1828,7 @@ dependencies = [
[[package]]
name = "datafusion-catalog"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"arrow",
"async-trait",
@@ -1879,11 +1851,10 @@ dependencies = [
[[package]]
name = "datafusion-catalog-listing"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"arrow",
"async-trait",
- "chrono",
"datafusion-catalog",
"datafusion-common",
"datafusion-datasource",
@@ -1902,7 +1873,7 @@ dependencies = [
[[package]]
name = "datafusion-cli"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"arrow",
"async-trait",
@@ -1926,7 +1897,6 @@ dependencies = [
"regex",
"rstest",
"rustyline",
- "serde_json",
"testcontainers-modules",
"tokio",
"url",
@@ -1934,7 +1904,7 @@ dependencies = [
[[package]]
name = "datafusion-common"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"arrow",
"arrow-ipc",
@@ -1962,7 +1932,7 @@ dependencies = [
[[package]]
name = "datafusion-common-runtime"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"futures",
"log",
@@ -1971,7 +1941,7 @@ dependencies = [
[[package]]
name = "datafusion-datasource"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"arrow",
"async-compression",
@@ -1997,7 +1967,6 @@ dependencies = [
"liblzma",
"log",
"object_store",
- "parking_lot",
"rand 0.9.4",
"tempfile",
"tokio",
@@ -2008,7 +1977,7 @@ dependencies = [
[[package]]
name = "datafusion-datasource-arrow"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"arrow",
"arrow-ipc",
@@ -2031,7 +2000,7 @@ dependencies = [
[[package]]
name = "datafusion-datasource-avro"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"arrow",
"arrow-avro",
@@ -2048,7 +2017,7 @@ dependencies = [
[[package]]
name = "datafusion-datasource-csv"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"arrow",
"async-trait",
@@ -2069,7 +2038,7 @@ dependencies = [
[[package]]
name = "datafusion-datasource-json"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"arrow",
"async-trait",
@@ -2091,7 +2060,7 @@ dependencies = [
[[package]]
name = "datafusion-datasource-parquet"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"arrow",
"async-trait",
@@ -2124,11 +2093,11 @@ dependencies = [
[[package]]
name = "datafusion-doc"
-version = "53.1.0"
+version = "53.0.0"
[[package]]
name = "datafusion-examples"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"arrow",
"arrow-flight",
@@ -2169,7 +2138,7 @@ dependencies = [
[[package]]
name = "datafusion-execution"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"arrow",
"arrow-buffer",
@@ -2192,7 +2161,7 @@ dependencies = [
[[package]]
name = "datafusion-expr"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"arrow",
"arrow-schema",
@@ -2216,7 +2185,7 @@ dependencies = [
[[package]]
name = "datafusion-expr-common"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"arrow",
"datafusion-common",
@@ -2227,7 +2196,7 @@ dependencies = [
[[package]]
name = "datafusion-ffi"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"abi_stable",
"arrow",
@@ -2262,7 +2231,7 @@ dependencies = [
[[package]]
name = "datafusion-functions"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"arrow",
"arrow-buffer",
@@ -2295,7 +2264,7 @@ dependencies = [
[[package]]
name = "datafusion-functions-aggregate"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"arrow",
"criterion",
@@ -2316,7 +2285,7 @@ dependencies = [
[[package]]
name = "datafusion-functions-aggregate-common"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"arrow",
"criterion",
@@ -2328,7 +2297,7 @@ dependencies = [
[[package]]
name = "datafusion-functions-nested"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"arrow",
"arrow-ord",
@@ -2353,7 +2322,7 @@ dependencies = [
[[package]]
name = "datafusion-functions-table"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"arrow",
"async-trait",
@@ -2366,7 +2335,7 @@ dependencies = [
[[package]]
name = "datafusion-functions-window"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"arrow",
"criterion",
@@ -2382,7 +2351,7 @@ dependencies = [
[[package]]
name = "datafusion-functions-window-common"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"datafusion-common",
"datafusion-physical-expr-common",
@@ -2390,7 +2359,7 @@ dependencies = [
[[package]]
name = "datafusion-macros"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"datafusion-doc",
"quote",
@@ -2399,7 +2368,7 @@ dependencies = [
[[package]]
name = "datafusion-optimizer"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"arrow",
"async-trait",
@@ -2426,7 +2395,7 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"arrow",
"criterion",
@@ -2451,7 +2420,7 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr-adapter"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"arrow",
"datafusion-common",
@@ -2464,7 +2433,7 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr-common"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"arrow",
"chrono",
@@ -2480,7 +2449,7 @@ dependencies = [
[[package]]
name = "datafusion-physical-optimizer"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"arrow",
"datafusion-common",
@@ -2501,10 +2470,9 @@ dependencies = [
[[package]]
name = "datafusion-physical-plan"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"arrow",
- "arrow-data",
"arrow-ord",
"arrow-schema",
"async-trait",
@@ -2538,7 +2506,7 @@ dependencies = [
[[package]]
name = "datafusion-proto"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"arrow",
"async-trait",
@@ -2576,7 +2544,7 @@ dependencies = [
[[package]]
name = "datafusion-proto-common"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"arrow",
"datafusion-common",
@@ -2588,7 +2556,7 @@ dependencies = [
[[package]]
name = "datafusion-pruning"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"arrow",
"datafusion-common",
@@ -2606,7 +2574,7 @@ dependencies = [
[[package]]
name = "datafusion-session"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"async-trait",
"datafusion-common",
@@ -2618,7 +2586,7 @@ dependencies = [
[[package]]
name = "datafusion-spark"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"arrow",
"bigdecimal",
@@ -2645,7 +2613,7 @@ dependencies = [
[[package]]
name = "datafusion-sql"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"arrow",
"bigdecimal",
@@ -2670,7 +2638,7 @@ dependencies = [
[[package]]
name = "datafusion-sqllogictest"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"arrow",
"async-trait",
@@ -2690,7 +2658,6 @@ dependencies = [
"object_store",
"postgres-types",
"regex",
- "serde_json",
"sqllogictest",
"sqlparser",
"tempfile",
@@ -2702,7 +2669,7 @@ dependencies = [
[[package]]
name = "datafusion-substrait"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"async-recursion",
"async-trait",
@@ -2723,7 +2690,7 @@ dependencies = [
[[package]]
name = "datafusion-wasmtest"
-version = "53.1.0"
+version = "53.0.0"
dependencies = [
"bytes",
"chrono",
@@ -2984,12 +2951,12 @@ checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6"
[[package]]
name = "ferroid"
-version = "2.0.0"
+version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee93edf3c501f0035bbeffeccfed0b79e14c311f12195ec0e661e114a0f60da4"
+checksum = "bb330bbd4cb7a5b9f559427f06f98a4f853a137c8298f3bd3f8ca57663e21986"
dependencies = [
"portable-atomic",
- "rand 0.10.1",
+ "rand 0.9.4",
"web-time",
]
@@ -3386,6 +3353,15 @@ version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
+[[package]]
+name = "hmac"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"
+dependencies = [
+ "digest 0.10.7",
+]
+
[[package]]
name = "hmac"
version = "0.13.0"
@@ -3961,9 +3937,9 @@ dependencies = [
[[package]]
name = "libbz2-rs-sys"
-version = "0.2.3"
+version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b3a6a8c165077efc8f3a971534c50ea6a1a18b329ef4a66e897a7e3a1494565f"
+checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7"
[[package]]
name = "libc"
@@ -4009,12 +3985,13 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
[[package]]
name = "libmimalloc-sys"
-version = "0.1.46"
+version = "0.1.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc89deee4af0429081d2a518c0431ae068222a5a262a3bc6ff4d8535ec2e02fe"
+checksum = "667f4fec20f29dfc6bc7357c582d91796c169ad7e2fce709468aefeb2c099870"
dependencies = [
"cc",
"cty",
+ "libc",
]
[[package]]
@@ -4123,9 +4100,9 @@ checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
[[package]]
name = "mimalloc"
-version = "0.1.49"
+version = "0.1.48"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aca3c01a711f395b4257b81674c0e90e8dd1f1e62c4b7db45f684cc7a4fcb18a"
+checksum = "e1ee66a4b64c74f4ef288bcbb9192ad9c3feaad75193129ac8509af543894fd8"
dependencies = [
"libmimalloc-sys",
]
@@ -4464,8 +4441,7 @@ dependencies = [
[[package]]
name = "parquet"
version = "58.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d3f9f2205199603564127932b89695f52b62322f541d0fc7179d57c2e1c9877"
+source = "git+https://github.com/xudong963/arrow-rs.git?branch=skip-filter-fully-matched-row-groups#76c028492c13c9319ca189416a84b0dd132e26ca"
dependencies = [
"ahash",
"arrow-array",
@@ -4481,7 +4457,7 @@ dependencies = [
"flate2",
"futures",
"half",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.0",
"lz4_flex",
"num-bigint",
"num-integer",
@@ -4723,9 +4699,9 @@ checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49"
[[package]]
name = "portable-atomic-util"
-version = "0.2.7"
+version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618"
+checksum = "091397be61a01d4be58e7841595bd4bfedb15f1cd54977d79b8271e94ed799a3"
dependencies = [
"portable-atomic",
]
@@ -4752,7 +4728,7 @@ dependencies = [
"byteorder",
"bytes",
"fallible-iterator",
- "hmac",
+ "hmac 0.13.0",
"md-5 0.11.0",
"memchr",
"rand 0.10.1",
@@ -5003,9 +4979,9 @@ dependencies = [
[[package]]
name = "rand"
-version = "0.8.6"
+version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a"
+checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
dependencies = [
"libc",
"rand_chacha 0.3.1",
@@ -5328,7 +5304,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3a8fb4672e840a587a66fc577a5491375df51ddb88f2a2c2a792598c326fe14"
dependencies = [
"quote",
- "rand 0.8.6",
+ "rand 0.8.5",
"syn 2.0.117",
]
@@ -5400,9 +5376,9 @@ dependencies = [
[[package]]
name = "rustls-webpki"
-version = "0.103.13"
+version = "0.103.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"
+checksum = "8279bb85272c9f10811ae6a6c547ff594d6a7f3c6c6b02ee9726d1d0dcfcdd06"
dependencies = [
"aws-lc-rs",
"ring",
@@ -5843,7 +5819,7 @@ dependencies = [
"libtest-mimic",
"md-5 0.10.6",
"owo-colors",
- "rand 0.8.6",
+ "rand 0.8.5",
"regex",
"similar",
"subst",
@@ -6076,9 +6052,9 @@ dependencies = [
[[package]]
name = "testcontainers"
-version = "0.27.3"
+version = "0.27.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfd5785b5483672915ed5fe3cddf9f546802779fc1eceff0a6fb7321fac81c1e"
+checksum = "0bd36b06a2a6c0c3c81a83be1ab05fe86460d054d4d51bf513bc56b3e15bdc22"
dependencies = [
"astral-tokio-tar",
"async-trait",
@@ -6231,9 +6207,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokio"
-version = "1.52.1"
+version = "1.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b67dee974fe86fd92cc45b7a95fdd2f99a36a6d7b0d431a231178d3d670bbcc6"
+checksum = "a91135f59b1cbf38c91e73cf3386fca9bb77915c45ce2771460c9d92f0f3d776"
dependencies = [
"bytes",
"libc",
@@ -6529,9 +6505,9 @@ checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a"
[[package]]
name = "typenum"
-version = "1.20.0"
+version = "1.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de"
+checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb"
[[package]]
name = "typewit"
@@ -6721,9 +6697,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
[[package]]
name = "uuid"
-version = "1.23.1"
+version = "1.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76"
+checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9"
dependencies = [
"getrandom 0.4.2",
"js-sys",
@@ -6784,11 +6760,11 @@ dependencies = [
[[package]]
name = "wasip2"
-version = "1.0.3+wasi-0.2.9"
+version = "1.0.2+wasi-0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6"
+checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5"
dependencies = [
- "wit-bindgen 0.57.1",
+ "wit-bindgen",
]
[[package]]
@@ -6797,7 +6773,7 @@ version = "0.4.0+wasi-0.3.0-rc-2026-01-06"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5"
dependencies = [
- "wit-bindgen 0.51.0",
+ "wit-bindgen",
]
[[package]]
@@ -7307,12 +7283,6 @@ dependencies = [
"wit-bindgen-rust-macro",
]
-[[package]]
-name = "wit-bindgen"
-version = "0.57.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
-
[[package]]
name = "wit-bindgen-core"
version = "0.51.0"
diff --git a/Cargo.toml b/Cargo.toml
index 82081c1e42..7e6e9f82d6 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -288,3 +288,22 @@ debug = false
debug-assertions = false
strip = "debuginfo"
incremental = false
+
+[patch.crates-io]
+parquet = { git = "https://github.com/xudong963/arrow-rs.git", branch = "skip-filter-fully-matched-row-groups" }
+arrow = { git = "https://github.com/xudong963/arrow-rs.git", branch = "skip-filter-fully-matched-row-groups" }
+arrow-array = { git = "https://github.com/xudong963/arrow-rs.git", branch = "skip-filter-fully-matched-row-groups" }
+arrow-buffer = { git = "https://github.com/xudong963/arrow-rs.git", branch = "skip-filter-fully-matched-row-groups" }
+arrow-cast = { git = "https://github.com/xudong963/arrow-rs.git", branch = "skip-filter-fully-matched-row-groups" }
+arrow-ipc = { git = "https://github.com/xudong963/arrow-rs.git", branch = "skip-filter-fully-matched-row-groups" }
+arrow-ord = { git = "https://github.com/xudong963/arrow-rs.git", branch = "skip-filter-fully-matched-row-groups" }
+arrow-schema = { git = "https://github.com/xudong963/arrow-rs.git", branch = "skip-filter-fully-matched-row-groups" }
+arrow-string = { git = "https://github.com/xudong963/arrow-rs.git", branch = "skip-filter-fully-matched-row-groups" }
+arrow-select = { git = "https://github.com/xudong963/arrow-rs.git", branch = "skip-filter-fully-matched-row-groups" }
+arrow-data = { git = "https://github.com/xudong963/arrow-rs.git", branch = "skip-filter-fully-matched-row-groups" }
+arrow-arith = { git = "https://github.com/xudong963/arrow-rs.git", branch = "skip-filter-fully-matched-row-groups" }
+arrow-csv = { git = "https://github.com/xudong963/arrow-rs.git", branch = "skip-filter-fully-matched-row-groups" }
+arrow-json = { git = "https://github.com/xudong963/arrow-rs.git", branch = "skip-filter-fully-matched-row-groups" }
+arrow-avro = { git = "https://github.com/xudong963/arrow-rs.git", branch = "skip-filter-fully-matched-row-groups" }
+arrow-flight = { git = "https://github.com/xudong963/arrow-rs.git", branch = "skip-filter-fully-matched-row-groups" }
+arrow-row = { git = "https://github.com/xudong963/arrow-rs.git", branch = "skip-filter-fully-matched-row-groups" }
diff --git a/datafusion/datasource-parquet/Cargo.toml b/datafusion/datasource-parquet/Cargo.toml
index a5855af17a..c7befa8eb2 100644
--- a/datafusion/datasource-parquet/Cargo.toml
+++ b/datafusion/datasource-parquet/Cargo.toml
@@ -56,6 +56,7 @@ parquet = { workspace = true }
tokio = { workspace = true }
[dev-dependencies]
+bytes = { workspace = true }
chrono = { workspace = true }
criterion = { workspace = true }
datafusion-functions = { workspace = true }
@@ -86,3 +87,7 @@ harness = false
[[bench]]
name = "parquet_struct_filter_pushdown"
harness = false
+
+[[bench]]
+name = "parquet_fully_matched_filter"
+harness = false
diff --git a/datafusion/datasource-parquet/benches/parquet_fully_matched_filter.rs b/datafusion/datasource-parquet/benches/parquet_fully_matched_filter.rs
new file mode 100644
index 0000000000..2d2e45b7ca
--- /dev/null
+++ b/datafusion/datasource-parquet/benches/parquet_fully_matched_filter.rs
@@ -0,0 +1,275 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Benchmark for skipping filter evaluation on fully matched row groups.
+//!
+//! This benchmark measures the performance improvement from skipping
+//! RowFilter evaluation when row group statistics prove that all rows
+//! in a row group satisfy the predicate.
+//!
+//! Dataset layout:
+//! - 20 row groups, each with 50_000 rows
+//! - Column `x`: i64, values in range [0, 100) for all row groups
+//! - Column `payload`: Utf8, 1 KB string (makes filter column decoding cost visible)
+//!
+//! Predicate: `x < 200`
+//! - ALL row groups are fully matched (max(x) < 200 for every row group)
+//! - Without the optimization: RowFilter decodes `x` and evaluates predicate for every row
+//! - With the optimization: RowFilter is skipped entirely (statistics prove all rows match)
+//!
+//! Uses `ParquetPushDecoder` directly to exercise the exact code path
+//! that DataFusion's async opener uses.
+
+use std::path::PathBuf;
+use std::sync::{Arc, LazyLock};
+
+use arrow::array::{Int64Array, RecordBatch, StringBuilder};
+use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
+use bytes::Bytes;
+use criterion::{Criterion, Throughput, criterion_group, criterion_main};
+use datafusion_common::ScalarValue;
+use datafusion_datasource_parquet::{ParquetFileMetrics, build_row_filter};
+use datafusion_expr::{Expr, col};
+use datafusion_physical_expr::planner::logical2physical;
+use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;
+use parquet::DecodeResult;
+use parquet::arrow::arrow_reader::ArrowReaderMetadata;
+use parquet::arrow::push_decoder::ParquetPushDecoderBuilder;
+use parquet::file::properties::WriterProperties;
+use parquet::{arrow::ArrowWriter, file::metadata::ParquetMetaData};
+use tempfile::TempDir;
+
+const ROW_GROUP_SIZE: usize = 50_000;
+const NUM_ROW_GROUPS: usize = 20;
+const TOTAL_ROWS: usize = ROW_GROUP_SIZE * NUM_ROW_GROUPS;
+const PAYLOAD_LEN: usize = 1024;
+
+struct BenchmarkDataset {
+ _tempdir: TempDir,
+ file_path: PathBuf,
+ file_bytes: Bytes,
+ metadata: Arc<ParquetMetaData>,
+ schema: SchemaRef,
+}
+
+static DATASET: LazyLock<BenchmarkDataset> = LazyLock::new(|| {
+ create_dataset().expect("failed to prepare parquet benchmark dataset")
+});
+
+fn bench_fully_matched_filter(c: &mut Criterion) {
+ let dataset = &*DATASET;
+ let mut group = c.benchmark_group("parquet_fully_matched_filter");
+ group.throughput(Throughput::Elements(TOTAL_ROWS as u64));
+
+ let metrics = ExecutionPlanMetricsSet::new();
+ let file_metrics =
+ ParquetFileMetrics::new(0, &dataset.file_path.display().to_string(), &metrics);
+
+ // Build the row filter (shared across benchmarks)
+ let predicate_expr = logical2physical(&x_lt(200), &dataset.schema);
+ let row_filter = build_row_filter(
+ &predicate_expr,
+ &dataset.schema,
+ &dataset.metadata,
+ false,
+ &file_metrics,
+ )
+ .expect("build row filter")
+ .expect("row filter should be Some");
+
+ // All row group indices
+ let all_rg_indices: Vec<usize> = (0..NUM_ROW_GROUPS).collect();
+
+ // Scenario 1: with pushdown, WITHOUT fully_matched optimization
+ // (RowFilter evaluated on every row group)
+ group.bench_function("all_fully_matched/pushdown_no_skip", |b| {
+ b.iter(|| {
+ // Rebuild row filter each iteration (it's consumed by the builder)
+ let rf = rebuild_row_filter(dataset, &file_metrics);
+ let reader_metadata = ArrowReaderMetadata::try_new(
+ Arc::clone(&dataset.metadata),
+ Default::default(),
+ )
+ .unwrap();
+ let decoder = ParquetPushDecoderBuilder::new_with_metadata(reader_metadata)
+ .with_batch_size(8192)
+ .with_row_filter(rf)
+ .with_row_groups(all_rg_indices.clone())
+ // No fully_matched_row_groups: filter runs on all row groups
+ .build()
+ .unwrap();
+ let rows = run_push_decoder(decoder, &dataset.file_bytes);
+ assert_eq!(rows, TOTAL_ROWS);
+ });
+ });
+
+ // Scenario 2: with pushdown, WITH fully_matched optimization
+ // (RowFilter skipped for all row groups)
+ group.bench_function("all_fully_matched/pushdown_with_skip", |b| {
+ b.iter(|| {
+ let rf = rebuild_row_filter(dataset, &file_metrics);
+ let reader_metadata = ArrowReaderMetadata::try_new(
+ Arc::clone(&dataset.metadata),
+ Default::default(),
+ )
+ .unwrap();
+ let decoder = ParquetPushDecoderBuilder::new_with_metadata(reader_metadata)
+ .with_batch_size(8192)
+ .with_row_filter(rf)
+ .with_row_groups(all_rg_indices.clone())
+ .with_fully_matched_row_groups(all_rg_indices.clone())
+ .build()
+ .unwrap();
+ let rows = run_push_decoder(decoder, &dataset.file_bytes);
+ assert_eq!(rows, TOTAL_ROWS);
+ });
+ });
+
+ // Scenario 3: no pushdown at all (no RowFilter set)
+ group.bench_function("all_fully_matched/no_pushdown", |b| {
+ b.iter(|| {
+ let reader_metadata = ArrowReaderMetadata::try_new(
+ Arc::clone(&dataset.metadata),
+ Default::default(),
+ )
+ .unwrap();
+ let decoder = ParquetPushDecoderBuilder::new_with_metadata(reader_metadata)
+ .with_batch_size(8192)
+ .with_row_groups(all_rg_indices.clone())
+ .build()
+ .unwrap();
+ let rows = run_push_decoder(decoder, &dataset.file_bytes);
+ assert_eq!(rows, TOTAL_ROWS);
+ });
+ });
+
+ drop(row_filter);
+ group.finish();
+}
+
+fn rebuild_row_filter(
+ dataset: &BenchmarkDataset,
+ file_metrics: &ParquetFileMetrics,
+) -> parquet::arrow::arrow_reader::RowFilter {
+ let predicate_expr = logical2physical(&x_lt(200), &dataset.schema);
+ build_row_filter(
+ &predicate_expr,
+ &dataset.schema,
+ &dataset.metadata,
+ false,
+ file_metrics,
+ )
+ .expect("build row filter")
+ .expect("row filter should be Some")
+}
+
+fn run_push_decoder(
+ mut decoder: parquet::arrow::push_decoder::ParquetPushDecoder,
+ file_bytes: &Bytes,
+) -> usize {
+ let mut total_rows = 0;
+ loop {
+ match decoder.try_decode().unwrap() {
+ DecodeResult::NeedsData(ranges) => {
+ let data: Vec<Bytes> = ranges
+ .iter()
+ .map(|r| {
+ let start = r.start as usize;
+ let end = r.end as usize;
+ file_bytes.slice(start..end)
+ })
+ .collect();
+ decoder.push_ranges(ranges, data).unwrap();
+ }
+ DecodeResult::Data(batch) => {
+ total_rows += batch.num_rows();
+ }
+ DecodeResult::Finished => break,
+ }
+ }
+ total_rows
+}
+
+/// `x < value`
+fn x_lt(value: i64) -> Expr {
+ col("x").lt(Expr::Literal(ScalarValue::Int64(Some(value)), None))
+}
+
+fn schema() -> SchemaRef {
+ Arc::new(Schema::new(vec![
+ Field::new("x", DataType::Int64, false),
+ Field::new("payload", DataType::Utf8, false),
+ ]))
+}
+
+fn create_dataset() -> datafusion_common::Result<BenchmarkDataset> {
+ let tempdir = TempDir::new()?;
+ let file_path = tempdir.path().join("fully_matched_filter.parquet");
+
+ let schema = schema();
+ let writer_props = WriterProperties::builder()
+ .set_max_row_group_row_count(Some(ROW_GROUP_SIZE))
+ .set_statistics_enabled(parquet::file::properties::EnabledStatistics::Chunk)
+ .build();
+
+ let file = std::fs::File::create(&file_path)?;
+ let mut writer = ArrowWriter::try_new(file, schema.clone(), Some(writer_props))?;
+
+ for _rg in 0..NUM_ROW_GROUPS {
+ let x_values: Vec<i64> = (0..ROW_GROUP_SIZE as i64).map(|i| i % 100).collect();
+ let x_array = Int64Array::from(x_values);
+
+ let mut payload_builder =
+ StringBuilder::with_capacity(ROW_GROUP_SIZE, ROW_GROUP_SIZE * PAYLOAD_LEN);
+ for _ in 0..ROW_GROUP_SIZE {
+ let s: String = (0..PAYLOAD_LEN)
+ .map(|i| (b'A' + (i % 26) as u8) as char)
+ .collect();
+ payload_builder.append_value(&s);
+ }
+ let payload_array = payload_builder.finish();
+
+ let batch = RecordBatch::try_new(
+ schema.clone(),
+ vec![Arc::new(x_array), Arc::new(payload_array)],
+ )?;
+ writer.write(&batch)?;
+ }
+
+ writer.close()?;
+
+ // Read the file into memory for push decoder benchmarks
+ let file_bytes = Bytes::from(std::fs::read(&file_path)?);
+
+ // Parse metadata
+ let file = std::fs::File::open(&file_path)?;
+ let builder =
+ parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder::try_new(file)?;
+ let metadata = builder.metadata().clone();
+ let arrow_schema = Arc::clone(builder.schema());
+
+ Ok(BenchmarkDataset {
+ _tempdir: tempdir,
+ file_path,
+ file_bytes,
+ metadata,
+ schema: arrow_schema,
+ })
+}
+
+criterion_group!(benches, bench_fully_matched_filter);
+criterion_main!(benches);
diff --git a/datafusion/datasource-parquet/src/opener.rs b/datafusion/datasource-parquet/src/opener.rs
index bad1c684b4..1dd1a8781c 100644
--- a/datafusion/datasource-parquet/src/opener.rs
+++ b/datafusion/datasource-parquet/src/opener.rs
@@ -1106,10 +1106,14 @@ impl RowGroupsPrunedParquetOpen {
row_groups.prune_by_limit(limit, rg_metadata, &prepared.file_metrics);
}
+ // Build the access plan and extract fully matched row group indices.
+ // Fully matched row groups have ALL rows satisfying the predicate,
+ // so we can skip both page pruning and row filter evaluation for them.
+ let (mut access_plan, fully_matched_row_groups) = row_groups.build();
+
// Page index pruning: if all data on individual pages can
// be ruled using page metadata, rows from other columns
// with that range can be skipped as well.
- let mut access_plan = row_groups.build();
if prepared.enable_page_index
&& !access_plan.is_empty()
&& let Some(page_pruning_predicate) = page_pruning_predicate
@@ -1120,6 +1124,7 @@ impl RowGroupsPrunedParquetOpen {
reader_metadata.parquet_schema(),
file_metadata.as_ref(),
&prepared.file_metrics,
+ &fully_matched_row_groups,
);
}
@@ -1148,6 +1153,10 @@ impl RowGroupsPrunedParquetOpen {
if let Some(row_filter) = row_filter {
decoder_builder = decoder_builder.with_row_filter(row_filter);
}
+ if !fully_matched_row_groups.is_empty() {
+ decoder_builder =
+ decoder_builder.with_fully_matched_row_groups(fully_matched_row_groups);
+ }
if prepared.force_filter_selections {
decoder_builder =
decoder_builder.with_row_selection_policy(RowSelectionPolicy::Selectors);
diff --git a/datafusion/datasource-parquet/src/page_filter.rs b/datafusion/datasource-parquet/src/page_filter.rs
index baef36ce14..3c5e4b5f50 100644
--- a/datafusion/datasource-parquet/src/page_filter.rs
+++ b/datafusion/datasource-parquet/src/page_filter.rs
@@ -160,6 +160,7 @@ impl PagePruningAccessPlanFilter {
parquet_schema: &SchemaDescriptor,
parquet_metadata: &ParquetMetaData,
file_metrics: &ParquetFileMetrics,
+ fully_matched_row_groups: &[usize],
) -> ParquetAccessPlan {
// scoped timer updates on drop
let _timer_guard = file_metrics.page_index_eval_time.timer();
@@ -197,6 +198,11 @@ impl PagePruningAccessPlanFilter {
// for each row group specified in the access plan
let row_group_indexes = access_plan.row_group_indexes();
for row_group_index in row_group_indexes {
+ // Skip page pruning for fully matched row groups: all rows are
+ // known to satisfy the predicate, so page-level pruning is wasted work.
+ if fully_matched_row_groups.contains(&row_group_index) {
+ continue;
+ }
// The selection for this particular row group
let mut overall_selection = None;
for predicate in page_index_predicates {
diff --git a/datafusion/datasource-parquet/src/row_group_filter.rs b/datafusion/datasource-parquet/src/row_group_filter.rs
index 3f254c9f55..77d54b7636 100644
--- a/datafusion/datasource-parquet/src/row_group_filter.rs
+++ b/datafusion/datasource-parquet/src/row_group_filter.rs
@@ -74,9 +74,18 @@ impl RowGroupAccessPlanFilter {
self.access_plan.row_group_index_iter()
}
- /// Returns the inner access plan
- pub fn build(self) -> ParquetAccessPlan {
- self.access_plan
+ /// Returns the inner access plan and the indices of fully matched row groups.
+ ///
+ /// Fully matched row groups are those where ALL rows are known to satisfy
+ /// the predicate based on row group statistics. The returned indices are
+ /// a subset of the row groups that will be scanned.
+ pub fn build(self) -> (ParquetAccessPlan, Vec<usize>) {
+ let fully_matched: Vec<usize> = self
+ .access_plan
+ .row_group_index_iter()
+ .filter(|&idx| self.is_fully_matched[idx])
+ .collect();
+ (self.access_plan, fully_matched)
}
/// Returns the is_fully_matched vector
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org