You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by su...@apache.org on 2026/10/06 18:46:51 UTC
(datafusion-comet) 01/01: perf: backport bitmap pruning for runtime join filters
This is an automated email from the ASF dual-hosted git repository.
sunchao pushed a commit to branch codex/comet-df-bitmap
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
commit 3cc9e67e8fd4e54e47d29ad272aaa7691b01ef17
Author: Chao Sun <ch...@openai.com>
AuthorDate: Tue Oct 6 18:46:34 2026 +0000
perf: backport bitmap pruning for runtime join filters
---
native/Cargo.lock | 305 +-
native/Cargo.toml | 8 +-
.../operators/dynamic_filter/join/tests.rs | 83 +
native/vendor/README.md | 91 +
native/vendor/bitmap-backport.patch | 1175 +++
native/vendor/datafusion-physical-plan/Cargo.toml | 308 +
.../datafusion-physical-plan/Cargo.toml.orig | 149 +
native/vendor/datafusion-physical-plan/LICENSE.txt | 212 +
native/vendor/datafusion-physical-plan/NOTICE.txt | 5 +
native/vendor/datafusion-physical-plan/README.md | 33 +
.../benches/aggregate_vectorized.rs | 309 +
.../benches/bounded_window.rs | 280 +
.../benches/compute_statistics.rs | 354 +
.../benches/dictionary_group_values.rs | 176 +
.../benches/hash_join_semi_anti.rs | 387 +
.../benches/multi_group_by.rs | 815 ++
.../benches/partial_ordering.rs | 60 +
.../benches/sort_merge_join.rs | 204 +
.../benches/sort_preserving_merge.rs | 197 +
.../datafusion-physical-plan/benches/spill_io.rs | 581 ++
.../src/aggregates/aggregate_hash_table/common.rs | 693 ++
.../aggregate_hash_table/common_ordered.rs | 410 +
.../aggregates/aggregate_hash_table/final_table.rs | 77 +
.../src/aggregates/aggregate_hash_table/mod.rs | 31 +
.../aggregate_hash_table/ordered_final_table.rs | 85 +
.../aggregate_hash_table/ordered_partial_table.rs | 106 +
.../aggregate_hash_table/partial_reduce_table.rs | 71 +
.../aggregate_hash_table/partial_table.rs | 219 +
.../aggregate_hash_table/single_table.rs | 76 +
.../src/aggregates/aggregate_stream.rs | 478 ++
.../src/aggregates/group_values/metrics.rs | 222 +
.../src/aggregates/group_values/mod.rs | 214 +
.../group_values/multi_group_by/boolean.rs | 493 ++
.../group_values/multi_group_by/bytes.rs | 701 ++
.../group_values/multi_group_by/bytes_view.rs | 1022 +++
.../multi_group_by/fixed_size_binary.rs | 515 ++
.../aggregates/group_values/multi_group_by/mod.rs | 2536 +++++++
.../group_values/multi_group_by/primitive.rs | 654 ++
.../group_values/multi_group_by/row_backed.rs | 1129 +++
.../src/aggregates/group_values/null_builder.rs | 100 +
.../src/aggregates/group_values/row.rs | 414 +
.../group_values/single_group_by/boolean.rs | 153 +
.../group_values/single_group_by/bytes.rs | 128 +
.../group_values/single_group_by/bytes_view.rs | 130 +
.../aggregates/group_values/single_group_by/mod.rs | 23 +
.../group_values/single_group_by/primitive.rs | 296 +
.../src/aggregates/grouped_hash_stream.rs | 1603 ++++
.../src/aggregates/grouped_topk_stream.rs | 306 +
.../src/aggregates/hash_stream.rs | 1838 +++++
.../datafusion-physical-plan/src/aggregates/mod.rs | 7982 ++++++++++++++++++++
.../src/aggregates/order/full.rs | 156 +
.../src/aggregates/order/mod.rs | 219 +
.../src/aggregates/order/partial.rs | 358 +
.../src/aggregates/ordered_final_stream.rs | 902 +++
.../src/aggregates/ordered_partial_stream.rs | 352 +
.../src/aggregates/partial_reduce_stream.rs | 385 +
.../src/aggregates/single_stream.rs | 833 ++
.../src/aggregates/skip_partial.rs | 305 +
.../src/aggregates/topk/hash_table.rs | 727 ++
.../src/aggregates/topk/heap.rs | 783 ++
.../src/aggregates/topk/mod.rs | 22 +
.../src/aggregates/topk/priority_map.rs | 805 ++
.../vendor/datafusion-physical-plan/src/analyze.rs | 566 ++
.../datafusion-physical-plan/src/async_func.rs | 550 ++
.../vendor/datafusion-physical-plan/src/buffer.rs | 753 ++
.../datafusion-physical-plan/src/coalesce/mod.rs | 375 +
.../src/coalesce_batches.rs | 459 ++
.../src/coalesce_partitions.rs | 657 ++
.../src/column_rewriter.rs | 382 +
.../vendor/datafusion-physical-plan/src/common.rs | 623 ++
native/vendor/datafusion-physical-plan/src/coop.rs | 517 ++
.../vendor/datafusion-physical-plan/src/display.rs | 1886 +++++
.../src/distribution_requirements.rs | 359 +
.../vendor/datafusion-physical-plan/src/empty.rs | 313 +
.../datafusion-physical-plan/src/execution_plan.rs | 3077 ++++++++
.../vendor/datafusion-physical-plan/src/explain.rs | 403 +
.../vendor/datafusion-physical-plan/src/filter.rs | 3907 ++++++++++
.../src/filter_pushdown.rs | 558 ++
.../src/joins/array_map.rs | 601 ++
.../datafusion-physical-plan/src/joins/chain.rs | 69 +
.../src/joins/cross_join.rs | 1081 +++
.../src/joins/hash_join/exec.rs | 7374 ++++++++++++++++++
.../src/joins/hash_join/inlist_builder.rs | 158 +
.../src/joins/hash_join/mod.rs | 27 +
.../src/joins/hash_join/partitioned_hash_eval.rs | 889 +++
.../src/joins/hash_join/shared_bounds.rs | 1522 ++++
.../src/joins/hash_join/stream.rs | 1144 +++
.../src/joins/join_filter.rs | 108 +
.../src/joins/join_hash_map.rs | 572 ++
.../src/joins/key_range_bitmap.rs | 238 +
.../datafusion-physical-plan/src/joins/mod.rs | 118 +
.../src/joins/nested_loop_join.rs | 4144 ++++++++++
.../src/joins/piecewise_merge_join/classic_join.rs | 1546 ++++
.../src/joins/piecewise_merge_join/exec.rs | 819 ++
.../src/joins/piecewise_merge_join/mod.rs | 24 +
.../src/joins/piecewise_merge_join/utils.rs | 61 +
.../datafusion-physical-plan/src/joins/proto.rs | 161 +
.../src/joins/sort_merge_join/bitwise_stream.rs | 1265 ++++
.../src/joins/sort_merge_join/exec.rs | 826 ++
.../src/joins/sort_merge_join/filter.rs | 388 +
.../joins/sort_merge_join/materializing_stream.rs | 2000 +++++
.../src/joins/sort_merge_join/metrics.rs | 82 +
.../src/joins/sort_merge_join/mod.rs | 29 +
.../src/joins/sort_merge_join/tests.rs | 5783 ++++++++++++++
.../src/joins/stream_join_utils.rs | 1185 +++
.../src/joins/symmetric_hash_join.rs | 3035 ++++++++
.../src/joins/test_utils.rs | 613 ++
.../datafusion-physical-plan/src/joins/utils.rs | 4968 ++++++++++++
native/vendor/datafusion-physical-plan/src/lib.rs | 115 +
.../vendor/datafusion-physical-plan/src/limit.rs | 1094 +++
.../vendor/datafusion-physical-plan/src/memory.rs | 993 +++
.../vendor/datafusion-physical-plan/src/metrics.rs | 21 +
.../src/operator_statistics/mod.rs | 2342 ++++++
.../datafusion-physical-plan/src/ordering.rs | 54 +
.../src/placeholder_row.rs | 331 +
.../datafusion-physical-plan/src/projection.rs | 2523 +++++++
.../vendor/datafusion-physical-plan/src/proto.rs | 386 +
.../src/recursive_query.rs | 581 ++
.../datafusion-physical-plan/src/render_tree.rs | 231 +
.../src/repartition/distributor_channels.rs | 855 +++
.../src/repartition/mod.rs | 4612 +++++++++++
.../src/scalar_subquery.rs | 673 ++
.../datafusion-physical-plan/src/sort_pushdown.rs | 120 +
.../datafusion-physical-plan/src/sorts/builder.rs | 359 +
.../datafusion-physical-plan/src/sorts/cursor.rs | 691 ++
.../datafusion-physical-plan/src/sorts/merge.rs | 729 ++
.../datafusion-physical-plan/src/sorts/mod.rs | 31 +
.../src/sorts/multi_level_merge.rs | 1100 +++
.../src/sorts/partial_sort.rs | 1347 ++++
.../src/sorts/partitioned_topk.rs | 530 ++
.../datafusion-physical-plan/src/sorts/sort.rs | 3627 +++++++++
.../src/sorts/sort_preserving_merge.rs | 1781 +++++
.../datafusion-physical-plan/src/sorts/stream.rs | 543 ++
.../src/sorts/streaming_merge.rs | 382 +
.../src/spill/in_progress_spill_file.rs | 212 +
.../datafusion-physical-plan/src/spill/mod.rs | 1527 ++++
.../src/spill/replayable_spill_input.rs | 447 ++
.../src/spill/spill_manager.rs | 409 +
.../src/spill/spill_pool.rs | 1648 ++++
.../datafusion-physical-plan/src/statistics.rs | 274 +
.../vendor/datafusion-physical-plan/src/stream.rs | 1163 +++
.../datafusion-physical-plan/src/streaming.rs | 486 ++
native/vendor/datafusion-physical-plan/src/test.rs | 570 ++
.../datafusion-physical-plan/src/test/exec.rs | 1087 +++
.../datafusion-physical-plan/src/topk/mod.rs | 3169 ++++++++
.../datafusion-physical-plan/src/tree_node.rs | 118 +
.../vendor/datafusion-physical-plan/src/union.rs | 1786 +++++
.../vendor/datafusion-physical-plan/src/unnest.rs | 2408 ++++++
.../vendor/datafusion-physical-plan/src/visitor.rs | 94 +
.../src/windows/bounded_window_agg_exec.rs | 3030 ++++++++
.../datafusion-physical-plan/src/windows/mod.rs | 1385 ++++
.../datafusion-physical-plan/src/windows/proto.rs | 263 +
.../datafusion-physical-plan/src/windows/utils.rs | 37 +
.../src/windows/window_agg_exec.rs | 678 ++
.../datafusion-physical-plan/src/work_table.rs | 375 +
native/vendor/datafusion-pruning/Cargo.toml | 126 +
native/vendor/datafusion-pruning/Cargo.toml.orig | 49 +
native/vendor/datafusion-pruning/LICENSE.txt | 212 +
native/vendor/datafusion-pruning/NOTICE.txt | 5 +
native/vendor/datafusion-pruning/README.md | 34 +
.../vendor/datafusion-pruning/src/file_pruner.rs | 171 +
.../src/key_range_bitmap_expr.rs | 103 +
native/vendor/datafusion-pruning/src/lib.rs | 28 +
.../datafusion-pruning/src/pruning_predicate.rs | 6424 ++++++++++++++++
native/vendor/rustfmt.toml | 24 +
165 files changed, 144881 insertions(+), 11 deletions(-)
diff --git a/native/Cargo.lock b/native/Cargo.lock
index c73688fe1d..81dd9ac1ee 100644
--- a/native/Cargo.lock
+++ b/native/Cargo.lock
@@ -99,6 +99,15 @@ dependencies = [
"alloc-no-stdlib",
]
+[[package]]
+name = "alloca"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4"
+dependencies = [
+ "cc",
+]
+
[[package]]
name = "allocator-api2"
version = "0.2.21"
@@ -243,6 +252,8 @@ dependencies = [
"arrow-schema",
"arrow-select",
"arrow-string",
+ "half",
+ "rand 0.9.5",
]
[[package]]
@@ -1241,6 +1252,16 @@ dependencies = [
"tinyvec",
]
+[[package]]
+name = "bstr"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f"
+dependencies = [
+ "memchr",
+ "serde_core",
+]
+
[[package]]
name = "bumpalo"
version = "3.20.3"
@@ -1543,6 +1564,17 @@ dependencies = [
"crossbeam-utils",
]
+[[package]]
+name = "console"
+version = "0.16.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e96a4956774c13c126a8b5af4daa79384f4d826534c95a02d76afb39e2ab64e3"
+dependencies = [
+ "encode_unicode",
+ "libc",
+ "windows-sys 0.61.2",
+]
+
[[package]]
name = "const-oid"
version = "0.9.6"
@@ -1663,7 +1695,7 @@ dependencies = [
"cast",
"ciborium",
"clap",
- "criterion-plot",
+ "criterion-plot 0.6.0",
"itertools 0.13.0",
"num-traits",
"oorandom",
@@ -1677,6 +1709,32 @@ dependencies = [
"walkdir",
]
+[[package]]
+name = "criterion"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3"
+dependencies = [
+ "alloca",
+ "anes",
+ "cast",
+ "ciborium",
+ "clap",
+ "criterion-plot 0.8.2",
+ "futures",
+ "itertools 0.13.0",
+ "num-traits",
+ "oorandom",
+ "page_size",
+ "plotters",
+ "rayon",
+ "regex",
+ "serde",
+ "serde_json",
+ "tinytemplate",
+ "walkdir",
+]
+
[[package]]
name = "criterion-plot"
version = "0.6.0"
@@ -1687,6 +1745,16 @@ dependencies = [
"itertools 0.13.0",
]
+[[package]]
+name = "criterion-plot"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea"
+dependencies = [
+ "cast",
+ "itertools 0.13.0",
+]
+
[[package]]
name = "crossbeam-channel"
version = "0.5.17"
@@ -1997,7 +2065,7 @@ dependencies = [
"comet-test-udfs",
"comet-udf-sdk",
"crc32fast",
- "criterion",
+ "criterion 0.7.0",
"datafusion",
"datafusion-comet-common",
"datafusion-comet-jni-bridge",
@@ -2085,7 +2153,7 @@ name = "datafusion-comet-operators"
version = "1.2.0"
dependencies = [
"arrow",
- "criterion",
+ "criterion 0.7.0",
"datafusion",
"datafusion-comet-common",
"datafusion-comet-spark-expr",
@@ -2114,7 +2182,7 @@ dependencies = [
"clap",
"crc32c",
"crc32fast",
- "criterion",
+ "criterion 0.7.0",
"datafusion",
"datafusion-comet-common",
"datafusion-comet-jni-bridge",
@@ -2139,7 +2207,7 @@ dependencies = [
"arrow",
"base64 0.23.1",
"chrono",
- "criterion",
+ "criterion 0.7.0",
"datafusion",
"datafusion-comet-common",
"datafusion-comet-jni-bridge",
@@ -2581,6 +2649,7 @@ dependencies = [
"datafusion-expr-common",
"datafusion-functions-aggregate-common",
"datafusion-physical-expr-common",
+ "datafusion-proto-models",
"half",
"hashbrown 0.17.1",
"indexmap 2.14.2",
@@ -2615,6 +2684,7 @@ dependencies = [
"chrono",
"datafusion-common",
"datafusion-expr-common",
+ "datafusion-proto-models",
"hashbrown 0.17.1",
"indexmap 2.14.2",
"itertools 0.15.0",
@@ -2644,8 +2714,6 @@ dependencies = [
[[package]]
name = "datafusion-physical-plan"
version = "55.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1265d58e5bce07d154e642a51ff43033b576a6ae40d50a29ac2b9f311004eb52"
dependencies = [
"arrow",
"arrow-data",
@@ -2654,42 +2722,76 @@ dependencies = [
"arrow-schema",
"async-trait",
"bytes",
+ "criterion 0.8.2",
"datafusion-common",
"datafusion-common-runtime",
"datafusion-execution",
"datafusion-expr",
"datafusion-functions",
+ "datafusion-functions-aggregate",
"datafusion-functions-aggregate-common",
+ "datafusion-functions-window",
"datafusion-functions-window-common",
"datafusion-physical-expr",
"datafusion-physical-expr-common",
+ "datafusion-proto-common",
+ "datafusion-proto-models",
"futures",
"half",
"hashbrown 0.17.1",
"indexmap 2.14.2",
+ "insta",
"itertools 0.15.0",
"log",
"num-traits",
"parking_lot",
"pin-project-lite",
+ "rand 0.9.5",
+ "rstest",
+ "rstest_reuse",
"serde_json",
"tokio",
]
[[package]]
-name = "datafusion-pruning"
+name = "datafusion-proto-common"
version = "55.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "74b06b333405c05015836ed36cea02cc29d1e2ee8fb414a77532d8346fb56492"
+checksum = "57901d70c724dab7cb604fea579f5324332244e3c400ff2d7c9b5fc065051437"
+dependencies = [
+ "arrow",
+ "datafusion-common",
+ "prost",
+]
+
+[[package]]
+name = "datafusion-proto-models"
+version = "55.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a574908455152a11abea49c7916cc8a504de946e5302c7e40bab632cbbcb9b2"
+dependencies = [
+ "datafusion-common",
+ "datafusion-proto-common",
+ "prost",
+]
+
+[[package]]
+name = "datafusion-pruning"
+version = "55.1.0"
dependencies = [
"arrow",
"datafusion-common",
"datafusion-datasource",
+ "datafusion-expr",
"datafusion-expr-common",
+ "datafusion-functions-nested",
"datafusion-physical-expr",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
+ "insta",
+ "itertools 0.15.0",
"log",
+ "rstest",
]
[[package]]
@@ -2942,6 +3044,12 @@ version = "1.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34"
+[[package]]
+name = "encode_unicode"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
+
[[package]]
name = "equator"
version = "0.4.2"
@@ -3195,6 +3303,12 @@ version = "0.3.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd"
+[[package]]
+name = "futures-timer"
+version = "3.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968"
+
[[package]]
name = "futures-util"
version = "0.3.34"
@@ -3283,6 +3397,19 @@ version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b"
+[[package]]
+name = "globset"
+version = "0.4.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "07c34a9410465b45bd9787443bc7370f37735bad04b0f0cd57ff1a3186c98988"
+dependencies = [
+ "aho-corasick",
+ "bstr",
+ "log",
+ "regex-automata",
+ "regex-syntax",
+]
+
[[package]]
name = "gloo-timers"
version = "0.3.0"
@@ -3323,6 +3450,8 @@ dependencies = [
"cfg-if",
"crunchy",
"num-traits",
+ "rand 0.9.5",
+ "rand_distr",
"zerocopy",
]
@@ -3827,6 +3956,22 @@ dependencies = [
"generic-array",
]
+[[package]]
+name = "insta"
+version = "1.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "86f0f8fee8c926415c58d6ae43a08523a26faccb2323f5e6b644fe7dd4ef6b82"
+dependencies = [
+ "console",
+ "globset",
+ "once_cell",
+ "regex",
+ "similar",
+ "strip-ansi-escapes",
+ "tempfile",
+ "walkdir",
+]
+
[[package]]
name = "inventory"
version = "0.3.24"
@@ -4880,6 +5025,16 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e"
+[[package]]
+name = "page_size"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da"
+dependencies = [
+ "libc",
+ "winapi",
+]
+
[[package]]
name = "parking"
version = "2.2.1"
@@ -5287,6 +5442,15 @@ dependencies = [
"syn 3.0.5",
]
+[[package]]
+name = "proc-macro-crate"
+version = "3.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f"
+dependencies = [
+ "toml_edit",
+]
+
[[package]]
name = "proc-macro2"
version = "1.0.107"
@@ -5490,6 +5654,7 @@ version = "0.8.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c"
dependencies = [
+ "libc",
"rand_chacha 0.3.1",
"rand_core 0.6.4",
]
@@ -5559,6 +5724,16 @@ version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
+[[package]]
+name = "rand_distr"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463"
+dependencies = [
+ "num-traits",
+ "rand 0.9.5",
+]
+
[[package]]
name = "rand_pcg"
version = "0.10.2"
@@ -5652,6 +5827,12 @@ version = "0.8.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
+[[package]]
+name = "relative-path"
+version = "1.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2"
+
[[package]]
name = "reqsign-aliyun-oss"
version = "3.1.5"
@@ -5915,6 +6096,46 @@ dependencies = [
"zeroize",
]
+[[package]]
+name = "rstest"
+version = "0.26.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f5a3193c063baaa2a95a33f03035c8a72b83d97a54916055ba22d35ed3839d49"
+dependencies = [
+ "futures-timer",
+ "futures-util",
+ "rstest_macros",
+]
+
+[[package]]
+name = "rstest_macros"
+version = "0.26.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c845311f0ff7951c5506121a9ad75aec44d083c31583b2ea5a30bcb0b0abba0"
+dependencies = [
+ "cfg-if",
+ "glob",
+ "proc-macro-crate",
+ "proc-macro2",
+ "quote",
+ "regex",
+ "relative-path",
+ "rustc_version",
+ "syn 2.0.119",
+ "unicode-ident",
+]
+
+[[package]]
+name = "rstest_reuse"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b3a8fb4672e840a587a66fc577a5491375df51ddb88f2a2c2a792598c326fe14"
+dependencies = [
+ "quote",
+ "rand 0.8.8",
+ "syn 2.0.119",
+]
+
[[package]]
name = "rust-ini"
version = "0.21.3"
@@ -6384,6 +6605,12 @@ version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"
+[[package]]
+name = "similar"
+version = "2.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa"
+
[[package]]
name = "siphasher"
version = "1.0.3"
@@ -6476,6 +6703,15 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f446288b699d66d0fd2e30d1cfe7869194312524b3b9252594868ed26ef056a"
+[[package]]
+name = "strip-ansi-escapes"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2a8f8038e7e7969abb3f1b7c2a811225e9296da208539e0f79c5251d6cac0025"
+dependencies = [
+ "vte",
+]
+
[[package]]
name = "strsim"
version = "0.11.1"
@@ -6587,7 +6823,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
dependencies = [
"fastrand",
- "getrandom 0.3.4",
+ "getrandom 0.4.3",
"once_cell",
"rustix",
"windows-sys 0.61.2",
@@ -6757,6 +6993,7 @@ dependencies = [
"bytes",
"libc",
"mio",
+ "parking_lot",
"pin-project-lite",
"signal-hook-registry",
"socket2",
@@ -6811,6 +7048,36 @@ dependencies = [
"tokio",
]
+[[package]]
+name = "toml_datetime"
+version = "1.1.1+spec-1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7"
+dependencies = [
+ "serde_core",
+]
+
+[[package]]
+name = "toml_edit"
+version = "0.25.15+spec-1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1340ea94a5856333492c9064b02c778b191dd2c853778d9609debdcdfea3a614"
+dependencies = [
+ "indexmap 2.14.2",
+ "toml_datetime",
+ "toml_parser",
+ "winnow",
+]
+
+[[package]]
+name = "toml_parser"
+version = "1.1.3+spec-1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56"
+dependencies = [
+ "winnow",
+]
+
[[package]]
name = "tower"
version = "0.5.3"
@@ -7082,6 +7349,15 @@ version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64"
+[[package]]
+name = "vte"
+version = "0.14.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "231fdcd7ef3037e8330d8e17e61011a2c244126acc0a982f4040ac3f9f0bc077"
+dependencies = [
+ "memchr",
+]
+
[[package]]
name = "walkdir"
version = "2.5.0"
@@ -7464,6 +7740,15 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
+[[package]]
+name = "winnow"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81"
+dependencies = [
+ "memchr",
+]
+
[[package]]
name = "wit-bindgen"
version = "0.57.1"
diff --git a/native/Cargo.toml b/native/Cargo.toml
index dac17e5a28..7ec6487439 100644
--- a/native/Cargo.toml
+++ b/native/Cargo.toml
@@ -17,7 +17,7 @@
[workspace]
default-members = ["core", "spark-expr", "common", "proto", "jni-bridge", "shuffle", "operators", "comet-udf-sdk", "comet-test-udfs"]
-members = ["core", "spark-expr", "common", "proto", "jni-bridge", "shuffle", "operators", "comet-udf-sdk", "comet-test-udfs"]
+members = ["core", "spark-expr", "common", "proto", "jni-bridge", "shuffle", "operators", "comet-udf-sdk", "comet-test-udfs", "vendor/datafusion-physical-plan", "vendor/datafusion-pruning"]
# Crates under ../contrib are intentionally NOT workspace members. Core pulls them in as
# optional path dependencies when their corresponding contrib feature is enabled.
exclude = ["../contrib"]
@@ -86,6 +86,12 @@ iceberg = { git = "https://github.com/apache/iceberg-rust", rev = "af1da4c5bc861
iceberg-storage-opendal = { git = "https://github.com/apache/iceberg-rust", rev = "af1da4c5bc86178c38c1c6db0578bdd9fb7021e3", features = ["opendal-memory", "opendal-fs", "opendal-s3", "opendal-gcs", "opendal-oss", "opendal-azdls"] }
reqsign-core = "3"
+# Backport DataFusion #25602 without changing Comet's Arrow/Iceberg dependencies.
+# Keep both crates together; provenance and removal instructions are in vendor/README.md.
+[patch.crates-io]
+datafusion-physical-plan = { path = "vendor/datafusion-physical-plan" }
+datafusion-pruning = { path = "vendor/datafusion-pruning" }
+
[profile.release]
debug = true
overflow-checks = false
diff --git a/native/core/src/execution/operators/dynamic_filter/join/tests.rs b/native/core/src/execution/operators/dynamic_filter/join/tests.rs
index 14242b4e4f..6ae974e853 100644
--- a/native/core/src/execution/operators/dynamic_filter/join/tests.rs
+++ b/native/core/src/execution/operators/dynamic_filter/join/tests.rs
@@ -1146,6 +1146,89 @@ async fn broadcast_filter_reaches_parquet_reader_after_complete_build() {
assert_eq!(unclustered_enabled_filter_rows, 400);
}
+#[tokio::test]
+async fn bitmap_filter_prunes_parquet_row_groups_inside_build_bounds() {
+ for mode in [PartitionMode::CollectLeft, PartitionMode::Partitioned] {
+ let mut scan_bytes = Vec::new();
+ for enabled in [false, true] {
+ let mut config = SessionConfig::new()
+ .with_target_partitions(1)
+ .with_parquet_page_index_pruning(false);
+ // Isolate metadata pruning from filtering decoded rows or pages.
+ config.options_mut().execution.parquet.pushdown_filters = false;
+ let session = Arc::new(SessionContext::new_with_config(config));
+ let (_file, scan) = parquet_probe((0..400).collect(), &session, 100);
+ let filter = filtered_probe(&scan);
+ let schema = Arc::new(Schema::new(vec![Field::new("key", DataType::Int32, false)]));
+ let build = memory_exec(
+ [50, 350]
+ .into_iter()
+ .map(|key| {
+ RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![Arc::new(Int32Array::from(vec![key]))],
+ )
+ .unwrap()
+ })
+ .collect(),
+ );
+ let join =
+ single_key_join_plans(build, Arc::clone(&filter) as Arc<dyn ExecutionPlan>, mode);
+ let plan = PhysicalPlanner::apply_join_dynamic_filter(
+ Arc::new(join),
+ enabled,
+ session.copied_config().options(),
+ )
+ .unwrap();
+ let expected = RecordBatch::try_new(
+ plan.schema(),
+ vec![
+ Arc::new(Int32Array::from(vec![50, 350])),
+ Arc::new(Int32Array::from(vec![50, 350])),
+ ],
+ )
+ .unwrap();
+ let output = collect(Arc::clone(&plan), session.task_ctx())
+ .await
+ .unwrap();
+ assert_eq!(
+ batches_to_sort_string(&output),
+ batches_to_sort_string(&[expected]),
+ "{mode:?}, enabled={enabled}"
+ );
+
+ // Every row group overlaps the build's [50, 350] bounds. Only the
+ // bitmap can exclude the interior groups [100, 199] and [200, 299].
+ // Comet's runtime join forces map membership rather than IN lists.
+ assert_eq!(
+ pruning_metric(&scan, "row_groups_pruned_statistics"),
+ if enabled { 2 } else { 0 },
+ "{mode:?}, enabled={enabled}"
+ );
+ assert_eq!(
+ filter.metrics().unwrap().output_rows().unwrap(),
+ if enabled { 200 } else { 400 }
+ );
+ if enabled {
+ assert_eq!(metric(&plan, "dynamic_filter_join_filters_attached"), 1);
+ }
+ scan_bytes.push(
+ scan.metrics()
+ .unwrap()
+ .sum_by_name("bytes_scanned")
+ .unwrap()
+ .as_usize(),
+ );
+ }
+ assert!(
+ scan_bytes[1] < scan_bytes[0],
+ "{mode:?}: bitmap pruning should avoid data reads: enabled={}, disabled={}",
+ scan_bytes[1],
+ scan_bytes[0]
+ );
+ }
+}
+
fn limited_session(bytes: usize) -> (SessionContext, Arc<dyn MemoryPool>) {
let pool: Arc<dyn MemoryPool> = Arc::new(GreedyMemoryPool::new(bytes));
let runtime = RuntimeEnvBuilder::new()
diff --git a/native/vendor/README.md b/native/vendor/README.md
new file mode 100644
index 0000000000..d7d3dadb51
--- /dev/null
+++ b/native/vendor/README.md
@@ -0,0 +1,91 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# DataFusion bitmap pruning backport
+
+These two crates backport [DataFusion #25602](https://github.com/apache/datafusion/pull/25602)
+to DataFusion 55.1.0. The native workspace patches both crates together, so hash joins and
+Parquet pruning use the same bitmap type. All other DataFusion dependencies remain on the
+existing locked release, as do Arrow, Parquet, and Iceberg.
+
+The upstream implementation first landed at
+`4bb9f8d3192f6c945f2bf933231380e60341af66`, whose dependency graph uses Arrow 60.
+Comet's pinned Iceberg revision uses Arrow 59. This temporary backport makes the bitmap
+available without also requiring an Iceberg migration or an unmerged Iceberg dependency.
+It does not enable Comet's experimental join runtime filtering by default or add Delta scans.
+
+## Provenance
+
+The source archives are the published crates.io 55.1.0 packages, based on DataFusion release
+commit `7d3835c71f30cbd3c3ae4041732267f1f453097a`. Their SHA-256 checksums match the
+original entries in Comet's lockfile:
+
+| Package | Archive SHA-256 |
+| -------------------------- | ------------------------------------------------------------------ |
+| `datafusion-physical-plan` | `1265d58e5bce07d154e642a51ff43033b576a6ae40d50a29ac2b9f311004eb52` |
+| `datafusion-pruning` | `74b06b333405c05015836ed36cea02cc29d1e2ee8fb414a77532d8346fb56492` |
+
+The crates retain their licenses, notices, normalized Cargo manifests, and original manifests.
+Manifest license headers and the upstream Rust formatting configuration are included for
+Comet's repository checks. Deprecation warnings are allowed in the physical-plan crate so
+its pre-Rust-1.99 atomic API remains compatible with DataFusion 55.1.0's minimum Rust version.
+Package-local lockfiles and Cargo cache markers are omitted; `native/Cargo.lock` controls the
+build. [bitmap-backport.patch](bitmap-backport.patch) records every source and manifest change
+relative to those archives.
+
+## Adaptations to 55.1.0
+
+- Preserve the four-argument `HashTableLookupExpr::new` API. The new
+ `with_pruning_bitmap` builder attaches the optional summary without changing other callers.
+- Use 55.1.0's existing hash-join build path. It has no prepared-build API; skip the summary
+ whenever dynamic filters are not computed. Failure to reserve bitmap memory only disables
+ this optional optimization.
+- Retain the merged PR's conservative handling of `CASE` without `ELSE`, which must preserve
+ implicit NULLs during Parquet full-match inference.
+- Keep existing IN-list pruning unchanged. Port only the bitmap statistics rewrite and CASE
+ support, adapting the recursive call to the 55.1.0 signature. Recognize lossless casts from
+ unsigned integers to larger signed integers locally, since 55.1.0's cast helper predates
+ that support.
+- Include upstream bitmap, expression-sharing, and memory-limit tests, plus regressions for
+ missing statistics and implicit NULLs. Comet's join test verifies that both supported native
+ hash-join modes skip interior row groups that min/max filtering alone cannot exclude.
+
+## Validation
+
+From `native/`, run:
+
+```sh
+cargo test -p datafusion-pruning --lib
+cargo test -p datafusion-physical-plan --lib key_range_bitmap
+cargo test -p datafusion-physical-plan --lib pruning_bitmap
+cargo test -p datafusion-comet --lib dynamic_filter
+```
+
+The Comet regression is named `bitmap_filter_prunes_parquet_row_groups_inside_build_bounds`.
+It checks exact join results, reader attachment, row groups pruned, decoded rows, and bytes
+read with filtering enabled and disabled. Page and decode-time row filtering are disabled
+in this test to isolate row-group pruning.
+
+## Removal
+
+When Comet can adopt a DataFusion release containing #25602 together with compatible Arrow
+and Iceberg dependencies, remove both vendored crates, their workspace-member entries, and
+the `[patch.crates-io]` entries in `native/Cargo.toml`, then update the lockfile. Retain the
+Comet regression test. Do not update just one patched crate: the producer and pruning reader
+must agree on the hash-lookup expression and bitmap implementation.
diff --git a/native/vendor/bitmap-backport.patch b/native/vendor/bitmap-backport.patch
new file mode 100644
index 0000000000..d55725c6da
--- /dev/null
+++ b/native/vendor/bitmap-backport.patch
@@ -0,0 +1,1175 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+--- a/datafusion-physical-plan/Cargo.toml
++++ b/datafusion-physical-plan/Cargo.toml
+@@ -1,3 +1,20 @@
++# Licensed to the Apache Software Foundation (ASF) under one
++# or more contributor license agreements. See the NOTICE file
++# distributed with this work for additional information
++# regarding copyright ownership. The ASF licenses this file
++# to you under the Apache License, Version 2.0 (the
++# "License"); you may not use this file except in compliance
++# with the License. You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing,
++# software distributed under the License is distributed on an
++# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
++# KIND, either express or implied. See the License for the
++# specific language governing permissions and limitations
++# under the License.
++
+ # THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
+ #
+ # When uploading crates to the registry Cargo will automatically
+@@ -276,6 +293,9 @@
+ used_underscore_binding = "warn"
+
+ [lints.rust]
++# Rust 1.99 renamed AtomicUsize::fetch_update. Retain the upstream implementation
++# and its Rust 1.94 minimum instead of requiring the new try_update method.
++deprecated = "allow"
+ unused_qualifications = "deny"
+
+ [lints.rust.unexpected_cfgs]
+--- a/datafusion-physical-plan/src/joins/hash_join/exec.rs
++++ b/datafusion-physical-plan/src/joins/hash_join/exec.rs
+@@ -40,6 +40,7 @@
+ BuildSide, BuildSideInitialState, HashJoinStream, HashJoinStreamState,
+ };
+ use crate::joins::join_hash_map::{JoinHashMapU32, JoinHashMapU64};
++use crate::joins::key_range_bitmap::KeyRangeBitmap;
+ use crate::joins::utils::{
+ OnceAsync, OnceFut, asymmetric_join_output_partitioning, reorder_output_after_swap,
+ swap_join_projection, update_hash,
+@@ -2384,7 +2385,8 @@
+
+ let map = Arc::new(join_hash_map);
+
+- let membership = if num_rows == 0 {
++ // Nothing reads the strategy unless this join computes dynamic filters.
++ let membership = if num_rows == 0 || !should_compute_dynamic_filters {
+ PushdownStrategy::Empty
+ } else {
+ // If the build side is small enough we can use IN list pushdown.
+@@ -2394,19 +2396,45 @@
+ .iter()
+ .map(|arr| arr.get_array_memory_size())
+ .sum::<usize>();
+- if left_values.is_empty()
+- || left_values[0].is_empty()
+- || estimated_size > config.optimizer.hash_join_inlist_pushdown_max_size
+- || map.num_of_distinct_key()
+- > config
++
++ let pushdown_inlist = !left_values.is_empty()
++ && !left_values[0].is_empty()
++ && estimated_size <= config.optimizer.hash_join_inlist_pushdown_max_size
++ && map.num_of_distinct_key()
++ <= config
+ .optimizer
+- .hash_join_inlist_pushdown_max_distinct_values
++ .hash_join_inlist_pushdown_max_distinct_values;
++
++ if pushdown_inlist
++ && let Some(in_list_values) = build_struct_inlist_values(&left_values)?
+ {
+- PushdownStrategy::Map(Arc::clone(&map))
+- } else if let Some(in_list_values) = build_struct_inlist_values(&left_values)? {
+ PushdownStrategy::InList(in_list_values)
+ } else {
+- PushdownStrategy::Map(Arc::clone(&map))
++ // Past the InList threshold use a bucket bitmap for container pruning.
++ let pruning_bitmap = match (left_values.as_slice(), bounds.as_ref()) {
++ ([keys], Some(bounds)) if !keys.is_empty() => bounds
++ .get_column_bounds(0)
++ .and_then(|b| {
++ KeyRangeBitmap::try_new(
++ keys,
++ &b.min,
++ &b.max,
++ map.num_of_distinct_key(),
++ )
++ })
++ .map(Arc::new),
++ _ => None,
++ };
++ // Held for the join's lifetime, so charge it like the maps; it is
++ // optional, so skip it rather than fail when the pool is full.
++ let pruning_bitmap = pruning_bitmap.filter(|bitmap| {
++ let ok = reservation.try_grow(bitmap.size()).is_ok();
++ if ok {
++ metrics.build_mem_used.add(bitmap.size());
++ }
++ ok
++ });
++ PushdownStrategy::Map(Arc::clone(&map), pruning_bitmap)
+ }
+ };
+
+@@ -2503,6 +2531,71 @@
+ use insta::{allow_duplicates, assert_snapshot};
+ use rstest::*;
+ use rstest_reuse::*;
++
++ /// Runs a join whose 200 keys spread over 2M would size a pruning bitmap
++ /// at the 128 KiB cap, and reports the bytes the build side reserved.
++ ///
++ /// Forces the IN-list threshold off: under `force_hash_collisions` every
++ /// key hashes to the same bucket, so `num_of_distinct_key()` no longer
++ /// reflects the real 200 distinct keys and would otherwise take the
++ /// IN-list branch instead of the one under test.
++ async fn build_mem_used(limit: usize, dynamic_filters: bool) -> Result<usize> {
++ let schema = Arc::new(Schema::new(vec![Field::new("k", DataType::Int64, false)]));
++ let batch = |keys: Vec<i64>| {
++ let column = Arc::new(Int64Array::from(keys)) as ArrayRef;
++ let batch = RecordBatch::try_new(Arc::clone(&schema), vec![column])?;
++ TestMemoryExec::try_new_exec(&[vec![batch]], Arc::clone(&schema), None)
++ };
++ let on = vec![(
++ Arc::new(Column::new_with_schema("k", &schema)?) as _,
++ Arc::new(Column::new_with_schema("k", &schema)?) as _,
++ )];
++ let build = batch((0..200).map(|i| i * 10_000).collect())?;
++ let probe = batch(vec![0, 500_000, 1_500_000])?;
++ let join = if dynamic_filters {
++ hash_join_with_dynamic_filter(build, probe, on, JoinType::Inner)?.0
++ } else {
++ join(
++ build,
++ probe,
++ on,
++ &JoinType::Inner,
++ NullEquality::NullEqualsNothing,
++ )?
++ };
++
++ let runtime = RuntimeEnvBuilder::new()
++ .with_memory_limit(limit, 1.0)
++ .build_arc()?;
++ let mut config = SessionConfig::new();
++ config
++ .options_mut()
++ .optimizer
++ .hash_join_inlist_pushdown_max_size = 0;
++ let task_ctx = Arc::new(
++ TaskContext::default()
++ .with_runtime(runtime)
++ .with_session_config(config),
++ );
++ let batches = common::collect(join.execute(0, task_ctx)?).await?;
++ assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 3);
++ Ok(join
++ .metrics()
++ .unwrap()
++ .sum_by_name("build_mem_used")
++ .unwrap()
++ .as_usize())
++ }
++
++ /// The bitmap is pruning-only: never built when no dynamic filter will read
++ /// it, and dropped rather than fatal when the pool cannot fit it.
++ #[tokio::test]
++ async fn pruning_bitmap_is_optional() -> Result<()> {
++ assert!(build_mem_used(1_000_000, false).await? < 100_000);
++ assert!(build_mem_used(1_000_000, true).await? > 100_000);
++ build_mem_used(100_000, true).await?;
++ Ok(())
++ }
+
+ #[derive(Debug)]
+ struct PartitionedTestExec {
+--- a/datafusion-physical-plan/src/joins/hash_join/partitioned_hash_eval.rs
++++ b/datafusion-physical-plan/src/joins/hash_join/partitioned_hash_eval.rs
+@@ -35,6 +35,7 @@
+ };
+
+ use crate::joins::Map;
++use crate::joins::key_range_bitmap::KeyRangeBitmap;
+
+ /// RandomState wrapper that preserves the seed used to create it.
+ ///
+@@ -265,6 +266,8 @@
+ map: Arc<Map>,
+ /// Description for display
+ description: String,
++ /// Shared pruning-only view of the build side's key range
++ pruning_bitmap: Option<Arc<KeyRangeBitmap>>,
+ }
+ impl HashTableLookupExpr {
+ /// Create a new HashTableLookupExpr
+@@ -288,7 +291,19 @@
+ random_state,
+ map,
+ description,
+- }
++ pruning_bitmap: None,
++ }
++ }
++
++ /// Attach a key-range summary for container pruning.
++ pub fn with_pruning_bitmap(mut self, bitmap: Option<Arc<KeyRangeBitmap>>) -> Self {
++ self.pruning_bitmap = bitmap;
++ self
++ }
++
++ /// Which parts of this build side's key range hold keys, for container pruning.
++ pub fn pruning_bitmap(&self) -> Option<&Arc<KeyRangeBitmap>> {
++ self.pruning_bitmap.as_ref()
+ }
+ }
+ impl std::fmt::Debug for HashTableLookupExpr {
+@@ -353,12 +368,13 @@
+ self: Arc<Self>,
+ children: Vec<Arc<dyn PhysicalExpr>>,
+ ) -> Result<Arc<dyn PhysicalExpr>> {
+- Ok(Arc::new(HashTableLookupExpr::new(
+- children,
+- self.random_state.clone(),
+- Arc::clone(&self.map),
+- self.description.clone(),
+- )))
++ Ok(Arc::new(HashTableLookupExpr {
++ on_columns: children,
++ random_state: self.random_state.clone(),
++ map: Arc::clone(&self.map),
++ description: self.description.clone(),
++ pruning_bitmap: self.pruning_bitmap.clone(),
++ }))
+ }
+
+ fn data_type(&self, _input_schema: &Schema) -> Result<DataType> {
+@@ -438,6 +454,7 @@
+ mod tests {
+ use super::*;
+ use crate::joins::join_hash_map::JoinHashMapU32;
++ use datafusion_common::ScalarValue;
+ use datafusion_physical_expr::expressions::Column;
+ use std::collections::hash_map::DefaultHasher;
+ use std::hash::Hasher;
+@@ -446,6 +463,38 @@
+ let mut hasher = DefaultHasher::new();
+ value.hash(&mut hasher);
+ hasher.finish()
++ }
++
++ /// The pruning bitmap must survive `with_new_children`, which a scan calls
++ /// once per file.
++ #[test]
++ fn test_pruning_bitmap_shared_with_derived_children() {
++ let keys: ArrayRef = Arc::new(arrow::array::Int64Array::from(vec![0, 1_000_000]));
++ let bitmap = Arc::new(
++ KeyRangeBitmap::try_new(
++ &keys,
++ &ScalarValue::Int64(Some(0)),
++ &ScalarValue::Int64(Some(1_000_000)),
++ 2,
++ )
++ .expect("two keys a million apart leave prunable gaps"),
++ );
++ let expr = Arc::new(
++ HashTableLookupExpr::new(
++ vec![Arc::new(Column::new("a", 0))],
++ SeededRandomState::with_seed(1),
++ Arc::new(Map::HashMap(Box::new(JoinHashMapU32::with_capacity(10)))),
++ "hash_lookup".to_string(),
++ )
++ .with_pruning_bitmap(Some(Arc::clone(&bitmap))),
++ );
++
++ let derived = expr
++ .with_new_children(vec![Arc::new(Column::new("a", 7))])
++ .unwrap();
++ let derived = derived.downcast_ref::<HashTableLookupExpr>().unwrap();
++ assert_eq!(derived.children()[0].to_string(), "a@7");
++ assert!(Arc::ptr_eq(&bitmap, derived.pruning_bitmap().unwrap()));
+ }
+
+ #[test]
+--- a/datafusion-physical-plan/src/joins/hash_join/shared_bounds.rs
++++ b/datafusion-physical-plan/src/joins/hash_join/shared_bounds.rs
+@@ -31,6 +31,7 @@
+ use crate::joins::hash_join::partitioned_hash_eval::{
+ HashExpr, HashTableLookupExpr, SeededRandomState,
+ };
++use crate::joins::key_range_bitmap::KeyRangeBitmap;
+ use crate::repartition::RangeExpr;
+ use arrow::array::ArrayRef;
+ use arrow::datatypes::{DataType, Field, Schema};
+@@ -136,12 +137,16 @@
+ )?)))
+ }
+ // Use hash table lookup for large build sides
+- PushdownStrategy::Map(hash_map) => Ok(Some(Arc::new(HashTableLookupExpr::new(
+- on_right.to_vec(),
+- random_state.clone(),
+- hash_map,
+- "hash_lookup".to_string(),
+- )) as Arc<dyn PhysicalExpr>)),
++ PushdownStrategy::Map(hash_map, pruning_bitmap) => Ok(Some(Arc::new(
++ HashTableLookupExpr::new(
++ on_right.to_vec(),
++ random_state.clone(),
++ hash_map,
++ "hash_lookup".to_string(),
++ )
++ .with_pruning_bitmap(pruning_bitmap),
++ )
++ as Arc<dyn PhysicalExpr>)),
+ // Empty partition - should not create a filter for this
+ PushdownStrategy::Empty => Ok(None),
+ }
+@@ -277,8 +282,9 @@
+ pub(crate) enum PushdownStrategy {
+ /// Use InList for small build sides (< 128MB)
+ InList(ArrayRef),
+- /// Use map lookup for large build sides
+- Map(Arc<Map>),
++ /// Use map lookup for large build sides. If `Some` the second field
++ /// represents the build side's keys using a bucket bitmap.
++ Map(Arc<Map>, Option<Arc<KeyRangeBitmap>>),
+ /// There was no data in this partition, do not build a dynamic filter for it
+ Empty,
+ }
+--- /dev/null
++++ b/datafusion-physical-plan/src/joins/key_range_bitmap.rs
+@@ -0,0 +1,238 @@
++// Licensed to the Apache Software Foundation (ASF) under one
++// or more contributor license agreements. See the NOTICE file
++// distributed with this work for additional information
++// regarding copyright ownership. The ASF licenses this file
++// to you under the Apache License, Version 2.0 (the
++// "License"); you may not use this file except in compliance
++// with the License. You may obtain a copy of the License at
++//
++// http://www.apache.org/licenses/LICENSE-2.0
++//
++// Unless required by applicable law or agreed to in writing,
++// software distributed under the License is distributed on an
++// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
++// KIND, either express or implied. See the License for the
++// specific language governing permissions and limitations
++// under the License.
++
++//! Bitmap over a hash join build side's key range, for container pruning.
++
++use arrow::array::{Array, ArrayRef, BooleanArray, downcast_integer_array};
++use arrow::buffer::{BooleanBuffer, MutableBuffer};
++use arrow::util::bit_util;
++use datafusion_common::ScalarValue;
++use num_traits::AsPrimitive;
++
++use crate::joins::array_map::ArrayMap;
++
++/// Largest bitmap, in bits (128 KiB), which bounds what one build side can hold.
++const MAX_BUCKETS: u64 = 1 << 20;
++
++/// Which parts of a build side's key range hold at least one key.
++///
++/// Keys map order-preservingly onto buckets spanning `[min, max]`, one bit each,
++/// so "can a container holding `[lo, hi]` match?" is a scan of the bits spanning
++/// `[lo, hi]`: all clear proves no build key lies in it.
++///
++/// Below, keys 3 to 97 over eight buckets: `offset` is 3, `span` is 97 - 3, and
++/// eight buckets need `shift` = 4, so each covers `1 << 4` keys starting at
++/// `offset`. Only buckets 0 to `span >> shift` can hold a key. Real bitmaps run
++/// to `MAX_BUCKETS`; eight is what fits on a line.
++///
++/// ```text
++/// keys 3 12 42 97
++/// v v v v
++/// bucket | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
++/// covers 3-18 19-34 35-50 51-66 67-82 83-98 unused unused
++/// bit 1 0 1 0 0 1 0 0
++/// ```
++///
++/// A container is kept when any bucket its `[min, max]` touches has a bit set.
++#[derive(Debug)]
++pub struct KeyRangeBitmap {
++ /// Smallest key, in the `u64` ordering the join's array map uses.
++ offset: u64,
++ /// `max - min`: a key is in range iff `key.wrapping_sub(offset) <= span`.
++ span: u64,
++ /// Each bucket covers `1 << shift` keys.
++ shift: u32,
++ /// One bit per bucket.
++ bits: BooleanBuffer,
++}
++
++impl KeyRangeBitmap {
++ /// Maps `array`'s non-null values, which must lie within `[min, max]`.
++ ///
++ /// `None` when the key type has no `u64` ordering, or when the bitmap could
++ /// not exclude anything the `[min, max]` bounds predicate does not already.
++ pub fn try_new(
++ array: &ArrayRef,
++ min: &ScalarValue,
++ max: &ScalarValue,
++ distinct_keys: usize,
++ ) -> Option<Self> {
++ if !ArrayMap::is_supported_type(array.data_type()) {
++ return None;
++ }
++ let offset = ArrayMap::key_to_u64(min)?;
++ let span = ArrayMap::key_to_u64(max)?.wrapping_sub(offset);
++
++ // One bucket per key value where the range allows it, so the bitmap is
++ // exact; otherwise as many as the cap permits.
++ let buckets = span.saturating_add(1).min(MAX_BUCKETS).next_power_of_two();
++
++ // Use a heuristic to bail-out for the common shape which renders each bucket
++ // set (contiguous surrogate keys, a date range) before actually computing it.
++ if (distinct_keys as u64).saturating_mul(2) > span.min(buckets) {
++ return None;
++ }
++
++ let shift = (0u32..63).find(|s| (span >> s) < buckets).unwrap_or(63);
++
++ let bits = bucket_bits(array, offset, shift, buckets);
++
++ // Every bucket set means nothing can be excluded.
++ let reachable = ((span >> shift) + 1) as usize;
++ bits.slice(0, reachable).has_false().then_some(Self {
++ offset,
++ span,
++ shift,
++ bits,
++ })
++ }
++
++ /// One Boolean per `[min[i], max[i]]` container interval: `false` proves no
++ /// build key lies in that container, `None` where a bound is absent or the
++ /// statistics are not integer keys.
++ pub fn may_contain_ranges(&self, min: &dyn Array, max: &dyn Array) -> BooleanArray {
++ // Both bounds must be the same integer type for the tuple pattern to match.
++ downcast_integer_array!(
++ (min, max) => {
++ min.iter()
++ .zip(max.iter())
++ .map(|bounds| match bounds {
++ (Some(lo), Some(hi)) => {
++ Some(self.may_contain_range(lo.as_(), hi.as_()))
++ }
++ _ => None,
++ })
++ .collect()
++ }
++ _ => BooleanArray::new_null(min.len()),
++ )
++ }
++
++ /// Might a container spanning `[lo, hi]` hold a build key? `false` proves it
++ /// cannot. Bounds use the same `u64` key ordering as the join's array map.
++ pub fn may_contain_range(&self, lo: u64, hi: u64) -> bool {
++ // Offsets from the minimum; anything outside `[min, max]` wraps to
++ // something huge, which the clamping below turns into "past the end".
++ let lo_off = lo.wrapping_sub(self.offset);
++ let hi_off = hi.wrapping_sub(self.offset);
++ if lo_off > self.span && hi_off > self.span && lo_off <= hi_off {
++ return false;
++ }
++ let lo_b = if lo_off > self.span {
++ 0
++ } else {
++ lo_off >> self.shift
++ };
++ let hi_b = (hi_off.min(self.span) >> self.shift).max(lo_b);
++ self.bits
++ .slice(lo_b as usize, (hi_b - lo_b + 1) as usize)
++ .has_true()
++ }
++
++ /// Bytes held, for the build-side memory reservation.
++ pub fn size(&self) -> usize {
++ self.bits.inner().capacity()
++ }
++
++ /// Buckets set and buckets total, for `EXPLAIN`.
++ pub fn fill_stats(&self) -> (usize, usize) {
++ (self.bits.count_set_bits(), self.bits.len())
++ }
++}
++
++/// One bit per bucket, set for each bucket holding a non-null key of `keys`.
++fn bucket_bits(keys: &dyn Array, offset: u64, shift: u32, buckets: u64) -> BooleanBuffer {
++ let mut words = MutableBuffer::new_null(buckets as usize);
++ downcast_integer_array!(
++ keys => {
++ let bytes = words.as_slice_mut();
++ let mut set = |key: u64| {
++ let bucket = (key.wrapping_sub(offset) >> shift).min(buckets - 1);
++ bit_util::set_bit(bytes, bucket as usize);
++ };
++ if keys.null_count() == 0 {
++ keys.values().iter().for_each(|key| set((*key).as_()));
++ } else {
++ keys.iter().flatten().for_each(|key| set(key.as_()));
++ }
++ }
++ _ => unreachable!("guarded by ArrayMap::is_supported_type"),
++ );
++ BooleanBuffer::new(words.into(), 0, buckets as usize)
++}
++
++#[cfg(test)]
++mod tests {
++ use super::*;
++ use arrow::array::Int64Array;
++ use std::sync::Arc;
++
++ fn bitmap(keys: &[i64]) -> Option<KeyRangeBitmap> {
++ let a: ArrayRef = Arc::new(Int64Array::from(keys.to_vec()));
++ KeyRangeBitmap::try_new(
++ &a,
++ &ScalarValue::Int64(Some(*keys.iter().min().unwrap())),
++ &ScalarValue::Int64(Some(*keys.iter().max().unwrap())),
++ keys.iter().collect::<std::collections::HashSet<_>>().len(),
++ )
++ }
++ fn may(b: &KeyRangeBitmap, lo: i64, hi: i64) -> bool {
++ b.may_contain_range(lo as u64, hi as u64)
++ }
++
++ #[test]
++ fn prunes_gaps_and_never_drops_a_match() {
++ let keys: Vec<i64> = (0..2_000_000).filter(|k| k % 10000 < 200).collect();
++ let s = bitmap(&keys).expect("not saturated");
++ let mut kept = 0;
++ for i in 0..2000i64 {
++ let (lo, hi) = (1000 * i, 1000 * i + 999);
++ let truth = keys
++ .binary_search(&lo)
++ .map_or_else(|p| keys.get(p).is_some_and(|k| *k <= hi), |_| true);
++ let got = may(&s, lo, hi);
++ assert!(
++ !truth || got,
++ "dropped a container holding a key: {lo}..{hi}"
++ );
++ kept += got as usize;
++ }
++ assert!(kept <= 220, "expected ~200 kept, got {kept}");
++ }
++
++ #[test]
++ fn contiguous_keys_no_bitmap() {
++ assert!(bitmap(&(0..5000i64).collect::<Vec<_>>()).is_none());
++ }
++
++ #[test]
++ fn test_bitmap_range_probing() {
++ let s = bitmap(&[0, 1_000_000]).expect("not saturated");
++ assert!(!may(&s, -500, -10));
++ assert!(!may(&s, 2_000_000, 3_000_000));
++ assert!(may(&s, -500, 10));
++
++ let s = bitmap(&[-1_000_000, -5, 0, 5, 1_000_000]).expect("not saturated");
++ assert!(may(&s, -10, 10));
++ assert!(!may(&s, -900_000, -800_000));
++
++ // The widest possible span must not overflow the bucket sizing.
++ let s = bitmap(&[i64::MIN, 0, i64::MAX]).expect("not saturated");
++ assert!(may(&s, -1, 1));
++ assert!(!may(&s, 1 << 50, 1 << 51));
++ }
++}
+--- a/datafusion-physical-plan/src/joins/mod.rs
++++ b/datafusion-physical-plan/src/joins/mod.rs
+@@ -48,6 +48,7 @@
+ /// Note: This module is public for internal testing purposes only
+ /// and is not guaranteed to be stable across versions.
+ pub mod join_hash_map;
++pub mod key_range_bitmap;
+
+ use array_map::ArrayMap;
+ use utils::JoinHashMapType;
+--- a/datafusion-pruning/Cargo.toml
++++ b/datafusion-pruning/Cargo.toml
+@@ -1,3 +1,20 @@
++# Licensed to the Apache Software Foundation (ASF) under one
++# or more contributor license agreements. See the NOTICE file
++# distributed with this work for additional information
++# regarding copyright ownership. The ASF licenses this file
++# to you under the Apache License, Version 2.0 (the
++# "License"); you may not use this file except in compliance
++# with the License. You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing,
++# software distributed under the License is distributed on an
++# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
++# KIND, either express or implied. See the License for the
++# specific language governing permissions and limitations
++# under the License.
++
+ # THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
+ #
+ # When uploading crates to the registry Cargo will automatically
+@@ -81,6 +98,9 @@
+ [dev-dependencies.itertools]
+ version = "0.15"
+
++[dev-dependencies.rstest]
++version = "0.26.1"
++
+ [lints.clippy]
+ allow_attributes = "warn"
+ assigning_clones = "warn"
+--- a/datafusion-pruning/Cargo.toml.orig
++++ b/datafusion-pruning/Cargo.toml.orig
+@@ -1,3 +1,20 @@
++# Licensed to the Apache Software Foundation (ASF) under one
++# or more contributor license agreements. See the NOTICE file
++# distributed with this work for additional information
++# regarding copyright ownership. The ASF licenses this file
++# to you under the Apache License, Version 2.0 (the
++# "License"); you may not use this file except in compliance
++# with the License. You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing,
++# software distributed under the License is distributed on an
++# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
++# KIND, either express or implied. See the License for the
++# specific language governing permissions and limitations
++# under the License.
++
+ [package]
+ name = "datafusion-pruning"
+ description = "DataFusion Pruning Logic"
+--- /dev/null
++++ b/datafusion-pruning/src/key_range_bitmap_expr.rs
+@@ -0,0 +1,103 @@
++// Licensed to the Apache Software Foundation (ASF) under one
++// or more contributor license agreements. See the NOTICE file
++// distributed with this work for additional information
++// regarding copyright ownership. The ASF licenses this file
++// to you under the Apache License, Version 2.0 (the
++// "License"); you may not use this file except in compliance
++// with the License. You may obtain a copy of the License at
++//
++// http://www.apache.org/licenses/LICENSE-2.0
++//
++// Unless required by applicable law or agreed to in writing,
++// software distributed under the License is distributed on an
++// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
++// KIND, either express or implied. See the License for the
++// specific language governing permissions and limitations
++// under the License.
++
++use std::fmt::{self, Display, Formatter};
++use std::hash::{Hash, Hasher};
++use std::sync::Arc;
++
++use arrow::datatypes::{DataType, Schema};
++use arrow::record_batch::RecordBatch;
++use datafusion_common::{Result, assert_eq_or_internal_err};
++use datafusion_physical_expr::{PhysicalExpr, PhysicalExprRef};
++use datafusion_physical_plan::ColumnarValue;
++use datafusion_physical_plan::joins::key_range_bitmap::KeyRangeBitmap;
++
++/// Tests containers' `[min, max]` statistics against a build side's
++/// [`KeyRangeBitmap`]: one nullable Boolean per container, where `false` proves
++/// no build key lies in it and `NULL` means the bounds could not decide.
++#[derive(Debug)]
++pub(crate) struct KeyRangeBitmapPruningExpr {
++ pub(crate) min: PhysicalExprRef,
++ pub(crate) max: PhysicalExprRef,
++ pub(crate) bitmap: Arc<KeyRangeBitmap>,
++}
++
++impl PartialEq for KeyRangeBitmapPruningExpr {
++ fn eq(&self, other: &Self) -> bool {
++ self.min.eq(&other.min)
++ && self.max.eq(&other.max)
++ && Arc::ptr_eq(&self.bitmap, &other.bitmap)
++ }
++}
++impl Eq for KeyRangeBitmapPruningExpr {}
++
++impl Hash for KeyRangeBitmapPruningExpr {
++ fn hash<H: Hasher>(&self, state: &mut H) {
++ self.min.hash(state);
++ self.max.hash(state);
++ Arc::as_ptr(&self.bitmap).hash(state);
++ }
++}
++
++impl Display for KeyRangeBitmapPruningExpr {
++ fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
++ let (set, total) = self.bitmap.fill_stats();
++ write!(
++ f,
++ "KEY_RANGE_BITMAP({}, {}, {set}/{total} buckets)",
++ self.min, self.max
++ )
++ }
++}
++
++impl PhysicalExpr for KeyRangeBitmapPruningExpr {
++ fn data_type(&self, _input_schema: &Schema) -> Result<DataType> {
++ Ok(DataType::Boolean)
++ }
++
++ fn nullable(&self, _input_schema: &Schema) -> Result<bool> {
++ Ok(true)
++ }
++
++ fn evaluate(&self, batch: &RecordBatch) -> Result<ColumnarValue> {
++ let rows = batch.num_rows();
++ let min = self.min.evaluate(batch)?.into_array(rows)?;
++ let max = self.max.evaluate(batch)?.into_array(rows)?;
++ let matches = self.bitmap.may_contain_ranges(&min, &max);
++ Ok(ColumnarValue::Array(Arc::new(matches)))
++ }
++
++ fn children(&self) -> Vec<&PhysicalExprRef> {
++ vec![&self.min, &self.max]
++ }
++
++ fn with_new_children(
++ self: Arc<Self>,
++ children: Vec<PhysicalExprRef>,
++ ) -> Result<PhysicalExprRef> {
++ assert_eq_or_internal_err!(children.len(), 2);
++ Ok(Arc::new(Self {
++ min: Arc::clone(&children[0]),
++ max: Arc::clone(&children[1]),
++ bitmap: Arc::clone(&self.bitmap),
++ }))
++ }
++
++ fn fmt_sql(&self, f: &mut Formatter<'_>) -> fmt::Result {
++ write!(f, "{self}")
++ }
++}
+--- a/datafusion-pruning/src/lib.rs
++++ b/datafusion-pruning/src/lib.rs
+@@ -18,6 +18,7 @@
+ #![cfg_attr(test, allow(clippy::needless_pass_by_value))]
+
+ mod file_pruner;
++mod key_range_bitmap_expr;
+ mod pruning_predicate;
+
+ pub use file_pruner::FilePruner;
+--- a/datafusion-pruning/src/pruning_predicate.rs
++++ b/datafusion-pruning/src/pruning_predicate.rs
+@@ -22,6 +22,8 @@
+ use std::collections::HashSet;
+ use std::sync::Arc;
+
++use crate::key_range_bitmap_expr::KeyRangeBitmapPruningExpr;
++
+ use arrow::array::AsArray;
+ use arrow::{
+ array::{ArrayRef, BooleanArray, new_null_array},
+@@ -48,6 +50,7 @@
+ use datafusion_physical_expr::utils::{Guarantee, LiteralGuarantee};
+ use datafusion_physical_expr::{PhysicalExprRef, expressions as phys_expr};
+ use datafusion_physical_expr_common::physical_expr::snapshot_physical_expr_opt;
++use datafusion_physical_plan::joins::HashTableLookupExpr;
+ use datafusion_physical_plan::{ColumnarValue, PhysicalExpr};
+
+ /// Used to prove that arbitrary predicates (boolean expression) can not
+@@ -1466,6 +1469,89 @@
+ /// `datafusion.execution.parquet.max_in_list_size` config option.
+ pub const MAX_IN_LIST_SIZE: usize = 20;
+
++/// Statistics expressions for `column`: its min, its max, and "not every row is
++/// NULL". Appends to `required_columns`, rolling the appends back if any part
++/// cannot be built, since [`RequiredColumns::stat_column_expr`] only appends.
++fn min_max_non_null_exprs(
++ column: &phys_expr::Column,
++ column_expr: &Arc<dyn PhysicalExpr>,
++ field: &Field,
++ schema: &Schema,
++ required_columns: &mut RequiredColumns,
++) -> Option<(PhysicalExprRef, PhysicalExprRef, PhysicalExprRef)> {
++ let appended = required_columns.columns.len();
++ let statistics = (|| {
++ let min = required_columns
++ .min_column_expr(column, column_expr, field)
++ .ok()?;
++ let max = required_columns
++ .max_column_expr(column, column_expr, field)
++ .ok()?;
++ // A cast on `column_expr` (widening it towards min/max's target type)
++ // does not affect nullability, so check it on the bare column beneath.
++ let bare_expr = match column_expr.downcast_ref::<phys_expr::CastExpr>() {
++ Some(cast) => cast.expr(),
++ None => column_expr,
++ };
++ let non_null =
++ build_is_null_column_expr(bare_expr, schema, required_columns, true)?;
++ Some((min, max, non_null))
++ })();
++ if statistics.is_none() {
++ required_columns.columns.truncate(appended);
++ }
++ statistics
++}
++
++/// Builds a "may match" expression for a [`HashTableLookupExpr`] (a large join
++/// build side pushed down as an opaque hash-table lookup): tests container
++/// min/max stats against the build side's key-range bitmap, which is fixed size
++/// and built once with the join's hash table.
++fn build_hash_lookup_pruning_expr(
++ lookup: &HashTableLookupExpr,
++ schema: &Schema,
++ required_columns: &mut RequiredColumns,
++) -> Option<Arc<dyn PhysicalExpr>> {
++ let bitmap = Arc::clone(lookup.pruning_bitmap()?);
++ let on_columns = lookup.children();
++ let [column_expr] = on_columns[..] else {
++ return None;
++ };
++ // The schema adapter often wraps the column in a same-type or widening cast
++ // (a nullable table column over a REQUIRED file column, or a narrower file
++ // type). `stat_column_expr` rewrites only the inner `Column`, keeping it.
++ let column = match column_expr.downcast_ref::<phys_expr::CastExpr>() {
++ Some(cast) => {
++ let column = cast.expr().downcast_ref::<phys_expr::Column>()?;
++ let from = schema.fields().get(column.index())?.data_type();
++ // 55.1.0's widening-cast helper predates unsigned-to-larger-signed
++ // support. These additional integer casts preserve every value,
++ // ordering, and null; keep other unrecognized casts conservative.
++ let widens_unsigned = matches!(
++ (from, cast.cast_type()),
++ (
++ DataType::UInt8,
++ DataType::Int16 | DataType::Int32 | DataType::Int64
++ ) | (DataType::UInt16, DataType::Int32 | DataType::Int64)
++ | (DataType::UInt32, DataType::Int64)
++ );
++ (cast.is_bigger_cast(from) || widens_unsigned).then_some(column)?
++ }
++ None => column_expr.downcast_ref::<phys_expr::Column>()?,
++ };
++ let field = schema.fields().get(column.index())?;
++ if field.name() != column.name() {
++ return None;
++ }
++ let (min, max, non_null) =
++ min_max_non_null_exprs(column, column_expr, field, schema, required_columns)?;
++ Some(Arc::new(phys_expr::BinaryExpr::new(
++ non_null,
++ Operator::And,
++ Arc::new(KeyRangeBitmapPruningExpr { min, max, bitmap }),
++ )))
++}
++
+ /// Rewrite a predicate expression in terms of statistics (min/max/null_counts)
+ /// for use as a [`PruningPredicate`].
+ pub struct PredicateRewriter {
+@@ -1621,6 +1707,44 @@
+ return unhandled_hook.handle(expr);
+ }
+ }
++ if let Some(lookup) = expr.downcast_ref::<HashTableLookupExpr>() {
++ return build_hash_lookup_pruning_expr(lookup, schema, required_columns)
++ .unwrap_or_else(|| unhandled_hook.handle(expr));
++ }
++ // A partitioned hash join hides its per-partition filters under a `CASE` on the
++ // repartition hash. A row takes exactly one branch, so a container may match
++ // only if some branch may: the branches' disjunction is a sound relaxation, and
++ // the `WHEN`s (a hash, which no statistics describe) can be dropped.
++ if let Some(case) = expr.downcast_ref::<phys_expr::CaseExpr>() {
++ // Only a Boolean `CASE` is a predicate; anything else is a value for
++ // whatever compares it to handle.
++ if !matches!(case.data_type(schema), Ok(DataType::Boolean)) {
++ return unhandled_hook.handle(expr);
++ }
++ // Without an `ELSE`, unmatched rows are UNKNOWN: not a match, but not
++ // FALSE either, which full-match inference must be able to tell apart.
++ if case.else_expr().is_none() {
++ return unhandled_hook.handle(expr);
++ }
++ return case
++ .when_then_expr()
++ .iter()
++ .map(|(_, then)| then)
++ .chain(case.else_expr())
++ .map(|branch| {
++ build_predicate_expression(
++ branch,
++ schema,
++ required_columns,
++ unhandled_hook,
++ max_in_list_size,
++ )
++ })
++ .reduce(|acc, branch| {
++ Arc::new(phys_expr::BinaryExpr::new(acc, Operator::Or, branch)) as _
++ })
++ .unwrap_or_else(|| unhandled_hook.handle(expr));
++ }
+
+ let (left, op, right) = {
+ if let Some(bin_expr) = expr.downcast_ref::<phys_expr::BinaryExpr>() {
+@@ -2197,7 +2321,11 @@
+ self as phys_expr, DynamicFilterPhysicalExpr,
+ };
+ use datafusion_physical_expr::planner::logical2physical;
++ use datafusion_physical_plan::joins::join_hash_map::JoinHashMapU32;
++ use datafusion_physical_plan::joins::key_range_bitmap::KeyRangeBitmap;
++ use datafusion_physical_plan::joins::{Map, SeededRandomState};
+ use itertools::Itertools;
++ use rstest::rstest;
+
+ #[derive(Debug, Default)]
+ /// Mock statistic provider for tests
+@@ -6061,4 +6189,236 @@
+ "c1_null_count@2 != row_count@3 AND c1_min@0 <= a AND a <= c1_max@1";
+ assert_eq!(res.to_string(), expected);
+ }
++
++ /// A hash-table lookup over `column` carrying a bitmap over `keys`.
++ fn hash_lookup(
++ column: &Arc<dyn PhysicalExpr>,
++ keys: &[i64],
++ ) -> Arc<dyn PhysicalExpr> {
++ let array: ArrayRef = Arc::new(Int64Array::from(keys.to_vec()));
++ let bitmap = KeyRangeBitmap::try_new(
++ &array,
++ &ScalarValue::Int64(Some(*keys.iter().min().unwrap())),
++ &ScalarValue::Int64(Some(*keys.iter().max().unwrap())),
++ keys.len(),
++ );
++ Arc::new(
++ HashTableLookupExpr::new(
++ vec![Arc::clone(column)],
++ SeededRandomState::with_seed(1),
++ Arc::new(Map::HashMap(Box::new(JoinHashMapU32::with_capacity(1)))),
++ "hash_lookup".to_string(),
++ )
++ .with_pruning_bitmap(bitmap.map(Arc::new)),
++ )
++ }
++
++ #[test]
++ fn test_hash_lookup_pruning_via_min_max() {
++ let schema = Arc::new(Schema::new(vec![Field::new("b", DataType::Int64, true)]));
++ let column: Arc<dyn PhysicalExpr> = Arc::new(phys_expr::Column::new("b", 0));
++ // Gapped keys: 10,20,30 then a far-away 100000, so the bitmap is sparse.
++ let lookup = hash_lookup(&column, &[10, 20, 30, 100_000]);
++
++ let predicate = PruningPredicateBuilder::new()
++ .with_file_schema(Arc::clone(&schema))
++ .try_build(lookup)
++ .unwrap();
++
++ // An opaque lookup yields no literal guarantees, so `contained()` is never
++ // consulted and every exclusion below comes from the min/max rewrite.
++ assert!(predicate.literal_guarantees().is_empty());
++
++ let statistics = TestStatistics::new().with(
++ "b",
++ ContainerStats::new_i64(
++ vec![Some(5000), Some(15), Some(50000)],
++ vec![Some(8000), Some(25), Some(60000)],
++ ),
++ );
++
++ let result = predicate.prune(&statistics).unwrap();
++ // Containers 0 and 2 sit in the gap between 30 and 100000; container 1
++ // holds 20. The bitmap excludes the first and third.
++ assert_eq!(result, vec![false, true, false]);
++ }
++
++ #[test]
++ fn test_hash_lookup_pruning_with_nulls_and_missing_statistics() {
++ let schema = Arc::new(Schema::new(vec![Field::new("b", DataType::Int64, true)]));
++ let column: Arc<dyn PhysicalExpr> = Arc::new(phys_expr::Column::new("b", 0));
++ let predicate = PruningPredicateBuilder::new()
++ .with_file_schema(schema)
++ .try_build(hash_lookup(&column, &[10, 20, 30, 100_000]))
++ .unwrap();
++ let statistics = TestStatistics::new().with(
++ "b",
++ ContainerStats::new_i64(
++ vec![Some(15), None, None, None, Some(5000), Some(5000)],
++ vec![Some(25), None, None, Some(25), None, Some(8000)],
++ )
++ .with_row_counts(vec![Some(2); 6])
++ .with_null_counts(vec![
++ Some(0),
++ Some(2),
++ Some(0),
++ Some(0),
++ Some(0),
++ None,
++ ]),
++ );
++
++ // A matching range is kept and an all-null group is rejected. Missing
++ // either bound cannot prove absence. A known gap still excludes a group
++ // when the null count is unknown: neither nulls nor non-nulls can match.
++ assert_eq!(
++ predicate.prune(&statistics).unwrap(),
++ vec![true, false, true, true, true, false]
++ );
++ }
++
++ #[test]
++ fn test_case_without_else_cannot_prove_full_match() {
++ let schema = Arc::new(Schema::new(vec![Field::new("b", DataType::Int32, false)]));
++ let condition = logical2physical(&col("b").eq(lit(1)), &schema);
++ let case: Arc<dyn PhysicalExpr> = Arc::new(
++ phys_expr::CaseExpr::try_new(
++ None,
++ vec![(condition, phys_expr::lit(false))],
++ None,
++ )
++ .unwrap(),
++ );
++ // NOT(CASE WHEN b = 1 THEN false END) yields NULL for b=0, even
++ // though the referenced column is non-nullable. Such a row must not be
++ // counted as a match when a Parquet scan tries to satisfy a LIMIT.
++ let predicate: Arc<dyn PhysicalExpr> = Arc::new(phys_expr::NotExpr::new(case));
++ let batch = RecordBatch::try_new(
++ Arc::clone(&schema),
++ vec![Arc::new(Int32Array::from(vec![0, 1]))],
++ )
++ .unwrap();
++ assert_eq!(
++ predicate
++ .evaluate(&batch)
++ .unwrap()
++ .into_array(2)
++ .unwrap()
++ .as_boolean(),
++ &BooleanArray::from(vec![None, Some(true)])
++ );
++
++ // Full-match inference negates the predicate, simplifies double NOT,
++ // and asks if the result prunes the group. Dropping the implicit NULL
++ // branch would produce false and incorrectly mark every row matched.
++ let inverse = PhysicalExprSimplifier::new(&schema)
++ .simplify(Arc::new(phys_expr::NotExpr::new(predicate)))
++ .unwrap();
++ let inverse = PruningPredicateBuilder::new()
++ .with_file_schema(schema)
++ .try_build(inverse)
++ .unwrap();
++ let statistics = TestStatistics::new().with(
++ "b",
++ ContainerStats::new_i32(vec![Some(0)], vec![Some(1)])
++ .with_row_counts(vec![Some(2)])
++ .with_null_counts(vec![Some(0)]),
++ );
++ assert_eq!(inverse.prune(&statistics).unwrap(), vec![true]);
++ }
++
++ #[rstest]
++ #[case::widening_i32_to_i64(DataType::Int32, DataType::Int64, true)]
++ #[case::widening_u32_to_i64(DataType::UInt32, DataType::Int64, true)]
++ #[case::widening_u16_to_i64(DataType::UInt16, DataType::Int64, true)]
++ #[case::same_type_i64_to_i64(DataType::Int64, DataType::Int64, true)]
++ #[case::narrowing_i64_to_i32(DataType::Int64, DataType::Int32, false)]
++ #[case::narrowing_u32_to_i16(DataType::UInt32, DataType::Int16, false)]
++ fn test_hash_lookup_pruning_through_cast(
++ #[case] file_type: DataType,
++ #[case] cast_type: DataType,
++ #[case] pruned: bool,
++ ) {
++ let schema =
++ Arc::new(Schema::new(vec![Field::new("b", file_type.clone(), true)]));
++ let column: Arc<dyn PhysicalExpr> = Arc::new(phys_expr::Column::new("b", 0));
++ let cast_column: Arc<dyn PhysicalExpr> =
++ Arc::new(phys_expr::CastExpr::new(column, cast_type, None));
++ let lookup = hash_lookup(&cast_column, &[10, 20, 30, 100_000]);
++
++ let predicate = PruningPredicateBuilder::new()
++ .with_file_schema(Arc::clone(&schema))
++ .try_build(lookup)
++ .unwrap();
++
++ let to_file_type = |values: Vec<Option<i64>>| -> ArrayRef {
++ let values: ArrayRef = Arc::new(values.into_iter().collect::<Int64Array>());
++ arrow::compute::cast(&values, &file_type).unwrap()
++ };
++ let statistics = TestStatistics::new().with(
++ "b",
++ ContainerStats::new()
++ .with_min(to_file_type(vec![Some(5000), Some(15), Some(50000)]))
++ .with_max(to_file_type(vec![Some(8000), Some(25), Some(60000)])),
++ );
++
++ let result = predicate.prune(&statistics).unwrap();
++ // Containers 0 and 2 sit in the gap between 30 and 100000; container 1
++ // holds 20. A working cast excludes the first and third; a declined one
++ // falls back to keeping every container.
++ let expected = if pruned {
++ vec![false, true, false]
++ } else {
++ vec![true, true, true]
++ };
++ assert_eq!(result, expected);
++ }
++
++ #[test]
++ fn test_partition_routed_hash_lookup_pruning() {
++ let schema = Arc::new(Schema::new(vec![Field::new("b", DataType::Int32, true)]));
++ let column: Arc<dyn PhysicalExpr> = Arc::new(phys_expr::Column::new("b", 0));
++ let literal = |value: ScalarValue| -> Arc<dyn PhysicalExpr> {
++ Arc::new(phys_expr::Literal::new(value))
++ };
++ // One branch per build partition, each holding its own slice of the build
++ // side, as `build_partitioned_filter` produces them.
++ let case: Arc<dyn PhysicalExpr> = Arc::new(
++ phys_expr::CaseExpr::try_new(
++ Some(literal(ScalarValue::UInt64(Some(0)))),
++ vec![
++ (
++ literal(ScalarValue::UInt64(Some(0))),
++ hash_lookup(&column, &[10, 20, 30]),
++ ),
++ (
++ literal(ScalarValue::UInt64(Some(1))),
++ hash_lookup(&column, &[40, 50, 60]),
++ ),
++ ],
++ // Partitions with no build rows reject everything routed to them.
++ Some(literal(ScalarValue::Boolean(Some(false)))),
++ )
++ .unwrap(),
++ );
++
++ let predicate = PruningPredicateBuilder::new()
++ .with_file_schema(Arc::clone(&schema))
++ .try_build(case)
++ .unwrap();
++
++ let statistics = TestStatistics::new().with(
++ "b",
++ ContainerStats::new_i32(
++ vec![Some(5), Some(15), Some(45), Some(100)],
++ vec![Some(8), Some(25), Some(55), Some(200)],
++ ),
++ );
++
++ let result = predicate.prune(&statistics).unwrap();
++ // Container 1 ([15,25]) holds 20 from the first branch and container 2
++ // ([45,55]) holds 50 from the second, so a branch may match in each. The
++ // other two intersect neither branch, nor the `ELSE`.
++ assert_eq!(result, vec![false, true, true, false]);
++ }
+ }
diff --git a/native/vendor/datafusion-physical-plan/Cargo.toml b/native/vendor/datafusion-physical-plan/Cargo.toml
new file mode 100644
index 0000000000..6cf15f78a3
--- /dev/null
+++ b/native/vendor/datafusion-physical-plan/Cargo.toml
@@ -0,0 +1,308 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
+#
+# When uploading crates to the registry Cargo will automatically
+# "normalize" Cargo.toml files for maximal compatibility
+# with all versions of Cargo and also rewrite `path` dependencies
+# to registry (e.g., crates.io) dependencies.
+#
+# If you are reading this file be aware that the original Cargo.toml
+# will likely look very different (and much more reasonable).
+# See Cargo.toml.orig for the original contents.
+
+[package]
+edition = "2024"
+rust-version = "1.94.0"
+name = "datafusion-physical-plan"
+version = "55.1.0"
+authors = ["Apache DataFusion <de...@datafusion.apache.org>"]
+build = false
+autolib = false
+autobins = false
+autoexamples = false
+autotests = false
+autobenches = false
+description = "Physical (ExecutionPlan) implementations for DataFusion query engine"
+homepage = "https://datafusion.apache.org"
+readme = "README.md"
+keywords = [
+ "arrow",
+ "query",
+ "sql",
+]
+license = "Apache-2.0"
+repository = "https://github.com/apache/datafusion"
+resolver = "2"
+
+[package.metadata.docs.rs]
+all-features = true
+
+[features]
+force_hash_collisions = []
+proto = [
+ "dep:datafusion-proto-models",
+ "dep:datafusion-proto-common",
+ "datafusion-physical-expr/proto",
+ "datafusion-physical-expr-common/proto",
+]
+test_utils = ["arrow/test_utils"]
+tokio_coop = []
+tokio_coop_fallback = []
+
+[lib]
+name = "datafusion_physical_plan"
+path = "src/lib.rs"
+
+[[bench]]
+name = "aggregate_vectorized"
+path = "benches/aggregate_vectorized.rs"
+harness = false
+required-features = ["test_utils"]
+
+[[bench]]
+name = "bounded_window"
+path = "benches/bounded_window.rs"
+harness = false
+
+[[bench]]
+name = "compute_statistics"
+path = "benches/compute_statistics.rs"
+harness = false
+
+[[bench]]
+name = "dictionary_group_values"
+path = "benches/dictionary_group_values.rs"
+harness = false
+
+[[bench]]
+name = "hash_join_semi_anti"
+path = "benches/hash_join_semi_anti.rs"
+harness = false
+required-features = ["test_utils"]
+
+[[bench]]
+name = "multi_group_by"
+path = "benches/multi_group_by.rs"
+harness = false
+required-features = ["test_utils"]
+
+[[bench]]
+name = "partial_ordering"
+path = "benches/partial_ordering.rs"
+harness = false
+
+[[bench]]
+name = "sort_merge_join"
+path = "benches/sort_merge_join.rs"
+harness = false
+required-features = ["test_utils"]
+
+[[bench]]
+name = "sort_preserving_merge"
+path = "benches/sort_preserving_merge.rs"
+harness = false
+
+[[bench]]
+name = "spill_io"
+path = "benches/spill_io.rs"
+harness = false
+
+[dependencies.arrow]
+version = "59.2.0"
+features = [
+ "prettyprint",
+ "chrono-tz",
+]
+
+[dependencies.arrow-data]
+version = "59.2.0"
+default-features = false
+
+[dependencies.arrow-ipc]
+version = "59.2.0"
+features = [
+ "lz4",
+ "zstd",
+ "lz4",
+ "zstd",
+]
+default-features = false
+
+[dependencies.arrow-ord]
+version = "59.2.0"
+default-features = false
+
+[dependencies.arrow-schema]
+version = "59.2.0"
+default-features = false
+
+[dependencies.async-trait]
+version = "0.1.89"
+
+[dependencies.bytes]
+version = "1.11"
+
+[dependencies.datafusion-common]
+version = "55.1.0"
+default-features = false
+
+[dependencies.datafusion-common-runtime]
+version = "55.1.0"
+
+[dependencies.datafusion-execution]
+version = "55.1.0"
+default-features = false
+
+[dependencies.datafusion-expr]
+version = "55.1.0"
+default-features = false
+
+[dependencies.datafusion-functions]
+version = "55.1.0"
+
+[dependencies.datafusion-functions-aggregate-common]
+version = "55.1.0"
+
+[dependencies.datafusion-functions-window-common]
+version = "55.1.0"
+
+[dependencies.datafusion-physical-expr]
+version = "55.1.0"
+default-features = true
+
+[dependencies.datafusion-physical-expr-common]
+version = "55.1.0"
+default-features = false
+
+[dependencies.datafusion-proto-common]
+version = "55.1.0"
+optional = true
+
+[dependencies.datafusion-proto-models]
+version = "55.1.0"
+optional = true
+
+[dependencies.futures]
+version = "0.3"
+
+[dependencies.half]
+version = "2.7.0"
+default-features = false
+
+[dependencies.hashbrown]
+version = "0.17.1"
+
+[dependencies.indexmap]
+version = "2.14.0"
+
+[dependencies.itertools]
+version = "0.15"
+features = ["use_std"]
+
+[dependencies.log]
+version = "^0.4"
+
+[dependencies.num-traits]
+version = "0.2"
+
+[dependencies.parking_lot]
+version = "0.12"
+
+[dependencies.pin-project-lite]
+version = "^0.2.7"
+
+[dependencies.serde_json]
+version = "1"
+features = ["preserve_order"]
+
+[dependencies.tokio]
+version = "1.52"
+features = [
+ "macros",
+ "rt",
+ "sync",
+]
+
+[dev-dependencies.arrow-data]
+version = "59.2.0"
+default-features = false
+
+[dev-dependencies.criterion]
+version = "0.8"
+features = ["async_futures"]
+
+[dev-dependencies.datafusion-functions-aggregate]
+version = "55.1.0"
+
+[dev-dependencies.datafusion-functions-window]
+version = "55.1.0"
+
+[dev-dependencies.insta]
+version = "1.47.2"
+features = [
+ "glob",
+ "filters",
+]
+
+[dev-dependencies.rand]
+version = "0.9"
+
+[dev-dependencies.rstest]
+version = "0.26.1"
+
+[dev-dependencies.rstest_reuse]
+version = "0.7.0"
+
+[dev-dependencies.tokio]
+version = "1.52"
+features = [
+ "macros",
+ "rt",
+ "sync",
+ "rt-multi-thread",
+ "fs",
+ "parking_lot",
+]
+
+[lints.clippy]
+allow_attributes = "warn"
+assigning_clones = "warn"
+inefficient_to_string = "warn"
+large_futures = "warn"
+needless_pass_by_value = "warn"
+or_fun_call = "warn"
+uninlined_format_args = "warn"
+unnecessary_lazy_evaluations = "warn"
+unused_async = "warn"
+used_underscore_binding = "warn"
+
+[lints.rust]
+# Rust 1.99 renamed AtomicUsize::fetch_update. Retain the upstream implementation
+# and its Rust 1.94 minimum instead of requiring the new try_update method.
+deprecated = "allow"
+unused_qualifications = "deny"
+
+[lints.rust.unexpected_cfgs]
+level = "warn"
+priority = 0
+check-cfg = [
+ 'cfg(datafusion_coop, values("tokio", "tokio_fallback", "per_stream"))',
+ "cfg(coverage)",
+ "cfg(coverage_nightly)",
+]
diff --git a/native/vendor/datafusion-physical-plan/Cargo.toml.orig b/native/vendor/datafusion-physical-plan/Cargo.toml.orig
new file mode 100644
index 0000000000..0f72b74840
--- /dev/null
+++ b/native/vendor/datafusion-physical-plan/Cargo.toml.orig
@@ -0,0 +1,149 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+name = "datafusion-physical-plan"
+description = "Physical (ExecutionPlan) implementations for DataFusion query engine"
+keywords = ["arrow", "query", "sql"]
+readme = "README.md"
+version = { workspace = true }
+edition = { workspace = true }
+homepage = { workspace = true }
+repository = { workspace = true }
+license = { workspace = true }
+authors = { workspace = true }
+rust-version = { workspace = true }
+
+[package.metadata.docs.rs]
+all-features = true
+
+# Note: add additional linter rules in lib.rs.
+# Rust does not support workspace + new linter rules in subcrates yet
+# https://github.com/rust-lang/cargo/issues/13157
+[lints]
+workspace = true
+
+[features]
+force_hash_collisions = []
+test_utils = ["arrow/test_utils"]
+tokio_coop = []
+tokio_coop_fallback = []
+# Enables `PhysicalExpr::try_to_proto` / `try_from_proto` hooks on the
+# physical expressions defined in this crate (e.g. `HashExpr`). Off by
+# default so consumers that never serialize plans pay nothing.
+proto = [
+ "dep:datafusion-proto-models",
+ "dep:datafusion-proto-common",
+ "datafusion-physical-expr/proto",
+ "datafusion-physical-expr-common/proto",
+]
+
+[lib]
+name = "datafusion_physical_plan"
+
+[dependencies]
+arrow = { workspace = true }
+arrow-data = { workspace = true }
+# Spill IPC writes require lz4 and zstd codec support. Keep these features in
+# sync with the SpillCompression variants in datafusion-common so codec
+# availability is explicit in the crate that owns spill handling.
+arrow-ipc = { workspace = true, features = ["lz4", "zstd"] }
+arrow-ord = { workspace = true }
+arrow-schema = { workspace = true }
+async-trait = { workspace = true }
+bytes = { workspace = true }
+datafusion-common = { workspace = true }
+datafusion-common-runtime = { workspace = true, default-features = true }
+datafusion-execution = { workspace = true }
+datafusion-expr = { workspace = true }
+datafusion-functions = { workspace = true }
+datafusion-functions-aggregate-common = { workspace = true }
+datafusion-functions-window-common = { workspace = true }
+datafusion-physical-expr = { workspace = true, default-features = true }
+datafusion-physical-expr-common = { workspace = true }
+datafusion-proto-common = { workspace = true, optional = true }
+datafusion-proto-models = { workspace = true, optional = true }
+futures = { workspace = true }
+half = { workspace = true }
+hashbrown = { workspace = true }
+indexmap = { workspace = true }
+itertools = { workspace = true, features = ["use_std"] }
+log = { workspace = true }
+num-traits = { workspace = true }
+parking_lot = { workspace = true }
+pin-project-lite = { workspace = true }
+serde_json = { workspace = true, features = ["preserve_order"] }
+tokio = { workspace = true }
+
+[dev-dependencies]
+arrow-data = { workspace = true }
+criterion = { workspace = true, features = ["async_futures"] }
+datafusion-functions-aggregate = { workspace = true }
+datafusion-functions-window = { workspace = true }
+insta = { workspace = true }
+rand = { workspace = true }
+rstest = { workspace = true }
+rstest_reuse = "0.7.0"
+tokio = { workspace = true, features = [
+ "rt-multi-thread",
+ "fs",
+ "parking_lot",
+] }
+
+[[bench]]
+harness = false
+name = "partial_ordering"
+
+[[bench]]
+harness = false
+name = "spill_io"
+
+[[bench]]
+harness = false
+name = "sort_preserving_merge"
+
+[[bench]]
+harness = false
+name = "sort_merge_join"
+required-features = ["test_utils"]
+
+[[bench]]
+harness = false
+name = "aggregate_vectorized"
+required-features = ["test_utils"]
+
+[[bench]]
+harness = false
+name = "compute_statistics"
+
+[[bench]]
+harness = false
+name = "dictionary_group_values"
+
+[[bench]]
+harness = false
+name = "hash_join_semi_anti"
+required-features = ["test_utils"]
+
+[[bench]]
+harness = false
+name = "multi_group_by"
+required-features = ["test_utils"]
+
+[[bench]]
+harness = false
+name = "bounded_window"
diff --git a/native/vendor/datafusion-physical-plan/LICENSE.txt b/native/vendor/datafusion-physical-plan/LICENSE.txt
new file mode 100644
index 0000000000..d74c6b599d
--- /dev/null
+++ b/native/vendor/datafusion-physical-plan/LICENSE.txt
@@ -0,0 +1,212 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+
+This project includes code from Apache Aurora.
+
+* dev/release/{release,changelog,release-candidate} are based on the scripts from
+ Apache Aurora
+
+Copyright: 2016 The Apache Software Foundation.
+Home page: https://aurora.apache.org/
+License: http://www.apache.org/licenses/LICENSE-2.0
diff --git a/native/vendor/datafusion-physical-plan/NOTICE.txt b/native/vendor/datafusion-physical-plan/NOTICE.txt
new file mode 100644
index 0000000000..0bd2d52368
--- /dev/null
+++ b/native/vendor/datafusion-physical-plan/NOTICE.txt
@@ -0,0 +1,5 @@
+Apache DataFusion
+Copyright 2019-2026 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
diff --git a/native/vendor/datafusion-physical-plan/README.md b/native/vendor/datafusion-physical-plan/README.md
new file mode 100644
index 0000000000..3a33100f2f
--- /dev/null
+++ b/native/vendor/datafusion-physical-plan/README.md
@@ -0,0 +1,33 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+# Apache DataFusion Physical Plan
+
+[Apache DataFusion] is an extensible query execution framework, written in Rust, that uses [Apache Arrow] as its in-memory format.
+
+This crate is a submodule of DataFusion that contains the `ExecutionPlan` trait and the various implementations of that
+trait for built in operators such as filters, projections, joins, aggregations, etc.
+
+Most projects should use the [`datafusion`] crate directly, which re-exports
+this module. If you are already using the [`datafusion`] crate, there is no
+reason to use this crate directly in your project as well.
+
+[apache arrow]: https://arrow.apache.org/
+[apache datafusion]: https://datafusion.apache.org/
+[`datafusion`]: https://crates.io/crates/datafusion
diff --git a/native/vendor/datafusion-physical-plan/benches/aggregate_vectorized.rs b/native/vendor/datafusion-physical-plan/benches/aggregate_vectorized.rs
new file mode 100644
index 0000000000..488647d5f8
--- /dev/null
+++ b/native/vendor/datafusion-physical-plan/benches/aggregate_vectorized.rs
@@ -0,0 +1,309 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use arrow::array::{ArrayRef, BooleanBufferBuilder};
+use arrow::datatypes::{Int32Type, StringViewType};
+use arrow::util::bench_util::{
+ create_primitive_array, create_string_view_array_with_len,
+ create_string_view_array_with_max_len,
+};
+use arrow_schema::DataType;
+use criterion::measurement::WallTime;
+use criterion::{
+ BenchmarkGroup, BenchmarkId, Criterion, criterion_group, criterion_main,
+};
+use datafusion_physical_plan::aggregates::group_values::multi_group_by::GroupColumn;
+use datafusion_physical_plan::aggregates::group_values::multi_group_by::bytes_view::ByteViewGroupValueBuilder;
+use datafusion_physical_plan::aggregates::group_values::multi_group_by::primitive::PrimitiveGroupValueBuilder;
+use rand::SeedableRng;
+use rand::distr::{Bernoulli, Distribution};
+use rand::rngs::StdRng;
+use std::hint::black_box;
+use std::sync::Arc;
+
+const SIZES: [usize; 3] = [1_000, 10_000, 100_000];
+const NULL_DENSITIES: [f32; 3] = [0.0, 0.1, 0.5];
+
+fn bench_vectorized_append(c: &mut Criterion) {
+ byte_view_vectorized_append(c);
+ primitive_vectorized_append(c);
+}
+
+fn byte_view_vectorized_append(c: &mut Criterion) {
+ let mut group = c.benchmark_group("ByteViewGroupValueBuilder_vectorized_append");
+
+ for &size in &SIZES {
+ let rows: Vec<usize> = (0..size).collect();
+
+ for &null_density in &NULL_DENSITIES {
+ let input = create_string_view_array_with_len(size, null_density, 8, false);
+ let input: ArrayRef = Arc::new(input);
+
+ bytes_bench(&mut group, "inline", size, &rows, null_density, &input);
+ }
+ }
+
+ for &size in &SIZES {
+ let rows: Vec<usize> = (0..size).collect();
+
+ for &null_density in &NULL_DENSITIES {
+ let input = create_string_view_array_with_len(size, null_density, 64, true);
+ let input: ArrayRef = Arc::new(input);
+
+ bytes_bench(&mut group, "scenario", size, &rows, null_density, &input);
+ }
+ }
+
+ for &size in &SIZES {
+ let rows: Vec<usize> = (0..size).collect();
+
+ for &null_density in &NULL_DENSITIES {
+ let input = create_string_view_array_with_max_len(size, null_density, 400);
+ let input: ArrayRef = Arc::new(input);
+
+ bytes_bench(&mut group, "random", size, &rows, null_density, &input);
+ }
+ }
+
+ group.finish();
+}
+
+fn bytes_bench(
+ group: &mut BenchmarkGroup<WallTime>,
+ bench_prefix: &str,
+ size: usize,
+ rows: &Vec<usize>,
+ null_density: f32,
+ input: &ArrayRef,
+) {
+ // vectorized_append
+ let function_name = format!("{bench_prefix}_null_{null_density:.1}_size_{size}");
+ let id = BenchmarkId::new(&function_name, "vectorized_append");
+ group.bench_function(id, |b| {
+ b.iter(|| {
+ let mut builder = ByteViewGroupValueBuilder::<StringViewType>::new();
+ builder.vectorized_append(input, rows).unwrap();
+ });
+ });
+
+ // append_val
+ let id = BenchmarkId::new(&function_name, "append_val");
+ group.bench_function(id, |b| {
+ b.iter(|| {
+ let mut builder = ByteViewGroupValueBuilder::<StringViewType>::new();
+ for &i in rows {
+ builder.append_val(input, i).unwrap();
+ }
+ });
+ });
+
+ // vectorized_equal_to
+ vectorized_equal_to(
+ group,
+ ByteViewGroupValueBuilder::<StringViewType>::new(),
+ &function_name,
+ rows,
+ input,
+ "all_true",
+ vec![true; size],
+ );
+ vectorized_equal_to(
+ group,
+ ByteViewGroupValueBuilder::<StringViewType>::new(),
+ &function_name,
+ rows,
+ input,
+ "0.75 true",
+ {
+ let mut rng = StdRng::seed_from_u64(42);
+ let d = Bernoulli::new(0.75).unwrap();
+ (0..size).map(|_| d.sample(&mut rng)).collect::<Vec<_>>()
+ },
+ );
+ vectorized_equal_to(
+ group,
+ ByteViewGroupValueBuilder::<StringViewType>::new(),
+ &function_name,
+ rows,
+ input,
+ "0.5 true",
+ {
+ let mut rng = StdRng::seed_from_u64(42);
+ let d = Bernoulli::new(0.5).unwrap();
+ (0..size).map(|_| d.sample(&mut rng)).collect::<Vec<_>>()
+ },
+ );
+ vectorized_equal_to(
+ group,
+ ByteViewGroupValueBuilder::<StringViewType>::new(),
+ &function_name,
+ rows,
+ input,
+ "0.25 true",
+ {
+ let mut rng = StdRng::seed_from_u64(42);
+ let d = Bernoulli::new(0.25).unwrap();
+ (0..size).map(|_| d.sample(&mut rng)).collect::<Vec<_>>()
+ },
+ );
+ // Not adding 0 true case here as if we optimize for 0 true cases the caller should avoid calling this method at all
+}
+
+fn primitive_vectorized_append(c: &mut Criterion) {
+ let mut group = c.benchmark_group("PrimitiveGroupValueBuilder_vectorized_append");
+
+ for &size in &SIZES {
+ let rows: Vec<usize> = (0..size).collect();
+
+ for &null_density in &NULL_DENSITIES {
+ if null_density == 0.0 {
+ bench_single_primitive::<false>(&mut group, size, &rows, null_density)
+ }
+ bench_single_primitive::<true>(&mut group, size, &rows, null_density);
+ }
+ }
+
+ group.finish();
+}
+
+fn bench_single_primitive<const NULLABLE: bool>(
+ group: &mut BenchmarkGroup<WallTime>,
+ size: usize,
+ rows: &Vec<usize>,
+ null_density: f32,
+) {
+ if !NULLABLE {
+ assert_eq!(
+ null_density, 0.0,
+ "non-nullable case must have null_density 0"
+ );
+ }
+
+ let input = create_primitive_array::<Int32Type>(size, null_density);
+ let input: ArrayRef = Arc::new(input);
+ let function_name = format!("null_{null_density:.1}_nullable_{NULLABLE}_size_{size}");
+
+ // vectorized_append
+ let id = BenchmarkId::new(&function_name, "vectorized_append");
+ group.bench_function(id, |b| {
+ b.iter(|| {
+ let mut builder =
+ PrimitiveGroupValueBuilder::<Int32Type, NULLABLE>::new(DataType::Int32);
+ builder.vectorized_append(&input, rows).unwrap();
+ });
+ });
+
+ // append_val
+ let id = BenchmarkId::new(&function_name, "append_val");
+ group.bench_function(id, |b| {
+ b.iter(|| {
+ let mut builder =
+ PrimitiveGroupValueBuilder::<Int32Type, NULLABLE>::new(DataType::Int32);
+ for &i in rows {
+ builder.append_val(&input, i).unwrap();
+ }
+ });
+ });
+
+ // vectorized_equal_to
+ vectorized_equal_to(
+ group,
+ PrimitiveGroupValueBuilder::<Int32Type, NULLABLE>::new(DataType::Int32),
+ &function_name,
+ rows,
+ &input,
+ "all_true",
+ vec![true; size],
+ );
+ vectorized_equal_to(
+ group,
+ PrimitiveGroupValueBuilder::<Int32Type, NULLABLE>::new(DataType::Int32),
+ &function_name,
+ rows,
+ &input,
+ "0.75 true",
+ {
+ let mut rng = StdRng::seed_from_u64(42);
+ let d = Bernoulli::new(0.75).unwrap();
+ (0..size).map(|_| d.sample(&mut rng)).collect::<Vec<_>>()
+ },
+ );
+ vectorized_equal_to(
+ group,
+ PrimitiveGroupValueBuilder::<Int32Type, NULLABLE>::new(DataType::Int32),
+ &function_name,
+ rows,
+ &input,
+ "0.5 true",
+ {
+ let mut rng = StdRng::seed_from_u64(42);
+ let d = Bernoulli::new(0.5).unwrap();
+ (0..size).map(|_| d.sample(&mut rng)).collect::<Vec<_>>()
+ },
+ );
+ vectorized_equal_to(
+ group,
+ PrimitiveGroupValueBuilder::<Int32Type, NULLABLE>::new(DataType::Int32),
+ &function_name,
+ rows,
+ &input,
+ "0.25 true",
+ {
+ let mut rng = StdRng::seed_from_u64(42);
+ let d = Bernoulli::new(0.25).unwrap();
+ (0..size).map(|_| d.sample(&mut rng)).collect::<Vec<_>>()
+ },
+ );
+ // Not adding 0 true case here as if we optimize for 0 true cases the caller should avoid calling this method at all
+}
+
+/// Test `vectorized_equal_to` with different number of true in the initial results
+#[expect(clippy::needless_pass_by_value)]
+fn vectorized_equal_to<GroupColumnBuilder: GroupColumn>(
+ group: &mut BenchmarkGroup<WallTime>,
+ mut builder: GroupColumnBuilder,
+ function_name: &str,
+ rows: &[usize],
+ input: &ArrayRef,
+ equal_to_result_description: &str,
+ equal_to_results: Vec<bool>,
+) {
+ let id = BenchmarkId::new(
+ function_name,
+ format!("vectorized_equal_to_{equal_to_result_description}"),
+ );
+ group.bench_function(id, |b| {
+ builder.vectorized_append(input, rows).unwrap();
+
+ b.iter(|| {
+ // Rebuild the buffer each iteration as `vectorized_equal_to` mutates
+ // it, and without a fresh buffer all iterations after the first one
+ // would not be meaningful.
+ let mut equal_to_buffer = BooleanBufferBuilder::new(equal_to_results.len());
+ for &v in &equal_to_results {
+ equal_to_buffer.append(v);
+ }
+ builder.vectorized_equal_to(rows, input, rows, &mut equal_to_buffer);
+
+ // Make sure that the compiler does not optimize away the call
+ black_box(equal_to_buffer);
+ });
+ });
+}
+
+criterion_group!(benches, bench_vectorized_append);
+criterion_main!(benches);
diff --git a/native/vendor/datafusion-physical-plan/benches/bounded_window.rs b/native/vendor/datafusion-physical-plan/benches/bounded_window.rs
new file mode 100644
index 0000000000..56e195afbd
--- /dev/null
+++ b/native/vendor/datafusion-physical-plan/benches/bounded_window.rs
@@ -0,0 +1,280 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Benchmarks for `BoundedWindowAggExec` with many partitions.
+//!
+//! The streaming window operator keeps per-partition state keyed by
+//! `PartitionKey` (`Vec<ScalarValue>`) and, in `Linear` mode (input sorted
+//! by the ORDER BY column but not by the partition columns), visits every
+//! live partition on every batch while never retiring partitions until the
+//! input is exhausted. The cases here stress that path in different ways:
+//!
+//! - `linear N partitions`: dense round-robin keys -- every partition
+//! receives rows in every batch, so per-visit fixed costs dominate.
+//! - `linear sparse N partitions`: keys are clustered in time, so each
+//! batch touches only a small, fresh subset of keys while the set of live
+//! partitions keeps growing -- per-batch work on quiet partitions
+//! dominates.
+//! - `linear rows N partitions`: the dense layout with a ROWS frame, whose
+//! results can only be finalized as more rows of the same partition
+//! arrive.
+//! - `linear multi N partitions`: two window expressions over the dense
+//! layout, doubling the per-partition evaluation sweeps.
+//! - `sorted N partitions`: control; input sorted by partition key, so
+//! finished partitions are pruned eagerly and the state maps stay small.
+
+use std::sync::Arc;
+
+use arrow::array::UInt64Array;
+use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
+use arrow::record_batch::RecordBatch;
+use criterion::{Criterion, criterion_group, criterion_main};
+use datafusion_common::ScalarValue;
+use datafusion_execution::TaskContext;
+use datafusion_expr::{
+ WindowFrame, WindowFrameBound, WindowFrameUnits, WindowFunctionDefinition,
+};
+use datafusion_functions_aggregate::count::count_udaf;
+use datafusion_functions_aggregate::sum::sum_udaf;
+use datafusion_physical_expr::expressions::col;
+use datafusion_physical_expr::{LexOrdering, PhysicalSortExpr};
+use datafusion_physical_plan::test::TestMemoryExec;
+use datafusion_physical_plan::windows::{BoundedWindowAggExec, create_window_expr};
+use datafusion_physical_plan::{ExecutionPlan, InputOrderMode, collect};
+
+const BATCH_SIZE: usize = 8192;
+const N_BATCHES: usize = 16;
+/// Distinct partition keys per batch in the sparse layout. Each batch
+/// introduces this many previously-unseen keys, so the total partition count
+/// is `N_BATCHES * SPARSE_KEYS_PER_BATCH`.
+const SPARSE_KEYS_PER_BATCH: usize = 2048;
+
+fn schema() -> SchemaRef {
+ Arc::new(Schema::new(vec![
+ Field::new("pk", DataType::UInt64, false),
+ Field::new("ts", DataType::UInt64, false),
+ ]))
+}
+
+/// Batches with `ts` ascending across the whole input and partition keys
+/// chosen by `pk_of_row`.
+fn make_batches(pk_of_row: impl Fn(usize) -> u64) -> Vec<RecordBatch> {
+ (0..N_BATCHES)
+ .map(|b| {
+ let start = b * BATCH_SIZE;
+ let pk: UInt64Array = (start..start + BATCH_SIZE)
+ .map(|i| Some(pk_of_row(i)))
+ .collect();
+ let ts: UInt64Array = (start..start + BATCH_SIZE)
+ .map(|i| Some(i as u64))
+ .collect();
+ RecordBatch::try_new(schema(), vec![Arc::new(pk), Arc::new(ts)]).unwrap()
+ })
+ .collect()
+}
+
+/// Round-robin over `n_partitions`: every partition receives rows in every
+/// batch (when `n_partitions <= BATCH_SIZE`).
+fn dense_batches(n_partitions: usize) -> Vec<RecordBatch> {
+ make_batches(move |i| (i % n_partitions) as u64)
+}
+
+/// Keys clustered in time: batch `b` only contains keys in
+/// `[b * SPARSE_KEYS_PER_BATCH, (b + 1) * SPARSE_KEYS_PER_BATCH)`, cycled so
+/// that consecutive rows belong to different partitions. Previously-seen
+/// keys never recur, but `Linear` mode cannot know that, so the live
+/// partition set grows for the whole run.
+fn sparse_batches() -> Vec<RecordBatch> {
+ make_batches(|i| {
+ ((i / BATCH_SIZE) * SPARSE_KEYS_PER_BATCH + (i % SPARSE_KEYS_PER_BATCH)) as u64
+ })
+}
+
+/// Input laid out partition-by-partition (the `Sorted` layout).
+fn sorted_batches(n_partitions: usize) -> Vec<RecordBatch> {
+ let rows_per_partition = BATCH_SIZE * N_BATCHES / n_partitions;
+ make_batches(move |i| (i / rows_per_partition) as u64)
+}
+
+fn sort_expr(name: &str) -> PhysicalSortExpr {
+ PhysicalSortExpr {
+ expr: col(name, &schema()).unwrap(),
+ options: Default::default(),
+ }
+}
+
+/// `RANGE BETWEEN CURRENT ROW AND 10 FOLLOWING`
+fn range_frame() -> WindowFrame {
+ WindowFrame::new_bounds(
+ WindowFrameUnits::Range,
+ WindowFrameBound::CurrentRow,
+ WindowFrameBound::Following(ScalarValue::UInt64(Some(10))),
+ )
+}
+
+/// `ROWS BETWEEN CURRENT ROW AND 2 FOLLOWING`
+fn rows_frame() -> WindowFrame {
+ WindowFrame::new_bounds(
+ WindowFrameUnits::Rows,
+ WindowFrameBound::CurrentRow,
+ WindowFrameBound::Following(ScalarValue::UInt64(Some(2))),
+ )
+}
+
+/// `<agg>(ts) OVER (PARTITION BY pk ORDER BY ts <window_frame>)` for each
+/// aggregate in `aggregates`.
+fn window_exec(
+ batches: Vec<RecordBatch>,
+ mode: InputOrderMode,
+ input_ordering: Vec<PhysicalSortExpr>,
+ window_frame: &WindowFrame,
+ aggregates: &[(WindowFunctionDefinition, &str)],
+) -> Arc<dyn ExecutionPlan> {
+ let schema = schema();
+ let source = TestMemoryExec::try_new(&[batches], Arc::clone(&schema), None)
+ .expect("memory exec")
+ .try_with_sort_information(LexOrdering::new(input_ordering).into_iter().collect())
+ .expect("sort information");
+ let input = Arc::new(TestMemoryExec::update_cache(&Arc::new(source)));
+ let args = vec![col("ts", &schema).unwrap()];
+ let partitionby_exprs = vec![col("pk", &schema).unwrap()];
+ let orderby_exprs = vec![PhysicalSortExpr {
+ expr: col("ts", &schema).unwrap(),
+ options: Default::default(),
+ }];
+ let window_expr = aggregates
+ .iter()
+ .map(|(fun, name)| {
+ create_window_expr(
+ fun,
+ name.to_string(),
+ &args,
+ &partitionby_exprs,
+ &orderby_exprs,
+ Arc::new(window_frame.clone()),
+ input.schema(),
+ false,
+ false,
+ None,
+ )
+ .expect("window expr")
+ })
+ .collect::<Vec<_>>();
+ Arc::new(
+ BoundedWindowAggExec::try_new(window_expr, input, mode, true)
+ .expect("bounded window exec"),
+ )
+}
+
+fn count() -> (WindowFunctionDefinition, &'static str) {
+ (
+ WindowFunctionDefinition::AggregateUDF(count_udaf()),
+ "count",
+ )
+}
+
+fn sum() -> (WindowFunctionDefinition, &'static str) {
+ (WindowFunctionDefinition::AggregateUDF(sum_udaf()), "sum")
+}
+
+fn bounded_window_benchmark(c: &mut Criterion) {
+ let rt = tokio::runtime::Runtime::new().unwrap();
+ let mut group = c.benchmark_group("bounded_window_partitions");
+ group.sample_size(10);
+
+ let mut run_case = |name: String, plan: Arc<dyn ExecutionPlan>| {
+ group.bench_function(name, |b| {
+ b.iter(|| {
+ let task_ctx = Arc::new(TaskContext::default());
+ let batches = rt
+ .block_on(collect(Arc::clone(&plan), task_ctx))
+ .expect("execution");
+ assert_eq!(
+ batches.iter().map(|b| b.num_rows()).sum::<usize>(),
+ BATCH_SIZE * N_BATCHES
+ );
+ })
+ });
+ };
+
+ for n_partitions in [100, 10_000] {
+ run_case(
+ format!("linear {n_partitions} partitions"),
+ window_exec(
+ dense_batches(n_partitions),
+ InputOrderMode::Linear,
+ vec![sort_expr("ts")],
+ &range_frame(),
+ &[count()],
+ ),
+ );
+ }
+
+ run_case(
+ format!(
+ "linear sparse {} partitions",
+ N_BATCHES * SPARSE_KEYS_PER_BATCH
+ ),
+ window_exec(
+ sparse_batches(),
+ InputOrderMode::Linear,
+ vec![sort_expr("ts")],
+ &range_frame(),
+ &[count()],
+ ),
+ );
+
+ run_case(
+ "linear rows 10000 partitions".to_string(),
+ window_exec(
+ dense_batches(10_000),
+ InputOrderMode::Linear,
+ vec![sort_expr("ts")],
+ &rows_frame(),
+ &[count()],
+ ),
+ );
+
+ run_case(
+ "linear multi 10000 partitions".to_string(),
+ window_exec(
+ dense_batches(10_000),
+ InputOrderMode::Linear,
+ vec![sort_expr("ts")],
+ &range_frame(),
+ &[count(), sum()],
+ ),
+ );
+
+ // Control: the same query over partition-sorted input, where finished
+ // partitions are pruned eagerly and the state maps stay small.
+ run_case(
+ "sorted 10000 partitions".to_string(),
+ window_exec(
+ sorted_batches(10_000),
+ InputOrderMode::Sorted,
+ vec![sort_expr("pk"), sort_expr("ts")],
+ &range_frame(),
+ &[count()],
+ ),
+ );
+
+ group.finish();
+}
+
+criterion_group!(benches, bounded_window_benchmark);
+criterion_main!(benches);
diff --git a/native/vendor/datafusion-physical-plan/benches/compute_statistics.rs b/native/vendor/datafusion-physical-plan/benches/compute_statistics.rs
new file mode 100644
index 0000000000..cddf4c2396
--- /dev/null
+++ b/native/vendor/datafusion-physical-plan/benches/compute_statistics.rs
@@ -0,0 +1,354 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Benchmarks for `compute_statistics` with `StatsCache`.
+//!
+//! Demonstrates that caching eliminates redundant subtree walks in plans
+//! containing partition-merging operators (CoalescePartitionsExec) and
+//! binary join trees (CrossJoinExec).
+//!
+//! The plan shapes here mirror the reproducers from the planning-speed
+//! EPIC (<https://github.com/apache/datafusion/issues/19795>):
+//! - Coalesce chain: deep linear plans (e.g. deeply nested subqueries)
+//! - Cross-join tree: balanced binary trees from multi-way joins
+//! (mirrors the `physical_many_self_joins` sql_planner benchmark)
+
+use std::fmt;
+use std::sync::Arc;
+
+use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
+use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
+use datafusion_common::ScalarValue;
+use datafusion_common::tree_node::TreeNodeRecursion;
+use datafusion_common::{Result, Statistics};
+use datafusion_execution::TaskContext;
+use datafusion_physical_expr::EquivalenceProperties;
+use datafusion_physical_expr::PhysicalExpr;
+use datafusion_physical_expr::expressions::Literal;
+use datafusion_physical_plan::coalesce_partitions::CoalescePartitionsExec;
+use datafusion_physical_plan::execution_plan::{
+ Boundedness, EmissionType, ExecutionPlan, PlanProperties,
+};
+use datafusion_physical_plan::filter::FilterExec;
+use datafusion_physical_plan::joins::CrossJoinExec;
+use datafusion_physical_plan::statistics::StatisticsArgs;
+use datafusion_physical_plan::{
+ ChildrenPropertiesMode, DisplayAs, DisplayFormatType, Partitioning,
+ ReplaceChildrenOptions, SendableRecordBatchStream, StatisticsContext,
+};
+
+/// Minimal leaf node for benchmarking
+#[derive(Debug)]
+struct BenchLeaf {
+ schema: SchemaRef,
+ cache: Arc<PlanProperties>,
+}
+
+impl BenchLeaf {
+ fn new(col_name: &str) -> Self {
+ let schema = Arc::new(Schema::new(vec![Field::new(
+ col_name,
+ DataType::Int32,
+ false,
+ )]));
+ let cache = Arc::new(PlanProperties::new(
+ EquivalenceProperties::new(Arc::clone(&schema)),
+ Partitioning::UnknownPartitioning(2),
+ EmissionType::Incremental,
+ Boundedness::Bounded,
+ ));
+ Self { schema, cache }
+ }
+}
+
+impl DisplayAs for BenchLeaf {
+ fn fmt_as(&self, _t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result {
+ write!(f, "BenchLeaf")
+ }
+}
+
+impl ExecutionPlan for BenchLeaf {
+ fn name(&self) -> &str {
+ "BenchLeaf"
+ }
+
+ fn schema(&self) -> SchemaRef {
+ Arc::clone(&self.schema)
+ }
+
+ fn properties(&self) -> &Arc<PlanProperties> {
+ &self.cache
+ }
+
+ fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
+ vec![]
+ }
+
+ fn replace_children(
+ self: Arc<Self>,
+ _: Vec<Arc<dyn ExecutionPlan>>,
+ _: ReplaceChildrenOptions,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ Ok(self)
+ }
+
+ fn with_new_children(
+ self: Arc<Self>,
+ children: Vec<Arc<dyn ExecutionPlan>>,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ self.replace_children(
+ children,
+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),
+ )
+ }
+
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
+ fn execute(
+ &self,
+ _partition: usize,
+ _context: Arc<TaskContext>,
+ ) -> Result<SendableRecordBatchStream> {
+ unimplemented!()
+ }
+
+ fn statistics_from_inputs(
+ &self,
+ _input_stats: &[Arc<Statistics>],
+ _args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
+ Ok(Arc::new(Statistics::new_unknown(&self.schema)))
+ }
+}
+
+/// Build: CoalescePartitions^depth -> BenchLeaf
+fn build_coalesce_chain(depth: usize) -> Arc<dyn ExecutionPlan> {
+ let mut plan: Arc<dyn ExecutionPlan> = Arc::new(BenchLeaf::new("a"));
+ for _ in 0..depth {
+ plan = Arc::new(CoalescePartitionsExec::new(plan));
+ }
+ plan
+}
+
+/// Build a balanced binary tree of CrossJoinExec with 2^depth leaves.
+/// Mirrors the plan shape produced by multi-way self-joins like the
+/// `physical_many_self_joins` benchmark in sql_planner.rs (#19795).
+fn build_cross_join_tree(depth: usize, next_col: &mut usize) -> Arc<dyn ExecutionPlan> {
+ if depth == 0 {
+ let col_name = format!("c{next_col}");
+ *next_col += 1;
+ return Arc::new(BenchLeaf::new(&col_name));
+ }
+ let left = build_cross_join_tree(depth - 1, next_col);
+ let right = build_cross_join_tree(depth - 1, next_col);
+ Arc::new(CrossJoinExec::new(left, right))
+}
+
+/// Build: Filter^depth -> BenchLeaf (always-true predicate).
+fn build_filter_chain(depth: usize) -> Arc<dyn ExecutionPlan> {
+ let mut plan: Arc<dyn ExecutionPlan> = Arc::new(BenchLeaf::new("a"));
+ let predicate: Arc<dyn PhysicalExpr> =
+ Arc::new(Literal::new(ScalarValue::Boolean(Some(true))));
+ for _ in 0..depth {
+ plan = Arc::new(
+ FilterExec::try_new(Arc::clone(&predicate), plan)
+ .expect("FilterExec::try_new failed"),
+ );
+ }
+ plan
+}
+
+/// Build a mixed chain alternating partition-merging and partition-preserving
+/// operators: (Coalesce -> Filter -> Filter) repeated `groups` times -> BenchLeaf.
+/// Exercises the cache with both None and Some(p) lookups in the same walk.
+fn build_mixed_chain(groups: usize) -> Arc<dyn ExecutionPlan> {
+ let mut plan: Arc<dyn ExecutionPlan> = Arc::new(BenchLeaf::new("a"));
+ let predicate: Arc<dyn PhysicalExpr> =
+ Arc::new(Literal::new(ScalarValue::Boolean(Some(true))));
+ for _ in 0..groups {
+ // Two partition-preserving filters
+ for _ in 0..2 {
+ plan = Arc::new(
+ FilterExec::try_new(Arc::clone(&predicate), plan)
+ .expect("FilterExec::try_new failed"),
+ );
+ }
+ // One partition-merging coalesce
+ plan = Arc::new(CoalescePartitionsExec::new(plan));
+ }
+ plan
+}
+
+/// Recursive walk without a shared cross-node cache, simulating pre-cache behavior.
+/// Each node is computed with a fresh `StatisticsContext`, so every call triggers a
+/// fresh subtree walk, resulting in O(n^2) total node visits for a chain of depth n.
+///
+/// Note: each `StatisticsContext::compute` re-walk still benefits from its own
+/// ephemeral cache; only the cross-node sharing is removed.
+fn compute_statistics_without_shared_cache(
+ plan: &dyn ExecutionPlan,
+ partition: Option<usize>,
+) -> Result<Arc<Statistics>> {
+ for child in plan.children() {
+ compute_statistics_without_shared_cache(child.as_ref(), None)?;
+ }
+ let args = StatisticsArgs::new().with_partition(partition);
+ StatisticsContext::new().compute(plan, &args)
+}
+
+fn bench_compute_statistics(c: &mut Criterion) {
+ // --- Coalesce chain (linear plan) ---
+ // Deep linear plans arise from deeply nested subqueries, CTEs, etc.
+ let mut group = c.benchmark_group("compute_statistics_coalesce_chain");
+ for depth in [10, 20, 50] {
+ let plan = build_coalesce_chain(depth);
+ group.bench_with_input(BenchmarkId::new("cached", depth), &plan, |b, plan| {
+ b.iter(|| {
+ StatisticsContext::new()
+ .compute(plan.as_ref(), &StatisticsArgs::new())
+ .unwrap()
+ });
+ });
+ group.bench_with_input(
+ BenchmarkId::new("no_shared_cache", depth),
+ &plan,
+ |b, plan| {
+ b.iter(|| {
+ compute_statistics_without_shared_cache(plan.as_ref(), None).unwrap()
+ });
+ },
+ );
+ }
+ group.finish();
+
+ // --- Cross-join tree (balanced binary plan) ---
+ // Binary trees arise from multi-way joins (e.g. physical_many_self_joins
+ // in sql_planner.rs, see #19795). CrossJoinExec calls
+ // StatisticsContext::compute for per-partition stats, re-walking the left
+ // subtree at each node. The gap between cached/uncached is smaller than
+ // the linear chain because only the left child triggers a re-walk.
+ let mut group = c.benchmark_group("compute_statistics_cross_join_tree");
+ for depth in [3, 5, 7] {
+ let mut next_col = 0;
+ let plan = build_cross_join_tree(depth, &mut next_col);
+ let label = format!("depth={depth}_leaves={}", 1usize << depth);
+ group.bench_with_input(BenchmarkId::new("cached", &label), &plan, |b, plan| {
+ b.iter(|| {
+ StatisticsContext::new()
+ .compute(
+ plan.as_ref(),
+ &StatisticsArgs::new().with_partition(Some(0)),
+ )
+ .unwrap()
+ });
+ });
+ group.bench_with_input(
+ BenchmarkId::new("no_shared_cache", &label),
+ &plan,
+ |b, plan| {
+ b.iter(|| {
+ compute_statistics_without_shared_cache(plan.as_ref(), Some(0))
+ .unwrap()
+ });
+ },
+ );
+ }
+ group.finish();
+
+ // --- Filter chain (partition-preserving linear plan) ---
+ // When called with Some(0), the framework first walks the entire tree
+ // computing None stats, then each filter requests Some(0) on demand.
+ // Both walks are cached, so the total cost is ~2n vs n node visits for None.
+ let mut group = c.benchmark_group("compute_statistics_filter_chain");
+ for depth in [10, 20, 50] {
+ let plan = build_filter_chain(depth);
+ group.bench_with_input(
+ BenchmarkId::new("cached_partition", depth),
+ &plan,
+ |b, plan| {
+ b.iter(|| {
+ StatisticsContext::new()
+ .compute(
+ plan.as_ref(),
+ &StatisticsArgs::new().with_partition(Some(0)),
+ )
+ .unwrap()
+ });
+ },
+ );
+ group.bench_with_input(
+ BenchmarkId::new("cached_overall", depth),
+ &plan,
+ |b, plan| {
+ b.iter(|| {
+ StatisticsContext::new()
+ .compute(plan.as_ref(), &StatisticsArgs::new())
+ .unwrap()
+ });
+ },
+ );
+ group.bench_with_input(
+ BenchmarkId::new("no_shared_cache", depth),
+ &plan,
+ |b, plan| {
+ b.iter(|| {
+ compute_statistics_without_shared_cache(plan.as_ref(), Some(0))
+ .unwrap()
+ });
+ },
+ );
+ }
+ group.finish();
+
+ // --- Mixed chain (partition-preserving + partition-merging) ---
+ // Alternates Filter (preserving) and CoalescePartitions (merging) to
+ // exercise the cache with both None and Some(p) lookups in a single walk.
+ let mut group = c.benchmark_group("compute_statistics_mixed_chain");
+ for groups in [3, 5, 10] {
+ let plan = build_mixed_chain(groups);
+ let depth = groups * 3; // 2 filters + 1 coalesce per group
+ group.bench_with_input(BenchmarkId::new("cached", depth), &plan, |b, plan| {
+ b.iter(|| {
+ StatisticsContext::new()
+ .compute(
+ plan.as_ref(),
+ &StatisticsArgs::new().with_partition(Some(0)),
+ )
+ .unwrap()
+ });
+ });
+ group.bench_with_input(
+ BenchmarkId::new("no_shared_cache", depth),
+ &plan,
+ |b, plan| {
+ b.iter(|| {
+ compute_statistics_without_shared_cache(plan.as_ref(), Some(0))
+ .unwrap()
+ });
+ },
+ );
+ }
+ group.finish();
+}
+
+criterion_group!(benches, bench_compute_statistics);
+criterion_main!(benches);
diff --git a/native/vendor/datafusion-physical-plan/benches/dictionary_group_values.rs b/native/vendor/datafusion-physical-plan/benches/dictionary_group_values.rs
new file mode 100644
index 0000000000..ded52aebd1
--- /dev/null
+++ b/native/vendor/datafusion-physical-plan/benches/dictionary_group_values.rs
@@ -0,0 +1,176 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Benchmarks for `GroupValues` over a single `Dictionary<Int32, Utf8>`
+//! column. Each iteration measures `intern` (once or N times) followed by
+//! `emit(EmitTo::All)`. The `Box<dyn GroupValues>` returned by
+//! `new_group_values` is constructed in the setup closure of
+//! `iter_batched_ref` and is not included in the timing.
+
+use arrow::array::{ArrayRef, DictionaryArray, PrimitiveArray, StringArray};
+use arrow::buffer::{Buffer, NullBuffer};
+use arrow::datatypes::{DataType, Field, Int32Type, Schema, SchemaRef};
+use criterion::{
+ BatchSize, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main,
+};
+use datafusion_expr::EmitTo;
+use datafusion_physical_plan::aggregates::group_values::new_group_values;
+use datafusion_physical_plan::aggregates::order::GroupOrdering;
+use rand::rngs::StdRng;
+use rand::seq::SliceRandom;
+use rand::{Rng, SeedableRng};
+use std::hint::black_box;
+use std::sync::Arc;
+
+const SIZES: [usize; 2] = [8 * 1024, 64 * 1024];
+const CARDS_RELATIVE: [usize; 4] = [20, 75, 300, 1000];
+const N_BATCHES: usize = 4;
+// Fixed for reproducibility.
+const SEED: u64 = 0xD1C7;
+
+fn dict_schema() -> SchemaRef {
+ let dict_ty =
+ DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8));
+ Arc::new(Schema::new(vec![Field::new("g", dict_ty, true)]))
+}
+
+/// Build a `Dictionary<Int32, Utf8>` column.
+fn make_dict(size: usize, cardinality: usize, null_density: f32, seed: u64) -> ArrayRef {
+ let strings: Vec<String> = (0..cardinality).map(|i| format!("v_{i:08}")).collect();
+ let values = Arc::new(StringArray::from(
+ strings.iter().map(String::as_str).collect::<Vec<_>>(),
+ ));
+
+ let mut rng = StdRng::seed_from_u64(seed);
+ let keys: Vec<i32> = if cardinality == size {
+ let mut perm: Vec<i32> = (0..size as i32).collect();
+ perm.shuffle(&mut rng);
+ perm
+ } else {
+ (0..size)
+ .map(|_| rng.random_range(0..cardinality) as i32)
+ .collect()
+ };
+ let keys_buf = Buffer::from_slice_ref(&keys);
+
+ let nulls: Option<NullBuffer> = (null_density > 0.0).then(|| {
+ (0..size)
+ .map(|_| !rng.random_bool(null_density as f64))
+ .collect()
+ });
+
+ let key_array = PrimitiveArray::<Int32Type>::new(keys_buf.into(), nulls);
+ Arc::new(DictionaryArray::<Int32Type>::try_new(key_array, values).unwrap())
+}
+
+fn bench_id(
+ label: &str,
+ size: usize,
+ cardinality: usize,
+ null_density: f32,
+) -> BenchmarkId {
+ BenchmarkId::new(
+ label,
+ format!("size_{size}_card_{cardinality}_null_{null_density:.2}"),
+ )
+}
+
+fn bench_intern_emit(c: &mut Criterion) {
+ let mut group = c.benchmark_group("dict_intern_emit");
+ let schema = dict_schema();
+ let null_density = 0.0;
+
+ for &size in &SIZES {
+ let mut cards = CARDS_RELATIVE.to_vec();
+ cards.push(size); // all-unique stress case
+ for cardinality in cards {
+ let array = make_dict(size, cardinality, null_density, SEED);
+ group.throughput(Throughput::Elements(size as u64));
+ group.bench_function(
+ bench_id("intern_emit", size, cardinality, null_density),
+ |b| {
+ b.iter_batched_ref(
+ || {
+ (
+ new_group_values(schema.clone(), &GroupOrdering::None)
+ .unwrap(),
+ Vec::<usize>::with_capacity(size),
+ )
+ },
+ |(gv, groups)| {
+ gv.intern(std::slice::from_ref(&array), groups).unwrap();
+ black_box(&*groups);
+ black_box(gv.emit(EmitTo::All).unwrap());
+ },
+ BatchSize::SmallInput,
+ );
+ },
+ );
+ }
+ }
+ group.finish();
+}
+
+fn bench_repeated_intern_emit(c: &mut Criterion) {
+ let mut group = c.benchmark_group("dict_repeated_intern_emit");
+ let schema = dict_schema();
+ let null_density = 0.10;
+
+ for &size in &SIZES {
+ let mut cards = CARDS_RELATIVE.to_vec();
+ cards.push(size);
+ for cardinality in cards {
+ let batches: Vec<ArrayRef> = (0..N_BATCHES)
+ .map(|i| {
+ make_dict(
+ size,
+ cardinality,
+ null_density,
+ SEED.wrapping_add(i as u64),
+ )
+ })
+ .collect();
+ group.throughput(Throughput::Elements((size * N_BATCHES) as u64));
+ group.bench_function(
+ bench_id("repeated_intern_emit", size, cardinality, null_density),
+ |b| {
+ b.iter_batched_ref(
+ || {
+ (
+ new_group_values(schema.clone(), &GroupOrdering::None)
+ .unwrap(),
+ Vec::<usize>::with_capacity(size),
+ )
+ },
+ |(gv, groups)| {
+ for arr in &batches {
+ gv.intern(std::slice::from_ref(arr), groups).unwrap();
+ black_box(&*groups);
+ }
+ black_box(gv.emit(EmitTo::All).unwrap());
+ },
+ BatchSize::SmallInput,
+ );
+ },
+ );
+ }
+ }
+ group.finish();
+}
+
+criterion_group!(benches, bench_intern_emit, bench_repeated_intern_emit);
+criterion_main!(benches);
diff --git a/native/vendor/datafusion-physical-plan/benches/hash_join_semi_anti.rs b/native/vendor/datafusion-physical-plan/benches/hash_join_semi_anti.rs
new file mode 100644
index 0000000000..1e11da36be
--- /dev/null
+++ b/native/vendor/datafusion-physical-plan/benches/hash_join_semi_anti.rs
@@ -0,0 +1,387 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Criterion benchmarks for Hash Join with RightSemi/RightAnti joins with Int32 keys.
+//!
+//! ## Key Benchmark Axes
+//!
+//! - **Density**: How tightly distinct keys pack into their numeric range.
+//! `density = num_distinct_keys / (max_key - min_key + 1)`.
+//! Examples for 5 distinct keys:
+//! - `[0, 1, 2, 3, 4]` → 5/5 = 100% (fully packed)
+//! - `[0, 2, 4, 6, 8]` → 5/9 ≈ 55% (every 2nd slot)
+//! - `[0, 10, 20, 30, 40]` → 5/41 ≈ 12% (every 10th slot)
+//!
+//! Why it matters for this workload: future potential semi/anti-join
+//! fast paths could exploit densely packed build keys to outperform the
+//! general hash-table path, which is largely insensitive to density.
+//! Varying density across benchmarks helps surface those potential gains
+//! under different key distributions. Density describes only the
+//! build-side key layout; the per-probe match count is tracked
+//! separately as fanout.
+//!
+//! - **Hit Rate**: The percentage of probe rows that find a match in the build side.
+//! This controls how often the join produces output rows.
+//!
+//! Semi/anti joins can short-circuit after finding the first match, so these
+//! benchmarks help evaluate optimization strategies for existence checks.
+
+use std::sync::Arc;
+
+use arrow::array::{Int32Array, RecordBatch, StringArray};
+use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
+use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
+use datafusion_common::{JoinType, NullEquality};
+use datafusion_execution::TaskContext;
+use datafusion_physical_expr::expressions::col;
+use datafusion_physical_plan::collect;
+use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode, utils::JoinOn};
+use datafusion_physical_plan::test::TestMemoryExec;
+use tokio::runtime::Runtime;
+
+/// Build RecordBatches with Int32 keys.
+///
+/// Schema: (key: Int32, data: Int32, payload: Utf8)
+///
+/// `key_mod` controls distinct key count: key = row_index % key_mod.
+/// `key_offset` shifts keys to control hit rate.
+fn build_batches(
+ num_rows: usize,
+ key_mod: usize,
+ key_offset: i32,
+ schema: &SchemaRef,
+) -> Vec<RecordBatch> {
+ let keys: Vec<i32> = (0..num_rows)
+ .map(|i| ((i % key_mod) as i32) + key_offset)
+ .collect();
+ let data: Vec<i32> = (0..num_rows).map(|i| i as i32).collect();
+ let payload: Vec<String> = data.iter().map(|d| format!("val_{d}")).collect();
+
+ let batch = RecordBatch::try_new(
+ Arc::clone(schema),
+ vec![
+ Arc::new(Int32Array::from(keys)),
+ Arc::new(Int32Array::from(data)),
+ Arc::new(StringArray::from(payload)),
+ ],
+ )
+ .unwrap();
+
+ let batch_size = 8192;
+ let mut batches = Vec::new();
+ let mut offset = 0;
+ while offset < batch.num_rows() {
+ let len = (batch.num_rows() - offset).min(batch_size);
+ batches.push(batch.slice(offset, len));
+ offset += len;
+ }
+ batches
+}
+
+fn make_exec(
+ batches: &[RecordBatch],
+ schema: &SchemaRef,
+) -> Arc<dyn datafusion_physical_plan::ExecutionPlan> {
+ TestMemoryExec::try_new_exec(&[batches.to_vec()], Arc::clone(schema), None).unwrap()
+}
+
+fn schema() -> SchemaRef {
+ Arc::new(Schema::new(vec![
+ Field::new("key", DataType::Int32, false),
+ Field::new("data", DataType::Int32, false),
+ Field::new("payload", DataType::Utf8, false),
+ ]))
+}
+
+fn do_hash_join(
+ left: Arc<dyn datafusion_physical_plan::ExecutionPlan>,
+ right: Arc<dyn datafusion_physical_plan::ExecutionPlan>,
+ join_type: JoinType,
+ rt: &Runtime,
+) -> usize {
+ let on: JoinOn = vec![(
+ col("key", &left.schema()).unwrap(),
+ col("key", &right.schema()).unwrap(),
+ )];
+ let join = HashJoinExec::try_new(
+ left,
+ right,
+ on,
+ None,
+ &join_type,
+ None,
+ PartitionMode::CollectLeft,
+ NullEquality::NullEqualsNothing,
+ false,
+ )
+ .unwrap();
+
+ let task_ctx = Arc::new(TaskContext::default());
+ rt.block_on(async {
+ let batches = collect(Arc::new(join), task_ctx).await.unwrap();
+ batches.iter().map(|b| b.num_rows()).sum()
+ })
+}
+
+/// Build batches with sparse keys (key = row_index % key_mod * multiplier + key_offset).
+/// The `multiplier` controls density: 1 = 100%, 2 = 50%, 10 = 10%.
+fn build_batches_sparse(
+ num_rows: usize,
+ key_mod: usize,
+ key_offset: i32,
+ multiplier: i32,
+ schema: &SchemaRef,
+) -> Vec<RecordBatch> {
+ let keys: Vec<i32> = (0..num_rows)
+ .map(|i| ((i % key_mod) as i32) * multiplier + key_offset)
+ .collect();
+ let data: Vec<i32> = (0..num_rows).map(|i| i as i32).collect();
+ let payload: Vec<String> = data.iter().map(|d| format!("val_{d}")).collect();
+
+ let batch = RecordBatch::try_new(
+ Arc::clone(schema),
+ vec![
+ Arc::new(Int32Array::from(keys)),
+ Arc::new(Int32Array::from(data)),
+ Arc::new(StringArray::from(payload)),
+ ],
+ )
+ .unwrap();
+
+ let batch_size = 8192;
+ let mut batches = Vec::new();
+ let mut offset = 0;
+ while offset < batch.num_rows() {
+ let len = (batch.num_rows() - offset).min(batch_size);
+ batches.push(batch.slice(offset, len));
+ offset += len;
+ }
+ batches
+}
+
+fn bench_hash_join_semi_anti(c: &mut Criterion) {
+ let rt = Runtime::new().unwrap();
+ let s = schema();
+
+ let mut group = c.benchmark_group("hash_join_semi_anti");
+
+ // Build side: 100K rows, Probe side: 1M rows
+ // Matching ratio: 1:1 (build keys are unique, each probe matches at most 1 build row)
+ let build_rows = 100_000;
+ let probe_rows = 1_000_000;
+
+ // =========================================================================
+ // RightSemi Join benchmarks
+ // =========================================================================
+
+ // RightSemi - 100% Density, 100% hit rate
+ // Keys: 0..100K contiguous, all probe rows find a match
+ {
+ let left_batches = build_batches(build_rows, build_rows, 0, &s);
+ let right_batches = build_batches(probe_rows, build_rows, 0, &s);
+ group.bench_function(BenchmarkId::new("right_semi_d100_h100", probe_rows), |b| {
+ b.iter(|| {
+ let left = make_exec(&left_batches, &s);
+ let right = make_exec(&right_batches, &s);
+ do_hash_join(left, right, JoinType::RightSemi, &rt)
+ })
+ });
+ }
+
+ // RightSemi - 100% Density, 10% hit rate
+ // Keys: 0..100K contiguous, only 10% of probe rows find a match
+ {
+ let left_batches = build_batches(build_rows, build_rows, 0, &s);
+ let right_batches = build_batches(probe_rows, build_rows * 10, 0, &s);
+ group.bench_function(BenchmarkId::new("right_semi_d100_h10", probe_rows), |b| {
+ b.iter(|| {
+ let left = make_exec(&left_batches, &s);
+ let right = make_exec(&right_batches, &s);
+ do_hash_join(left, right, JoinType::RightSemi, &rt)
+ })
+ });
+ }
+
+ // RightSemi - 50% Density, 100% hit rate
+ // Keys: 0, 2, 4, ... (sparse, multiplier=2), all probe rows find a match
+ {
+ let left_batches = build_batches_sparse(build_rows, build_rows, 0, 2, &s);
+ let right_batches = build_batches_sparse(probe_rows, build_rows, 0, 2, &s);
+ group.bench_function(BenchmarkId::new("right_semi_d50_h100", probe_rows), |b| {
+ b.iter(|| {
+ let left = make_exec(&left_batches, &s);
+ let right = make_exec(&right_batches, &s);
+ do_hash_join(left, right, JoinType::RightSemi, &rt)
+ })
+ });
+ }
+
+ // RightSemi - 50% Density, 10% hit rate
+ // Keys: 0, 2, 4, ... (sparse), only 10% of probe rows find a match
+ {
+ let left_batches = build_batches_sparse(build_rows, build_rows, 0, 2, &s);
+ let right_batches = build_batches_sparse(probe_rows, build_rows * 10, 0, 2, &s);
+ group.bench_function(BenchmarkId::new("right_semi_d50_h10", probe_rows), |b| {
+ b.iter(|| {
+ let left = make_exec(&left_batches, &s);
+ let right = make_exec(&right_batches, &s);
+ do_hash_join(left, right, JoinType::RightSemi, &rt)
+ })
+ });
+ }
+
+ // RightSemi - 10% Density, 100% hit rate
+ // Keys: 0, 10, 20, ... (very sparse, multiplier=10), all probe rows find a match
+ {
+ let left_batches = build_batches_sparse(build_rows, build_rows, 0, 10, &s);
+ let right_batches = build_batches_sparse(probe_rows, build_rows, 0, 10, &s);
+ group.bench_function(BenchmarkId::new("right_semi_d10_h100", probe_rows), |b| {
+ b.iter(|| {
+ let left = make_exec(&left_batches, &s);
+ let right = make_exec(&right_batches, &s);
+ do_hash_join(left, right, JoinType::RightSemi, &rt)
+ })
+ });
+ }
+
+ // RightSemi - 10% Density, 10% hit rate
+ // Keys: 0, 10, 20, ... (very sparse), only 10% of probe rows find a match
+ {
+ let left_batches = build_batches_sparse(build_rows, build_rows, 0, 10, &s);
+ let right_batches = build_batches_sparse(probe_rows, build_rows * 10, 0, 10, &s);
+ group.bench_function(BenchmarkId::new("right_semi_d10_h10", probe_rows), |b| {
+ b.iter(|| {
+ let left = make_exec(&left_batches, &s);
+ let right = make_exec(&right_batches, &s);
+ do_hash_join(left, right, JoinType::RightSemi, &rt)
+ })
+ });
+ }
+
+ // RightSemi - 100% Density, ~1% hit rate, fanout ~100
+ // Build keys are duplicated: 100K rows over 1K distinct keys. Matching
+ // probe rows produce many duplicate probe indices before RightSemi
+ // deduplication.
+ {
+ let fanout_keys = 1_000;
+ let left_batches = build_batches(build_rows, fanout_keys, 0, &s);
+ let right_batches = build_batches(probe_rows, build_rows, 0, &s);
+ group.bench_function(
+ BenchmarkId::new("right_semi_fanout100_h1", probe_rows),
+ |b| {
+ b.iter(|| {
+ let left = make_exec(&left_batches, &s);
+ let right = make_exec(&right_batches, &s);
+ do_hash_join(left, right, JoinType::RightSemi, &rt)
+ })
+ },
+ );
+ }
+
+ // =========================================================================
+ // RightAnti Join benchmarks
+ // =========================================================================
+
+ // RightAnti - 100% Density, 100% hit rate (no output)
+ // Keys: 0..100K contiguous, all probe rows find a match -> no output
+ {
+ let left_batches = build_batches(build_rows, build_rows, 0, &s);
+ let right_batches = build_batches(probe_rows, build_rows, 0, &s);
+ group.bench_function(BenchmarkId::new("right_anti_d100_h100", probe_rows), |b| {
+ b.iter(|| {
+ let left = make_exec(&left_batches, &s);
+ let right = make_exec(&right_batches, &s);
+ do_hash_join(left, right, JoinType::RightAnti, &rt)
+ })
+ });
+ }
+
+ // RightAnti - 100% Density, 10% hit rate (90% output)
+ // Keys: 0..100K contiguous, only 10% of probe rows find a match -> 90% output
+ {
+ let left_batches = build_batches(build_rows, build_rows, 0, &s);
+ let right_batches = build_batches(probe_rows, build_rows * 10, 0, &s);
+ group.bench_function(BenchmarkId::new("right_anti_d100_h10", probe_rows), |b| {
+ b.iter(|| {
+ let left = make_exec(&left_batches, &s);
+ let right = make_exec(&right_batches, &s);
+ do_hash_join(left, right, JoinType::RightAnti, &rt)
+ })
+ });
+ }
+
+ // RightAnti - 50% Density, 100% hit rate (no output)
+ // Keys: 0, 2, 4, ... (sparse), all probe rows find a match -> no output
+ {
+ let left_batches = build_batches_sparse(build_rows, build_rows, 0, 2, &s);
+ let right_batches = build_batches_sparse(probe_rows, build_rows, 0, 2, &s);
+ group.bench_function(BenchmarkId::new("right_anti_d50_h100", probe_rows), |b| {
+ b.iter(|| {
+ let left = make_exec(&left_batches, &s);
+ let right = make_exec(&right_batches, &s);
+ do_hash_join(left, right, JoinType::RightAnti, &rt)
+ })
+ });
+ }
+
+ // RightAnti - 50% Density, 10% hit rate (90% output)
+ // Keys: 0, 2, 4, ... (sparse), only 10% of probe rows find a match -> 90% output
+ {
+ let left_batches = build_batches_sparse(build_rows, build_rows, 0, 2, &s);
+ let right_batches = build_batches_sparse(probe_rows, build_rows * 10, 0, 2, &s);
+ group.bench_function(BenchmarkId::new("right_anti_d50_h10", probe_rows), |b| {
+ b.iter(|| {
+ let left = make_exec(&left_batches, &s);
+ let right = make_exec(&right_batches, &s);
+ do_hash_join(left, right, JoinType::RightAnti, &rt)
+ })
+ });
+ }
+
+ // RightAnti - 10% Density, 100% hit rate (no output)
+ // Keys: 0, 10, 20, ... (very sparse), all probe rows find a match -> no output
+ {
+ let left_batches = build_batches_sparse(build_rows, build_rows, 0, 10, &s);
+ let right_batches = build_batches_sparse(probe_rows, build_rows, 0, 10, &s);
+ group.bench_function(BenchmarkId::new("right_anti_d10_h100", probe_rows), |b| {
+ b.iter(|| {
+ let left = make_exec(&left_batches, &s);
+ let right = make_exec(&right_batches, &s);
+ do_hash_join(left, right, JoinType::RightAnti, &rt)
+ })
+ });
+ }
+
+ // RightAnti - 10% Density, 10% hit rate (90% output)
+ // Keys: 0, 10, 20, ... (very sparse), only 10% of probe rows find a match -> 90% output
+ {
+ let left_batches = build_batches_sparse(build_rows, build_rows, 0, 10, &s);
+ let right_batches = build_batches_sparse(probe_rows, build_rows * 10, 0, 10, &s);
+ group.bench_function(BenchmarkId::new("right_anti_d10_h10", probe_rows), |b| {
+ b.iter(|| {
+ let left = make_exec(&left_batches, &s);
+ let right = make_exec(&right_batches, &s);
+ do_hash_join(left, right, JoinType::RightAnti, &rt)
+ })
+ });
+ }
+
+ group.finish();
+}
+
+criterion_group!(benches, bench_hash_join_semi_anti);
+criterion_main!(benches);
diff --git a/native/vendor/datafusion-physical-plan/benches/multi_group_by.rs b/native/vendor/datafusion-physical-plan/benches/multi_group_by.rs
new file mode 100644
index 0000000000..0c689f9fcb
--- /dev/null
+++ b/native/vendor/datafusion-physical-plan/benches/multi_group_by.rs
@@ -0,0 +1,815 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Benchmarks for multi-column GROUP BY performance comparing vectorized
+//! (`GroupValuesColumn`) vs row-based (`GroupValuesRows`) implementations.
+//!
+//! Motivated by <https://github.com/apache/datafusion/issues/17850> which
+//! showed vectorized can regress for low-cardinality, high-row-count scenarios.
+//!
+//! Uses the direct `GroupValues::intern()` API with identical data for both
+//! implementations — a fair apples-to-apples comparison with the same hashing
+//! and data layout. Most experiments use `Int32` columns; `bench_fixed_size_binary`
+//! covers a `(FixedSizeBinary, Int32)` key to exercise the
+//! `FixedSizeBinaryGroupValueBuilder`.
+
+use arrow::array::{
+ ArrayRef, Decimal256Array, DurationMicrosecondArray, Float16Array, Int32Array,
+ IntervalMonthDayNanoArray, UInt32Array,
+};
+use arrow::compute::take;
+use arrow::datatypes::{
+ DataType, Field, IntervalMonthDayNano, IntervalUnit, Schema, SchemaRef, TimeUnit,
+ i256,
+};
+use arrow::util::bench_util::create_fsb_array;
+use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
+use datafusion_physical_plan::aggregates::group_values::GroupValues;
+use datafusion_physical_plan::aggregates::group_values::GroupValuesRows;
+use datafusion_physical_plan::aggregates::group_values::multi_group_by::GroupValuesColumn;
+use half::f16;
+use std::hint::black_box;
+use std::sync::Arc;
+
+const DEFAULT_BATCH_SIZE: usize = 8192;
+
+fn make_schema(num_cols: usize) -> SchemaRef {
+ let fields: Vec<Field> = (0..num_cols)
+ .map(|i| Field::new(format!("col_{i}"), DataType::Int32, false))
+ .collect();
+ Arc::new(Schema::new(fields))
+}
+
+fn generate_batches(
+ num_cols: usize,
+ num_distinct_groups: usize,
+ num_rows: usize,
+ batch_size: usize,
+) -> Vec<Vec<ArrayRef>> {
+ let per_col_card = (num_distinct_groups as f64)
+ .powf(1.0 / num_cols as f64)
+ .ceil() as usize;
+
+ let num_full_batches = num_rows / batch_size;
+ let remainder = num_rows % batch_size;
+ let num_batches = num_full_batches + if remainder > 0 { 1 } else { 0 };
+
+ (0..num_batches)
+ .map(|batch_idx| {
+ let batch_start = batch_idx * batch_size;
+ let current_batch_size = if batch_idx == num_batches - 1 && remainder > 0 {
+ remainder
+ } else {
+ batch_size
+ };
+ (0..num_cols)
+ .map(|col_idx| {
+ let values: Vec<i32> = (0..current_batch_size)
+ .map(|row| {
+ let global_row = batch_start + row;
+ let group_id = global_row % num_distinct_groups;
+ let divisor = per_col_card.pow(col_idx as u32);
+ ((group_id / divisor) % per_col_card) as i32
+ })
+ .collect();
+ Arc::new(Int32Array::from(values)) as ArrayRef
+ })
+ .collect()
+ })
+ .collect()
+}
+
+fn create_group_values(schema: &SchemaRef, vectorized: bool) -> Box<dyn GroupValues> {
+ if vectorized {
+ Box::new(GroupValuesColumn::<false>::try_new(Arc::clone(schema)).unwrap())
+ } else {
+ Box::new(GroupValuesRows::try_new(Arc::clone(schema)).unwrap())
+ }
+}
+
+fn bench_intern(
+ gv: &mut Box<dyn GroupValues>,
+ batches: &[Vec<ArrayRef>],
+ groups: &mut Vec<usize>,
+) {
+ for batch in batches {
+ groups.clear();
+ gv.intern(batch, groups).unwrap();
+ }
+ black_box(&*groups);
+}
+
+/// Experiment 1: Issue #17850 regression scenario.
+/// 3 columns, 64 groups (4^3), scaling row count.
+fn bench_issue_17850_regression(c: &mut Criterion) {
+ let mut group = c.benchmark_group("issue_17850_regression");
+ group.sample_size(10);
+
+ let num_cols = 3;
+ let num_groups = 64;
+ let schema = make_schema(num_cols);
+
+ for num_rows in [1_000_000, 5_000_000, 10_000_000, 20_000_000, 50_000_000] {
+ let batches =
+ generate_batches(num_cols, num_groups, num_rows, DEFAULT_BATCH_SIZE);
+
+ for vectorized in [true, false] {
+ let label = if vectorized {
+ "vectorized"
+ } else {
+ "row_based"
+ };
+ group.bench_with_input(
+ BenchmarkId::new(label, format!("{num_rows}_rows")),
+ &batches,
+ |b, batches| {
+ b.iter_batched_ref(
+ || {
+ (
+ create_group_values(&schema, vectorized),
+ Vec::<usize>::with_capacity(DEFAULT_BATCH_SIZE),
+ )
+ },
+ |(gv, groups)| bench_intern(gv, batches, groups),
+ criterion::BatchSize::LargeInput,
+ );
+ },
+ );
+ }
+ }
+ group.finish();
+}
+
+/// Experiment 2: Low cardinality sweep.
+fn bench_low_cardinality(c: &mut Criterion) {
+ let mut group = c.benchmark_group("low_cardinality");
+ group.sample_size(15);
+
+ for (num_cols, per_col_card) in
+ [(3usize, 2usize), (3, 4), (3, 8), (4, 2), (4, 4), (4, 8)]
+ {
+ let num_groups = per_col_card.pow(num_cols as u32);
+ let schema = make_schema(num_cols);
+ let batches =
+ generate_batches(num_cols, num_groups, 1_000_000, DEFAULT_BATCH_SIZE);
+
+ for vectorized in [true, false] {
+ let label = if vectorized {
+ "vectorized"
+ } else {
+ "row_based"
+ };
+ group.bench_with_input(
+ BenchmarkId::new(
+ label,
+ format!("cols_{num_cols}_card_{per_col_card}_grp_{num_groups}"),
+ ),
+ &batches,
+ |b, batches| {
+ b.iter_batched_ref(
+ || {
+ (
+ create_group_values(&schema, vectorized),
+ Vec::<usize>::with_capacity(DEFAULT_BATCH_SIZE),
+ )
+ },
+ |(gv, groups)| bench_intern(gv, batches, groups),
+ criterion::BatchSize::LargeInput,
+ );
+ },
+ );
+ }
+ }
+ group.finish();
+}
+
+/// Experiment 3: Batch size sensitivity.
+fn bench_batch_size_sensitivity(c: &mut Criterion) {
+ let mut group = c.benchmark_group("batch_size_sensitivity");
+ group.sample_size(10);
+
+ let num_cols = 3;
+ let num_groups = 64;
+ let schema = make_schema(num_cols);
+
+ for batch_size in [1024, 4096, 8192, 16384, 32768] {
+ let batches = generate_batches(num_cols, num_groups, 1_000_000, batch_size);
+
+ for vectorized in [true, false] {
+ let label = if vectorized {
+ "vectorized"
+ } else {
+ "row_based"
+ };
+ group.bench_with_input(
+ BenchmarkId::new(label, format!("batch_{batch_size}")),
+ &batches,
+ |b, batches| {
+ b.iter_batched_ref(
+ || {
+ (
+ create_group_values(&schema, vectorized),
+ Vec::<usize>::with_capacity(batch_size),
+ )
+ },
+ |(gv, groups)| bench_intern(gv, batches, groups),
+ criterion::BatchSize::LargeInput,
+ );
+ },
+ );
+ }
+ }
+ group.finish();
+}
+
+/// Experiment 4: Column count scaling with low groups.
+fn bench_column_scaling(c: &mut Criterion) {
+ let mut group = c.benchmark_group("column_scaling");
+ group.sample_size(15);
+
+ let cases: &[(usize, usize)] =
+ &[(2, 100), (3, 125), (4, 81), (6, 729), (8, 256), (10, 1024)];
+
+ for &(num_cols, num_groups) in cases {
+ let schema = make_schema(num_cols);
+ let batches =
+ generate_batches(num_cols, num_groups, 1_000_000, DEFAULT_BATCH_SIZE);
+
+ for vectorized in [true, false] {
+ let label = if vectorized {
+ "vectorized"
+ } else {
+ "row_based"
+ };
+ group.bench_with_input(
+ BenchmarkId::new(label, format!("cols_{num_cols}_grp_{num_groups}")),
+ &batches,
+ |b, batches| {
+ b.iter_batched_ref(
+ || {
+ (
+ create_group_values(&schema, vectorized),
+ Vec::<usize>::with_capacity(DEFAULT_BATCH_SIZE),
+ )
+ },
+ |(gv, groups)| bench_intern(gv, batches, groups),
+ criterion::BatchSize::LargeInput,
+ );
+ },
+ );
+ }
+ }
+ group.finish();
+}
+
+/// Experiment 5: High cardinality column scaling (~1M groups).
+fn bench_high_cardinality_scaling(c: &mut Criterion) {
+ let mut group = c.benchmark_group("high_cardinality_scaling");
+ group.sample_size(10);
+
+ for num_cols in [2, 3, 4, 6, 8, 10] {
+ let num_groups = 1_000_000;
+ let schema = make_schema(num_cols);
+ let batches =
+ generate_batches(num_cols, num_groups, 1_000_000, DEFAULT_BATCH_SIZE);
+
+ for vectorized in [true, false] {
+ let label = if vectorized {
+ "vectorized"
+ } else {
+ "row_based"
+ };
+ group.bench_with_input(
+ BenchmarkId::new(label, format!("cols_{num_cols}_grp_1M")),
+ &batches,
+ |b, batches| {
+ b.iter_batched_ref(
+ || {
+ (
+ create_group_values(&schema, vectorized),
+ Vec::<usize>::with_capacity(DEFAULT_BATCH_SIZE),
+ )
+ },
+ |(gv, groups)| bench_intern(gv, batches, groups),
+ criterion::BatchSize::LargeInput,
+ );
+ },
+ );
+ }
+ }
+ group.finish();
+}
+
+/// Experiment 6: Group count sweep with fixed 4 columns.
+fn bench_group_count_sweep(c: &mut Criterion) {
+ let mut group = c.benchmark_group("group_count_sweep");
+ group.sample_size(15);
+
+ let num_cols = 4;
+ let schema = make_schema(num_cols);
+
+ for num_groups in [
+ 16, 64, 256, 1000, 5000, 10_000, 50_000, 100_000, 500_000, 1_000_000,
+ ] {
+ let batches =
+ generate_batches(num_cols, num_groups, 1_000_000, DEFAULT_BATCH_SIZE);
+
+ for vectorized in [true, false] {
+ let label = if vectorized {
+ "vectorized"
+ } else {
+ "row_based"
+ };
+ group.bench_with_input(
+ BenchmarkId::new(label, format!("grp_{num_groups}")),
+ &batches,
+ |b, batches| {
+ b.iter_batched_ref(
+ || {
+ (
+ create_group_values(&schema, vectorized),
+ Vec::<usize>::with_capacity(DEFAULT_BATCH_SIZE),
+ )
+ },
+ |(gv, groups)| bench_intern(gv, batches, groups),
+ criterion::BatchSize::LargeInput,
+ );
+ },
+ );
+ }
+ }
+ group.finish();
+}
+
+/// Width in bytes of the FixedSizeBinary group column (UUID-sized).
+const FSB_WIDTH: usize = 16;
+
+/// Schema for the FixedSizeBinary experiment: a `FixedSizeBinary` group column
+/// paired with an `Int32` column, exercising a multi-column GROUP BY that
+/// includes a fixed-width binary key (e.g. grouping on a UUID).
+fn make_fsb_schema() -> SchemaRef {
+ Arc::new(Schema::new(vec![
+ Field::new("fsb", DataType::FixedSizeBinary(FSB_WIDTH as i32), false),
+ Field::new("id", DataType::Int32, false),
+ ]))
+}
+
+/// Generate `(FixedSizeBinary, Int32)` batches with exactly
+/// `num_distinct_groups` distinct keys.
+///
+/// The distinct FixedSizeBinary values come from arrow-rs's `create_fsb_array`
+/// benchmark generator; rows cycle through that pool (mirroring how
+/// `generate_batches` controls Int32 cardinality) so the group count is
+/// controlled. The `Int32` column is keyed identically, keeping the combined
+/// cardinality equal to `num_distinct_groups`.
+fn generate_fsb_batches(
+ num_distinct_groups: usize,
+ num_rows: usize,
+ batch_size: usize,
+) -> Vec<Vec<ArrayRef>> {
+ // Pool of distinct FixedSizeBinary values (fixed seed, no nulls).
+ let pool = create_fsb_array(num_distinct_groups, 0.0, FSB_WIDTH);
+
+ let num_full_batches = num_rows / batch_size;
+ let remainder = num_rows % batch_size;
+ let num_batches = num_full_batches + if remainder > 0 { 1 } else { 0 };
+
+ (0..num_batches)
+ .map(|batch_idx| {
+ let batch_start = batch_idx * batch_size;
+ let current_batch_size = if batch_idx == num_batches - 1 && remainder > 0 {
+ remainder
+ } else {
+ batch_size
+ };
+
+ let group_ids = (0..current_batch_size)
+ .map(|row| (batch_start + row) % num_distinct_groups);
+
+ let indices: UInt32Array = group_ids.clone().map(|g| g as u32).collect();
+ let fsb = take(&pool, &indices, None).unwrap();
+ let id: Int32Array = group_ids.map(|g| g as i32).collect();
+
+ vec![fsb, Arc::new(id) as ArrayRef]
+ })
+ .collect()
+}
+
+/// Experiment 7: Group count sweep for a `(FixedSizeBinary, Int32)` key.
+///
+/// Exercises the `FixedSizeBinaryGroupValueBuilder` used by multi-column
+/// GROUP BY. Before FixedSizeBinary support, such a schema fell back to the
+/// row-based `GroupValuesRows`; this compares the vectorized columnar path
+/// (`vectorized`) against that baseline (`row_based`).
+fn bench_fixed_size_binary(c: &mut Criterion) {
+ let mut group = c.benchmark_group("fixed_size_binary");
+ group.sample_size(15);
+
+ let schema = make_fsb_schema();
+
+ for num_groups in [1_000, 1_000_000] {
+ let batches = generate_fsb_batches(num_groups, 1_000_000, DEFAULT_BATCH_SIZE);
+
+ for vectorized in [true, false] {
+ let label = if vectorized {
+ "vectorized"
+ } else {
+ "row_based"
+ };
+ group.bench_with_input(
+ BenchmarkId::new(label, format!("grp_{num_groups}")),
+ &batches,
+ |b, batches| {
+ b.iter_batched_ref(
+ || {
+ (
+ create_group_values(&schema, vectorized),
+ Vec::<usize>::with_capacity(DEFAULT_BATCH_SIZE),
+ )
+ },
+ |(gv, groups)| bench_intern(gv, batches, groups),
+ criterion::BatchSize::LargeInput,
+ );
+ },
+ );
+ }
+ }
+ group.finish();
+}
+
+fn make_f16_schema() -> SchemaRef {
+ Arc::new(Schema::new(vec![
+ Field::new("f16", DataType::Float16, false),
+ Field::new("id", DataType::Int32, false),
+ ]))
+}
+
+/// Generate `(Float16, Int32)` batches with `num_distinct_groups` distinct keys.
+///
+/// `f16` has only ~63.5k finite values, so `num_distinct_groups` must stay well
+/// under that (see `bench_float16`). Distinct keys are the low finite `f16` bit
+/// patterns, skipping NaN and inf. The `Int32` column is keyed identically so
+/// the combined cardinality equals `num_distinct_groups`.
+fn generate_f16_batches(
+ num_distinct_groups: usize,
+ num_rows: usize,
+ batch_size: usize,
+) -> Vec<Vec<ArrayRef>> {
+ let pool: Vec<f16> = (0u16..)
+ .map(f16::from_bits)
+ .filter(|v| v.is_finite())
+ .take(num_distinct_groups)
+ .collect();
+
+ let num_full_batches = num_rows / batch_size;
+ let remainder = num_rows % batch_size;
+ let num_batches = num_full_batches + if remainder > 0 { 1 } else { 0 };
+
+ (0..num_batches)
+ .map(|batch_idx| {
+ let batch_start = batch_idx * batch_size;
+ let current_batch_size = if batch_idx == num_batches - 1 && remainder > 0 {
+ remainder
+ } else {
+ batch_size
+ };
+
+ let group_ids = (0..current_batch_size)
+ .map(|row| (batch_start + row) % num_distinct_groups);
+
+ let keys = Float16Array::from_iter_values(group_ids.clone().map(|g| pool[g]));
+ let id: Int32Array = group_ids.map(|g| g as i32).collect();
+
+ vec![Arc::new(keys) as ArrayRef, Arc::new(id) as ArrayRef]
+ })
+ .collect()
+}
+
+/// Experiment 8: Group count sweep for a `(Float16, Int32)` key.
+///
+/// Exercises the primitive `GroupColumn` builder for `Float16` on the
+/// multi-column path (previously such a schema fell back to `GroupValuesRows`).
+/// Group counts are capped below `f16`'s ~63.5k distinct finite values.
+fn bench_float16(c: &mut Criterion) {
+ let mut group = c.benchmark_group("float16");
+ group.sample_size(15);
+
+ let schema = make_f16_schema();
+
+ for num_groups in [1_000, 60_000] {
+ let batches = generate_f16_batches(num_groups, 1_000_000, DEFAULT_BATCH_SIZE);
+
+ for vectorized in [true, false] {
+ let label = if vectorized {
+ "vectorized"
+ } else {
+ "row_based"
+ };
+ group.bench_with_input(
+ BenchmarkId::new(label, format!("grp_{num_groups}")),
+ &batches,
+ |b, batches| {
+ b.iter_batched_ref(
+ || {
+ (
+ create_group_values(&schema, vectorized),
+ Vec::<usize>::with_capacity(DEFAULT_BATCH_SIZE),
+ )
+ },
+ |(gv, groups)| bench_intern(gv, batches, groups),
+ criterion::BatchSize::LargeInput,
+ );
+ },
+ );
+ }
+ }
+ group.finish();
+}
+
+fn make_duration_schema() -> SchemaRef {
+ Arc::new(Schema::new(vec![
+ Field::new("dur", DataType::Duration(TimeUnit::Microsecond), false),
+ Field::new("id", DataType::Int32, false),
+ ]))
+}
+
+/// Generate `(Duration(Microsecond), Int32)` batches with `num_distinct_groups`
+/// distinct keys.
+///
+/// Each distinct duration is `g` microseconds. The `Int32` column is keyed
+/// identically so the combined cardinality equals `num_distinct_groups`.
+fn generate_duration_batches(
+ num_distinct_groups: usize,
+ num_rows: usize,
+ batch_size: usize,
+) -> Vec<Vec<ArrayRef>> {
+ let num_full_batches = num_rows / batch_size;
+ let remainder = num_rows % batch_size;
+ let num_batches = num_full_batches + if remainder > 0 { 1 } else { 0 };
+
+ (0..num_batches)
+ .map(|batch_idx| {
+ let batch_start = batch_idx * batch_size;
+ let current_batch_size = if batch_idx == num_batches - 1 && remainder > 0 {
+ remainder
+ } else {
+ batch_size
+ };
+
+ let group_ids = (0..current_batch_size)
+ .map(|row| (batch_start + row) % num_distinct_groups);
+
+ let keys = DurationMicrosecondArray::from_iter_values(
+ group_ids.clone().map(|g| g as i64),
+ );
+ let id: Int32Array = group_ids.map(|g| g as i32).collect();
+
+ vec![Arc::new(keys) as ArrayRef, Arc::new(id) as ArrayRef]
+ })
+ .collect()
+}
+
+/// Experiment 9: Group count sweep for a `(Duration, Int32)` key.
+///
+/// Exercises the primitive `GroupColumn` builder for `Duration` on the
+/// multi-column path (previously such a schema fell back to `GroupValuesRows`).
+fn bench_duration(c: &mut Criterion) {
+ let mut group = c.benchmark_group("duration");
+ group.sample_size(15);
+
+ let schema = make_duration_schema();
+
+ for num_groups in [1_000, 1_000_000] {
+ let batches =
+ generate_duration_batches(num_groups, 1_000_000, DEFAULT_BATCH_SIZE);
+
+ for vectorized in [true, false] {
+ let label = if vectorized {
+ "vectorized"
+ } else {
+ "row_based"
+ };
+ group.bench_with_input(
+ BenchmarkId::new(label, format!("grp_{num_groups}")),
+ &batches,
+ |b, batches| {
+ b.iter_batched_ref(
+ || {
+ (
+ create_group_values(&schema, vectorized),
+ Vec::<usize>::with_capacity(DEFAULT_BATCH_SIZE),
+ )
+ },
+ |(gv, groups)| bench_intern(gv, batches, groups),
+ criterion::BatchSize::LargeInput,
+ );
+ },
+ );
+ }
+ }
+ group.finish();
+}
+
+fn make_interval_schema() -> SchemaRef {
+ Arc::new(Schema::new(vec![
+ Field::new("iv", DataType::Interval(IntervalUnit::MonthDayNano), false),
+ Field::new("id", DataType::Int32, false),
+ ]))
+}
+
+/// Generate `(Interval(MonthDayNano), Int32)` batches with `num_distinct_groups`
+/// distinct keys.
+///
+/// Each distinct interval is `MonthDayNano(g, 0, 0)`. The `Int32` column is
+/// keyed identically so the combined cardinality equals `num_distinct_groups`.
+fn generate_interval_batches(
+ num_distinct_groups: usize,
+ num_rows: usize,
+ batch_size: usize,
+) -> Vec<Vec<ArrayRef>> {
+ let num_full_batches = num_rows / batch_size;
+ let remainder = num_rows % batch_size;
+ let num_batches = num_full_batches + if remainder > 0 { 1 } else { 0 };
+
+ (0..num_batches)
+ .map(|batch_idx| {
+ let batch_start = batch_idx * batch_size;
+ let current_batch_size = if batch_idx == num_batches - 1 && remainder > 0 {
+ remainder
+ } else {
+ batch_size
+ };
+
+ let group_ids = (0..current_batch_size)
+ .map(|row| (batch_start + row) % num_distinct_groups);
+
+ let keys = IntervalMonthDayNanoArray::from_iter_values(
+ group_ids
+ .clone()
+ .map(|g| IntervalMonthDayNano::new(g as i32, 0, 0)),
+ );
+ let id: Int32Array = group_ids.map(|g| g as i32).collect();
+
+ vec![Arc::new(keys) as ArrayRef, Arc::new(id) as ArrayRef]
+ })
+ .collect()
+}
+
+/// Experiment 10: Group count sweep for an `(Interval, Int32)` key.
+///
+/// Exercises the primitive `GroupColumn` builder for `Interval` on the
+/// multi-column path (previously such a schema fell back to `GroupValuesRows`).
+fn bench_interval(c: &mut Criterion) {
+ let mut group = c.benchmark_group("interval");
+ group.sample_size(15);
+
+ let schema = make_interval_schema();
+
+ for num_groups in [1_000, 1_000_000] {
+ let batches =
+ generate_interval_batches(num_groups, 1_000_000, DEFAULT_BATCH_SIZE);
+
+ for vectorized in [true, false] {
+ let label = if vectorized {
+ "vectorized"
+ } else {
+ "row_based"
+ };
+ group.bench_with_input(
+ BenchmarkId::new(label, format!("grp_{num_groups}")),
+ &batches,
+ |b, batches| {
+ b.iter_batched_ref(
+ || {
+ (
+ create_group_values(&schema, vectorized),
+ Vec::<usize>::with_capacity(DEFAULT_BATCH_SIZE),
+ )
+ },
+ |(gv, groups)| bench_intern(gv, batches, groups),
+ criterion::BatchSize::LargeInput,
+ );
+ },
+ );
+ }
+ }
+ group.finish();
+}
+
+fn make_decimal256_schema() -> SchemaRef {
+ Arc::new(Schema::new(vec![
+ Field::new("dec", DataType::Decimal256(50, 0), false),
+ Field::new("id", DataType::Int32, false),
+ ]))
+}
+
+/// Generate `(Decimal256(50, 0), Int32)` batches with `num_distinct_groups`
+/// distinct keys.
+///
+/// Each distinct value is `i256::from_i128(g)`, and precision > 38 keeps it a
+/// genuine `Decimal256`. The `Int32` column is keyed identically so the combined
+/// cardinality equals `num_distinct_groups`.
+fn generate_decimal256_batches(
+ num_distinct_groups: usize,
+ num_rows: usize,
+ batch_size: usize,
+) -> Vec<Vec<ArrayRef>> {
+ let num_full_batches = num_rows / batch_size;
+ let remainder = num_rows % batch_size;
+ let num_batches = num_full_batches + if remainder > 0 { 1 } else { 0 };
+
+ (0..num_batches)
+ .map(|batch_idx| {
+ let batch_start = batch_idx * batch_size;
+ let current_batch_size = if batch_idx == num_batches - 1 && remainder > 0 {
+ remainder
+ } else {
+ batch_size
+ };
+
+ let group_ids = (0..current_batch_size)
+ .map(|row| (batch_start + row) % num_distinct_groups);
+
+ let keys = Decimal256Array::from_iter_values(
+ group_ids.clone().map(|g| i256::from_i128(g as i128)),
+ )
+ .with_precision_and_scale(50, 0)
+ .unwrap();
+ let id: Int32Array = group_ids.map(|g| g as i32).collect();
+
+ vec![Arc::new(keys) as ArrayRef, Arc::new(id) as ArrayRef]
+ })
+ .collect()
+}
+
+/// Experiment 11: Group count sweep for a `(Decimal256, Int32)` key.
+///
+/// Exercises the primitive `GroupColumn` builder for `Decimal256` (32-byte
+/// `i256` native) on the multi-column path (previously such a schema fell back
+/// to `GroupValuesRows`).
+fn bench_decimal256(c: &mut Criterion) {
+ let mut group = c.benchmark_group("decimal256");
+ group.sample_size(15);
+
+ let schema = make_decimal256_schema();
+
+ for num_groups in [1_000, 1_000_000] {
+ let batches =
+ generate_decimal256_batches(num_groups, 1_000_000, DEFAULT_BATCH_SIZE);
+
+ for vectorized in [true, false] {
+ let label = if vectorized {
+ "vectorized"
+ } else {
+ "row_based"
+ };
+ group.bench_with_input(
+ BenchmarkId::new(label, format!("grp_{num_groups}")),
+ &batches,
+ |b, batches| {
+ b.iter_batched_ref(
+ || {
+ (
+ create_group_values(&schema, vectorized),
+ Vec::<usize>::with_capacity(DEFAULT_BATCH_SIZE),
+ )
+ },
+ |(gv, groups)| bench_intern(gv, batches, groups),
+ criterion::BatchSize::LargeInput,
+ );
+ },
+ );
+ }
+ }
+ group.finish();
+}
+
+criterion_group!(
+ benches,
+ bench_issue_17850_regression,
+ bench_low_cardinality,
+ bench_batch_size_sensitivity,
+ bench_column_scaling,
+ bench_high_cardinality_scaling,
+ bench_group_count_sweep,
+ bench_fixed_size_binary,
+ bench_float16,
+ bench_duration,
+ bench_interval,
+ bench_decimal256,
+);
+criterion_main!(benches);
diff --git a/native/vendor/datafusion-physical-plan/benches/partial_ordering.rs b/native/vendor/datafusion-physical-plan/benches/partial_ordering.rs
new file mode 100644
index 0000000000..bdadd6274b
--- /dev/null
+++ b/native/vendor/datafusion-physical-plan/benches/partial_ordering.rs
@@ -0,0 +1,60 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use arrow::array::{ArrayRef, Int32Array};
+use datafusion_physical_plan::aggregates::order::GroupOrderingPartial;
+
+use criterion::{Criterion, criterion_group, criterion_main};
+
+const BATCH_SIZE: usize = 8192;
+
+fn create_test_arrays(num_columns: usize) -> Vec<ArrayRef> {
+ (0..num_columns)
+ .map(|i| {
+ Arc::new(Int32Array::from_iter_values(
+ (0..BATCH_SIZE as i32).map(|x| x * (i + 1) as i32),
+ )) as ArrayRef
+ })
+ .collect()
+}
+fn bench_new_groups(c: &mut Criterion) {
+ let mut group = c.benchmark_group("group_ordering_partial");
+
+ // Test with 1, 2, 4, and 8 order indices
+ for num_columns in [1, 2, 4, 8] {
+ let order_indices: Vec<usize> = (0..num_columns).collect();
+
+ group.bench_function(format!("order_indices_{num_columns}"), |b| {
+ let batch_group_values = create_test_arrays(num_columns);
+ let group_indices: Vec<usize> = (0..BATCH_SIZE).collect();
+
+ b.iter(|| {
+ let mut ordering =
+ GroupOrderingPartial::try_new(order_indices.clone()).unwrap();
+ ordering
+ .new_groups(&batch_group_values, &group_indices, BATCH_SIZE)
+ .unwrap();
+ });
+ });
+ }
+ group.finish();
+}
+
+criterion_group!(benches, bench_new_groups);
+criterion_main!(benches);
diff --git a/native/vendor/datafusion-physical-plan/benches/sort_merge_join.rs b/native/vendor/datafusion-physical-plan/benches/sort_merge_join.rs
new file mode 100644
index 0000000000..82610b2a54
--- /dev/null
+++ b/native/vendor/datafusion-physical-plan/benches/sort_merge_join.rs
@@ -0,0 +1,204 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Criterion benchmarks for Sort Merge Join
+//!
+//! These benchmarks measure the join kernel in isolation by feeding
+//! pre-sorted RecordBatches directly into SortMergeJoinExec, avoiding
+//! sort / scan overhead.
+
+use std::sync::Arc;
+
+use arrow::array::{Int64Array, RecordBatch, StringArray};
+use arrow::compute::SortOptions;
+use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
+use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
+use datafusion_common::NullEquality;
+use datafusion_execution::TaskContext;
+use datafusion_physical_expr::expressions::col;
+use datafusion_physical_plan::collect;
+use datafusion_physical_plan::joins::{SortMergeJoinExec, utils::JoinOn};
+use datafusion_physical_plan::test::TestMemoryExec;
+use tokio::runtime::Runtime;
+
+/// Build pre-sorted RecordBatches (split into ~8192-row chunks).
+///
+/// Schema: (key: Int64, data: Int64, payload: Utf8)
+///
+/// `key_mod` controls distinct key count: key = row_index % key_mod.
+fn build_sorted_batches(
+ num_rows: usize,
+ key_mod: usize,
+ schema: &SchemaRef,
+) -> Vec<RecordBatch> {
+ let mut rows: Vec<(i64, i64)> = (0..num_rows)
+ .map(|i| ((i % key_mod) as i64, i as i64))
+ .collect();
+ rows.sort();
+
+ let keys: Vec<i64> = rows.iter().map(|(k, _)| *k).collect();
+ let data: Vec<i64> = rows.iter().map(|(_, d)| *d).collect();
+ let payload: Vec<String> = data.iter().map(|d| format!("val_{d}")).collect();
+
+ let batch = RecordBatch::try_new(
+ Arc::clone(schema),
+ vec![
+ Arc::new(Int64Array::from(keys)),
+ Arc::new(Int64Array::from(data)),
+ Arc::new(StringArray::from(payload)),
+ ],
+ )
+ .unwrap();
+
+ let batch_size = 8192;
+ let mut batches = Vec::new();
+ let mut offset = 0;
+ while offset < batch.num_rows() {
+ let len = (batch.num_rows() - offset).min(batch_size);
+ batches.push(batch.slice(offset, len));
+ offset += len;
+ }
+ batches
+}
+
+fn make_exec(
+ batches: &[RecordBatch],
+ schema: &SchemaRef,
+) -> Arc<dyn datafusion_physical_plan::ExecutionPlan> {
+ TestMemoryExec::try_new_exec(&[batches.to_vec()], Arc::clone(schema), None).unwrap()
+}
+
+fn schema() -> SchemaRef {
+ Arc::new(Schema::new(vec![
+ Field::new("key", DataType::Int64, false),
+ Field::new("data", DataType::Int64, false),
+ Field::new("payload", DataType::Utf8, false),
+ ]))
+}
+
+fn do_join(
+ left: Arc<dyn datafusion_physical_plan::ExecutionPlan>,
+ right: Arc<dyn datafusion_physical_plan::ExecutionPlan>,
+ join_type: datafusion_common::JoinType,
+ rt: &Runtime,
+) -> usize {
+ let on: JoinOn = vec![(
+ col("key", &left.schema()).unwrap(),
+ col("key", &right.schema()).unwrap(),
+ )];
+ let join = SortMergeJoinExec::try_new(
+ left,
+ right,
+ on,
+ None,
+ join_type,
+ vec![SortOptions::default()],
+ NullEquality::NullEqualsNothing,
+ )
+ .unwrap();
+
+ let task_ctx = Arc::new(TaskContext::default());
+ rt.block_on(async {
+ let batches = collect(Arc::new(join), task_ctx).await.unwrap();
+ batches.iter().map(|b| b.num_rows()).sum()
+ })
+}
+
+fn bench_smj(c: &mut Criterion) {
+ let rt = Runtime::new().unwrap();
+ let s = schema();
+
+ let mut group = c.benchmark_group("sort_merge_join");
+
+ // 1:1 Inner Join — 100K rows each, unique keys
+ // Best case for contiguous-range optimization: every index array is [0,1,2,...].
+ {
+ let n = 100_000;
+ let left_batches = build_sorted_batches(n, n, &s);
+ let right_batches = build_sorted_batches(n, n, &s);
+ group.bench_function(BenchmarkId::new("inner_1to1", n), |b| {
+ b.iter(|| {
+ let left = make_exec(&left_batches, &s);
+ let right = make_exec(&right_batches, &s);
+ do_join(left, right, datafusion_common::JoinType::Inner, &rt)
+ })
+ });
+ }
+
+ // 1:10 Inner Join — 100K left, 100K right, 10K distinct keys
+ {
+ let n = 100_000;
+ let key_mod = 10_000;
+ let left_batches = build_sorted_batches(n, key_mod, &s);
+ let right_batches = build_sorted_batches(n, key_mod, &s);
+ group.bench_function(BenchmarkId::new("inner_1to10", n), |b| {
+ b.iter(|| {
+ let left = make_exec(&left_batches, &s);
+ let right = make_exec(&right_batches, &s);
+ do_join(left, right, datafusion_common::JoinType::Inner, &rt)
+ })
+ });
+ }
+
+ // Left Join — 100K each, ~5% unmatched on left
+ {
+ let n = 100_000;
+ let left_batches = build_sorted_batches(n, n + n / 20, &s);
+ let right_batches = build_sorted_batches(n, n, &s);
+ group.bench_function(BenchmarkId::new("left_1to1_unmatched", n), |b| {
+ b.iter(|| {
+ let left = make_exec(&left_batches, &s);
+ let right = make_exec(&right_batches, &s);
+ do_join(left, right, datafusion_common::JoinType::Left, &rt)
+ })
+ });
+ }
+
+ // Left Semi Join — 100K left, 100K right, 10K keys
+ {
+ let n = 100_000;
+ let key_mod = 10_000;
+ let left_batches = build_sorted_batches(n, key_mod, &s);
+ let right_batches = build_sorted_batches(n, key_mod, &s);
+ group.bench_function(BenchmarkId::new("left_semi_1to10", n), |b| {
+ b.iter(|| {
+ let left = make_exec(&left_batches, &s);
+ let right = make_exec(&right_batches, &s);
+ do_join(left, right, datafusion_common::JoinType::LeftSemi, &rt)
+ })
+ });
+ }
+
+ // Left Anti Join — 100K left, 100K right, partial match
+ {
+ let n = 100_000;
+ let left_batches = build_sorted_batches(n, n + n / 5, &s);
+ let right_batches = build_sorted_batches(n, n, &s);
+ group.bench_function(BenchmarkId::new("left_anti_partial", n), |b| {
+ b.iter(|| {
+ let left = make_exec(&left_batches, &s);
+ let right = make_exec(&right_batches, &s);
+ do_join(left, right, datafusion_common::JoinType::LeftAnti, &rt)
+ })
+ });
+ }
+
+ group.finish();
+}
+
+criterion_group!(benches, bench_smj);
+criterion_main!(benches);
diff --git a/native/vendor/datafusion-physical-plan/benches/sort_preserving_merge.rs b/native/vendor/datafusion-physical-plan/benches/sort_preserving_merge.rs
new file mode 100644
index 0000000000..76ebf230a3
--- /dev/null
+++ b/native/vendor/datafusion-physical-plan/benches/sort_preserving_merge.rs
@@ -0,0 +1,197 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use arrow::{
+ array::{ArrayRef, StringArray, UInt64Array},
+ record_batch::RecordBatch,
+};
+use arrow_schema::{SchemaRef, SortOptions};
+use criterion::{BatchSize, Criterion, criterion_group, criterion_main};
+use datafusion_execution::TaskContext;
+use datafusion_physical_expr::{LexOrdering, PhysicalSortExpr, expressions::col};
+use datafusion_physical_plan::test::TestMemoryExec;
+use datafusion_physical_plan::{
+ collect, sorts::sort_preserving_merge::SortPreservingMergeExec,
+};
+
+use std::sync::Arc;
+
+const BENCH_ROWS: usize = 1_000_000; // 1 million rows
+
+fn get_large_string(idx: usize) -> String {
+ let base_content = [
+ concat!(
+ "# Advanced Topics in Computer Science\n\n",
+ "## Summary\nThis article explores complex system design patterns and...\n\n",
+ "```rust\nfn process_data(data: &mut [i32]) {\n // Parallel processing example\n data.par_iter_mut().for_each(|x| *x *= 2);\n}\n```\n\n",
+ "## Performance Considerations\nWhen implementing concurrent systems...\n"
+ ),
+ concat!(
+ "## API Documentation\n\n",
+ "```json\n{\n \"endpoint\": \"/api/v2/users\",\n \"methods\": [\"GET\", \"POST\"],\n \"parameters\": {\n \"page\": \"number\"\n }\n}\n```\n\n",
+ "# Authentication Guide\nSecure your API access using OAuth 2.0...\n"
+ ),
+ concat!(
+ "# Data Processing Pipeline\n\n",
+ "```python\nfrom multiprocessing import Pool\n\ndef main():\n with Pool(8) as p:\n results = p.map(process_item, data)\n```\n\n",
+ "## Summary of Optimizations\n1. Batch processing\n2. Memory pooling\n3. Concurrent I/O operations\n"
+ ),
+ concat!(
+ "# System Architecture Overview\n\n",
+ "## Components\n- Load Balancer\n- Database Cluster\n- Cache Service\n\n",
+ "```go\nfunc main() {\n router := gin.Default()\n router.GET(\"/api/health\", healthCheck)\n router.Run(\":8080\")\n}\n```\n"
+ ),
+ concat!(
+ "## Configuration Reference\n\n",
+ "```yaml\nserver:\n port: 8080\n max_threads: 32\n\ndatabase:\n url: postgres://user@prod-db:5432/main\n```\n\n",
+ "# Deployment Strategies\nBlue-green deployment patterns with...\n"
+ ),
+ ];
+ base_content[idx % base_content.len()].to_string()
+}
+
+fn generate_sorted_string_column(rows: usize) -> ArrayRef {
+ let mut values = Vec::with_capacity(rows);
+ for i in 0..rows {
+ values.push(get_large_string(i));
+ }
+ values.sort();
+ Arc::new(StringArray::from(values))
+}
+
+fn generate_sorted_u64_column(rows: usize) -> ArrayRef {
+ Arc::new(UInt64Array::from((0_u64..rows as u64).collect::<Vec<_>>()))
+}
+
+fn create_partitions<const IS_LARGE_COLUMN_TYPE: bool>(
+ num_partitions: usize,
+ num_columns: usize,
+ num_rows: usize,
+) -> Vec<Vec<RecordBatch>> {
+ (0..num_partitions)
+ .map(|_| {
+ let rows = (0..num_columns)
+ .map(|i| {
+ (
+ format!("col-{i}"),
+ if IS_LARGE_COLUMN_TYPE {
+ generate_sorted_string_column(num_rows)
+ } else {
+ generate_sorted_u64_column(num_rows)
+ },
+ )
+ })
+ .collect::<Vec<_>>();
+
+ let batch = RecordBatch::try_from_iter(rows).unwrap();
+ vec![batch]
+ })
+ .collect()
+}
+
+struct BenchData {
+ bench_name: String,
+ partitions: Vec<Vec<RecordBatch>>,
+ schema: SchemaRef,
+ sort_order: LexOrdering,
+}
+
+fn get_bench_data() -> Vec<BenchData> {
+ let mut ret = Vec::new();
+ let mut push_bench_data = |bench_name: &str, partitions: Vec<Vec<RecordBatch>>| {
+ let schema = partitions[0][0].schema();
+ // Define sort order (col1 ASC, col2 ASC, col3 ASC)
+ let sort_order = LexOrdering::new(schema.fields().iter().map(|field| {
+ PhysicalSortExpr::new(
+ col(field.name(), &schema).unwrap(),
+ SortOptions::default(),
+ )
+ }))
+ .unwrap();
+ ret.push(BenchData {
+ bench_name: bench_name.to_string(),
+ partitions,
+ schema,
+ sort_order,
+ });
+ };
+ // 1. single large string column
+ {
+ let partitions = create_partitions::<true>(3, 1, BENCH_ROWS);
+ push_bench_data("single_large_string_column_with_1m_rows", partitions);
+ }
+ // 2. single u64 column
+ {
+ let partitions = create_partitions::<false>(3, 1, BENCH_ROWS);
+ push_bench_data("single_u64_column_with_1m_rows", partitions);
+ }
+ // 3. multiple large string columns
+ {
+ let partitions = create_partitions::<true>(3, 3, BENCH_ROWS);
+ push_bench_data("multiple_large_string_columns_with_1m_rows", partitions);
+ }
+ // 4. multiple u64 columns
+ {
+ let partitions = create_partitions::<false>(3, 3, BENCH_ROWS);
+ push_bench_data("multiple_u64_columns_with_1m_rows", partitions);
+ }
+ ret
+}
+
+/// Add a benchmark to test the optimization effect of reusing Rows.
+/// Run this benchmark with:
+/// ```sh
+/// cargo bench --features="bench" --bench sort_preserving_merge -- --sample-size=10
+/// ```
+fn bench_merge_sorted_preserving(c: &mut Criterion) {
+ let task_ctx = Arc::new(TaskContext::default());
+ let bench_data = get_bench_data();
+ for data in bench_data.into_iter() {
+ let BenchData {
+ bench_name,
+ partitions,
+ schema,
+ sort_order,
+ } = data;
+ c.bench_function(
+ &format!("bench_merge_sorted_preserving/{bench_name}"),
+ |b| {
+ b.iter_batched(
+ || {
+ let exec = TestMemoryExec::try_new_exec(
+ &partitions,
+ schema.clone(),
+ None,
+ )
+ .unwrap();
+ Arc::new(SortPreservingMergeExec::new(sort_order.clone(), exec))
+ },
+ |merge_exec| {
+ let rt = tokio::runtime::Runtime::new().unwrap();
+ rt.block_on(async {
+ collect(merge_exec, task_ctx.clone()).await.unwrap();
+ });
+ },
+ BatchSize::LargeInput,
+ )
+ },
+ );
+ }
+}
+
+criterion_group!(benches, bench_merge_sorted_preserving);
+criterion_main!(benches);
diff --git a/native/vendor/datafusion-physical-plan/benches/spill_io.rs b/native/vendor/datafusion-physical-plan/benches/spill_io.rs
new file mode 100644
index 0000000000..ddd83ca565
--- /dev/null
+++ b/native/vendor/datafusion-physical-plan/benches/spill_io.rs
@@ -0,0 +1,581 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use arrow::array::{
+ Date32Builder, Decimal128Builder, Int32Builder, Int64Builder, RecordBatch,
+ StringBuilder,
+};
+use arrow::datatypes::{DataType, Field, Schema};
+use criterion::measurement::WallTime;
+use criterion::{
+ BatchSize, BenchmarkGroup, BenchmarkId, Criterion, criterion_group, criterion_main,
+};
+use datafusion_common::config::SpillCompression;
+use datafusion_common::human_readable_size;
+use datafusion_common::instant::Instant;
+use datafusion_execution::runtime_env::RuntimeEnv;
+use datafusion_physical_plan::SpillManager;
+use datafusion_physical_plan::common::collect;
+use datafusion_physical_plan::metrics::{ExecutionPlanMetricsSet, SpillMetrics};
+use rand::{Rng, SeedableRng};
+use std::sync::Arc;
+use tokio::runtime::Runtime;
+
+pub fn create_batch(num_rows: usize, allow_nulls: bool) -> RecordBatch {
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("c0", DataType::Int32, true),
+ Field::new("c1", DataType::Utf8, true),
+ Field::new("c2", DataType::Date32, true),
+ Field::new("c3", DataType::Decimal128(11, 2), true),
+ ]));
+
+ let mut a = Int32Builder::new();
+ let mut b = StringBuilder::new();
+ let mut c = Date32Builder::new();
+ let mut d = Decimal128Builder::new()
+ .with_precision_and_scale(11, 2)
+ .unwrap();
+
+ for i in 0..num_rows {
+ a.append_value(i as i32);
+ c.append_value(i as i32);
+ d.append_value((i * 1000000) as i128);
+ if allow_nulls && i % 10 == 0 {
+ b.append_null();
+ } else {
+ b.append_value(format!("this is string number {i}"));
+ }
+ }
+
+ let a = a.finish();
+ let b = b.finish();
+ let c = c.finish();
+ let d = d.finish();
+
+ RecordBatch::try_new(
+ schema.clone(),
+ vec![Arc::new(a), Arc::new(b), Arc::new(c), Arc::new(d)],
+ )
+ .unwrap()
+}
+
+// BENCHMARK: REVALIDATION OVERHEAD COMPARISON
+// ---------------------------------------------------------
+// To compare performance with/without Arrow IPC validation:
+//
+// 1. Locate the function `read_spill`
+// 2. Modify the `skip_validation` flag:
+// - Set to `false` to enable validation
+// 3. Rerun `cargo bench --bench spill_io`
+fn bench_spill_io(c: &mut Criterion) {
+ let env = Arc::new(RuntimeEnv::default());
+ let metrics = SpillMetrics::new(&ExecutionPlanMetricsSet::new(), 0);
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("c0", DataType::Int32, true),
+ Field::new("c1", DataType::Utf8, true),
+ Field::new("c2", DataType::Date32, true),
+ Field::new("c3", DataType::Decimal128(11, 2), true),
+ ]));
+ let spill_manager = SpillManager::new(env, metrics, schema);
+
+ let mut group = c.benchmark_group("spill_io");
+ let rt = Runtime::new().unwrap();
+
+ group.bench_with_input(
+ BenchmarkId::new("StreamReader/read_100", ""),
+ &spill_manager,
+ |b, spill_manager| {
+ b.iter_batched(
+ // Setup phase: Create fresh state for each benchmark iteration.
+ // - generate an ipc file.
+ // This ensures each iteration starts with clean resources.
+ || {
+ let batch = create_batch(8192, true);
+ spill_manager
+ .spill_record_batch_and_finish(&vec![batch; 100], "Test")
+ .unwrap()
+ .unwrap()
+ },
+ // Benchmark phase:
+ // - Execute the read operation via SpillManager
+ // - Wait for the consumer to finish processing
+ |spill_file| {
+ rt.block_on(async {
+ let stream = spill_manager
+ .read_spill_as_stream(spill_file, None)
+ .unwrap();
+ let _ = collect(stream).await.unwrap();
+ })
+ },
+ BatchSize::LargeInput,
+ )
+ },
+ );
+ group.finish();
+}
+
+// Generate `num_batches` RecordBatches mimicking TPC-H Q2's partial aggregate result:
+// GROUP BY ps_partkey -> MIN(ps_supplycost)
+fn create_q2_like_batches(
+ num_batches: usize,
+ num_rows: usize,
+) -> (Arc<Schema>, Vec<RecordBatch>) {
+ // use fixed seed
+ let seed = 2;
+ let mut rng = rand::rngs::StdRng::seed_from_u64(seed);
+ let mut batches = Vec::with_capacity(num_batches);
+
+ let mut current_key = 400000_i64;
+
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("ps_partkey", DataType::Int64, false),
+ Field::new("min_ps_supplycost", DataType::Decimal128(15, 2), true),
+ ]));
+
+ for _ in 0..num_batches {
+ let mut partkey_builder = Int64Builder::new();
+ let mut cost_builder = Decimal128Builder::new()
+ .with_precision_and_scale(15, 2)
+ .unwrap();
+
+ for _ in 0..num_rows {
+ // Occasionally skip a few partkey values to simulate sparsity
+ let jump = if rng.random_bool(0.05) {
+ rng.random_range(2..10)
+ } else {
+ 1
+ };
+ current_key += jump;
+
+ let supply_cost = rng.random_range(10_00..100_000) as i128;
+
+ partkey_builder.append_value(current_key);
+ cost_builder.append_value(supply_cost);
+ }
+
+ let batch = RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![
+ Arc::new(partkey_builder.finish()),
+ Arc::new(cost_builder.finish()),
+ ],
+ )
+ .unwrap();
+
+ batches.push(batch);
+ }
+
+ (schema, batches)
+}
+
+/// Generate `num_batches` RecordBatches mimicking TPC-H Q16's partial aggregate result:
+/// GROUP BY (p_brand, p_type, p_size) -> COUNT(DISTINCT ps_suppkey)
+pub fn create_q16_like_batches(
+ num_batches: usize,
+ num_rows: usize,
+) -> (Arc<Schema>, Vec<RecordBatch>) {
+ let seed = 16;
+ let mut rng = rand::rngs::StdRng::seed_from_u64(seed);
+ let mut batches = Vec::with_capacity(num_batches);
+
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("p_brand", DataType::Utf8, false),
+ Field::new("p_type", DataType::Utf8, false),
+ Field::new("p_size", DataType::Int32, false),
+ Field::new("alias1", DataType::Int64, false), // COUNT(DISTINCT ps_suppkey)
+ ]));
+
+ // Representative string pools
+ let brands = ["Brand#32", "Brand#33", "Brand#41", "Brand#42", "Brand#55"];
+ let types = [
+ "PROMO ANODIZED NICKEL",
+ "STANDARD BRUSHED NICKEL",
+ "PROMO POLISHED COPPER",
+ "ECONOMY ANODIZED BRASS",
+ "LARGE BURNISHED COPPER",
+ "STANDARD POLISHED TIN",
+ "SMALL PLATED STEEL",
+ "MEDIUM POLISHED COPPER",
+ ];
+ let sizes = [3, 9, 14, 19, 23, 36, 45, 49];
+
+ for _ in 0..num_batches {
+ let mut brand_builder = StringBuilder::new();
+ let mut type_builder = StringBuilder::new();
+ let mut size_builder = Int32Builder::new();
+ let mut count_builder = Int64Builder::new();
+
+ for _ in 0..num_rows {
+ let brand = brands[rng.random_range(0..brands.len())];
+ let ptype = types[rng.random_range(0..types.len())];
+ let size = sizes[rng.random_range(0..sizes.len())];
+ let count = rng.random_range(1000..100_000);
+
+ brand_builder.append_value(brand);
+ type_builder.append_value(ptype);
+ size_builder.append_value(size);
+ count_builder.append_value(count);
+ }
+
+ let batch = RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![
+ Arc::new(brand_builder.finish()),
+ Arc::new(type_builder.finish()),
+ Arc::new(size_builder.finish()),
+ Arc::new(count_builder.finish()),
+ ],
+ )
+ .unwrap();
+
+ batches.push(batch);
+ }
+
+ (schema, batches)
+}
+
+// Generate `num_batches` RecordBatches mimicking TPC-H Q20's partial aggregate result:
+// GROUP BY (l_partkey, l_suppkey) -> SUM(l_quantity)
+fn create_q20_like_batches(
+ num_batches: usize,
+ num_rows: usize,
+) -> (Arc<Schema>, Vec<RecordBatch>) {
+ let seed = 20;
+ let mut rng = rand::rngs::StdRng::seed_from_u64(seed);
+ let mut batches = Vec::with_capacity(num_batches);
+
+ let mut current_partkey = 400000_i64;
+
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("l_partkey", DataType::Int64, false),
+ Field::new("l_suppkey", DataType::Int64, false),
+ Field::new("sum_l_quantity", DataType::Decimal128(25, 2), true),
+ ]));
+
+ for _ in 0..num_batches {
+ let mut partkey_builder = Int64Builder::new();
+ let mut suppkey_builder = Int64Builder::new();
+ let mut quantity_builder = Decimal128Builder::new()
+ .with_precision_and_scale(25, 2)
+ .unwrap();
+
+ for _ in 0..num_rows {
+ // Occasionally skip a few partkey values to simulate sparsity
+ let partkey_jump = if rng.random_bool(0.03) {
+ rng.random_range(2..6)
+ } else {
+ 1
+ };
+ current_partkey += partkey_jump;
+
+ let suppkey = rng.random_range(10_000..99_999);
+ let quantity = rng.random_range(500..20_000) as i128;
+
+ partkey_builder.append_value(current_partkey);
+ suppkey_builder.append_value(suppkey);
+ quantity_builder.append_value(quantity);
+ }
+
+ let batch = RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![
+ Arc::new(partkey_builder.finish()),
+ Arc::new(suppkey_builder.finish()),
+ Arc::new(quantity_builder.finish()),
+ ],
+ )
+ .unwrap();
+
+ batches.push(batch);
+ }
+
+ (schema, batches)
+}
+
+/// Generate `num_batches` wide RecordBatches resembling sort-tpch Q10 for benchmarking.
+/// This includes multiple numeric, date, and Utf8View columns (15 total).
+pub fn create_wide_batches(
+ num_batches: usize,
+ num_rows: usize,
+) -> (Arc<Schema>, Vec<RecordBatch>) {
+ let seed = 10;
+ let mut rng = rand::rngs::StdRng::seed_from_u64(seed);
+ let mut batches = Vec::with_capacity(num_batches);
+
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("l_linenumber", DataType::Int32, false),
+ Field::new("l_suppkey", DataType::Int64, false),
+ Field::new("l_orderkey", DataType::Int64, false),
+ Field::new("l_partkey", DataType::Int64, false),
+ Field::new("l_quantity", DataType::Decimal128(15, 2), false),
+ Field::new("l_extendedprice", DataType::Decimal128(15, 2), false),
+ Field::new("l_discount", DataType::Decimal128(15, 2), false),
+ Field::new("l_tax", DataType::Decimal128(15, 2), false),
+ Field::new("l_returnflag", DataType::Utf8, false),
+ Field::new("l_linestatus", DataType::Utf8, false),
+ Field::new("l_shipdate", DataType::Date32, false),
+ Field::new("l_commitdate", DataType::Date32, false),
+ Field::new("l_receiptdate", DataType::Date32, false),
+ Field::new("l_shipinstruct", DataType::Utf8, false),
+ Field::new("l_shipmode", DataType::Utf8, false),
+ ]));
+
+ for _ in 0..num_batches {
+ let mut linenum = Int32Builder::new();
+ let mut suppkey = Int64Builder::new();
+ let mut orderkey = Int64Builder::new();
+ let mut partkey = Int64Builder::new();
+ let mut quantity = Decimal128Builder::new()
+ .with_precision_and_scale(15, 2)
+ .unwrap();
+ let mut extprice = Decimal128Builder::new()
+ .with_precision_and_scale(15, 2)
+ .unwrap();
+ let mut discount = Decimal128Builder::new()
+ .with_precision_and_scale(15, 2)
+ .unwrap();
+ let mut tax = Decimal128Builder::new()
+ .with_precision_and_scale(15, 2)
+ .unwrap();
+ let mut retflag = StringBuilder::new();
+ let mut linestatus = StringBuilder::new();
+ let mut shipdate = Date32Builder::new();
+ let mut commitdate = Date32Builder::new();
+ let mut receiptdate = Date32Builder::new();
+ let mut shipinstruct = StringBuilder::new();
+ let mut shipmode = StringBuilder::new();
+
+ let return_flags = ["A", "N", "R"];
+ let statuses = ["F", "O"];
+ let instructs = ["DELIVER IN PERSON", "COLLECT COD", "NONE"];
+ let modes = ["TRUCK", "MAIL", "SHIP", "RAIL", "AIR"];
+
+ for i in 0..num_rows {
+ linenum.append_value((i % 7) as i32);
+ suppkey.append_value(rng.random_range(0..100_000));
+ orderkey.append_value(1_000_000 + i as i64);
+ partkey.append_value(rng.random_range(0..200_000));
+
+ quantity.append_value(rng.random_range(100..10000) as i128);
+ extprice.append_value(rng.random_range(1_000..1_000_000) as i128);
+ discount.append_value(rng.random_range(0..10000) as i128);
+ tax.append_value(rng.random_range(0..5000) as i128);
+
+ retflag.append_value(return_flags[rng.random_range(0..return_flags.len())]);
+ linestatus.append_value(statuses[rng.random_range(0..statuses.len())]);
+
+ let base_date = 10_000;
+ shipdate.append_value(base_date + (i % 1000) as i32);
+ commitdate.append_value(base_date + (i % 1000) as i32 + 1);
+ receiptdate.append_value(base_date + (i % 1000) as i32 + 2);
+
+ shipinstruct.append_value(instructs[rng.random_range(0..instructs.len())]);
+ shipmode.append_value(modes[rng.random_range(0..modes.len())]);
+ }
+
+ let batch = RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![
+ Arc::new(linenum.finish()),
+ Arc::new(suppkey.finish()),
+ Arc::new(orderkey.finish()),
+ Arc::new(partkey.finish()),
+ Arc::new(quantity.finish()),
+ Arc::new(extprice.finish()),
+ Arc::new(discount.finish()),
+ Arc::new(tax.finish()),
+ Arc::new(retflag.finish()),
+ Arc::new(linestatus.finish()),
+ Arc::new(shipdate.finish()),
+ Arc::new(commitdate.finish()),
+ Arc::new(receiptdate.finish()),
+ Arc::new(shipinstruct.finish()),
+ Arc::new(shipmode.finish()),
+ ],
+ )
+ .unwrap();
+ batches.push(batch);
+ }
+ (schema, batches)
+}
+
+// Benchmarks spill write + read performance across multiple compression codecs
+// using realistic input data inspired by TPC-H aggregate spill scenarios.
+//
+// This function prepares synthetic RecordBatches that mimic the schema and distribution
+// of intermediate aggregate results from representative TPC-H queries (Q2, Q16, Q20) and sort-tpch Q10.
+// The schemas of these batches are:
+// Q2 [Int64, Decimal128]
+// Q16 [Utf8, Utf8, Int32, Int64]
+// Q20 [Int64, Int64, Decimal128]
+// sort-tpch Q10 (wide batch) [Int32, Int64 * 3, Decimal128 * 4, Date * 3, Utf8 * 4]
+// For each dataset:
+// - It evaluates spill performance under different compression codecs (e.g., Uncompressed, Zstd, LZ4).
+// - It measures end-to-end spill write + read performance using Criterion.
+// - It prints the observed memory-to-disk compression ratio for each codec.
+//
+// This helps evaluate the tradeoffs between compression ratio and runtime overhead for various codecs.
+fn bench_spill_compression(c: &mut Criterion) {
+ let env = Arc::new(RuntimeEnv::default());
+ let mut group = c.benchmark_group("spill_compression");
+ let rt = Runtime::new().unwrap();
+ let compressions = vec![
+ SpillCompression::Uncompressed,
+ SpillCompression::Zstd,
+ SpillCompression::Lz4Frame,
+ ];
+
+ // Modify these values to change data volume. Note that each batch contains `num_rows` rows.
+ let num_batches = 50;
+ let num_rows = 8192;
+
+ // Q2 [Int64, Decimal128]
+ let (schema, batches) = create_q2_like_batches(num_batches, num_rows);
+ benchmark_spill_batches_for_all_codec(
+ &mut group,
+ "q2",
+ batches,
+ &compressions,
+ &rt,
+ env.clone(),
+ schema,
+ );
+ // Q16 [Utf8, Utf8, Int32, Int64]
+ let (schema, batches) = create_q16_like_batches(num_batches, num_rows);
+ benchmark_spill_batches_for_all_codec(
+ &mut group,
+ "q16",
+ batches,
+ &compressions,
+ &rt,
+ env.clone(),
+ schema,
+ );
+ // Q20 [Int64, Int64, Decimal128]
+ let (schema, batches) = create_q20_like_batches(num_batches, num_rows);
+ benchmark_spill_batches_for_all_codec(
+ &mut group,
+ "q20",
+ batches,
+ &compressions,
+ &rt,
+ env.clone(),
+ schema,
+ );
+ // sort-tpch Q10 (wide batch) [Int32, Int64 * 3, Decimal128 * 4, Date * 3, Utf8 * 4]
+ let (schema, batches) = create_wide_batches(num_batches, num_rows);
+ benchmark_spill_batches_for_all_codec(
+ &mut group,
+ "wide",
+ batches,
+ &compressions,
+ &rt,
+ env,
+ schema,
+ );
+ group.finish();
+}
+
+#[expect(clippy::needless_pass_by_value)]
+fn benchmark_spill_batches_for_all_codec(
+ group: &mut BenchmarkGroup<'_, WallTime>,
+ batch_label: &str,
+ batches: Vec<RecordBatch>,
+ compressions: &[SpillCompression],
+ rt: &Runtime,
+ env: Arc<RuntimeEnv>,
+ schema: Arc<Schema>,
+) {
+ let mem_bytes: usize = batches.iter().map(|b| b.get_array_memory_size()).sum();
+
+ for &compression in compressions {
+ let metrics = SpillMetrics::new(&ExecutionPlanMetricsSet::new(), 0);
+ let spill_manager =
+ SpillManager::new(Arc::clone(&env), metrics.clone(), Arc::clone(&schema))
+ .with_compression_type(compression);
+
+ let bench_id = BenchmarkId::new(batch_label, compression.to_string());
+ group.bench_with_input(bench_id, &spill_manager, |b, spill_manager| {
+ b.iter_batched(
+ || batches.clone(),
+ |batches| {
+ rt.block_on(async {
+ let spill_file = spill_manager
+ .spill_record_batch_and_finish(
+ &batches,
+ &format!("{batch_label}_{compression}"),
+ )
+ .unwrap()
+ .unwrap();
+ let stream = spill_manager
+ .read_spill_as_stream(spill_file, None)
+ .unwrap();
+ let _ = collect(stream).await.unwrap();
+ })
+ },
+ BatchSize::LargeInput,
+ )
+ });
+
+ // Run Spilling Read & Write once more to read file size & calculate bandwidth
+ let start = Instant::now();
+
+ let spill_file = spill_manager
+ .spill_record_batch_and_finish(
+ &batches,
+ &format!("{batch_label}_{compression}"),
+ )
+ .unwrap()
+ .unwrap();
+
+ // calculate write_throughput (includes both compression and I/O time) based on in memory batch size
+ let write_time = start.elapsed();
+ let write_throughput = (mem_bytes as u128 / write_time.as_millis().max(1)) * 1000;
+
+ // calculate compression ratio
+ let disk_bytes = std::fs::metadata(spill_file.path().unwrap())
+ .expect("metadata read fail")
+ .len() as usize;
+ let ratio = mem_bytes as f64 / disk_bytes.max(1) as f64;
+
+ // calculate read_throughput (includes both compression and I/O time) based on in memory batch size
+ let rt = Runtime::new().unwrap();
+ let start = Instant::now();
+ rt.block_on(async {
+ let stream = spill_manager
+ .read_spill_as_stream(spill_file, None)
+ .unwrap();
+ let _ = collect(stream).await.unwrap();
+ });
+ let read_time = start.elapsed();
+ let read_throughput = (mem_bytes as u128 / read_time.as_millis().max(1)) * 1000;
+
+ println!(
+ "[{} | {:?}] mem: {}| disk: {}| compression ratio: {:.3}x| throughput: (w) {}/s (r) {}/s",
+ batch_label,
+ compression,
+ human_readable_size(mem_bytes),
+ human_readable_size(disk_bytes),
+ ratio,
+ human_readable_size(write_throughput as usize),
+ human_readable_size(read_throughput as usize),
+ );
+ }
+}
+
+criterion_group!(benches, bench_spill_io, bench_spill_compression);
+criterion_main!(benches);
diff --git a/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/common.rs b/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/common.rs
new file mode 100644
index 0000000000..91e9d6555c
--- /dev/null
+++ b/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/common.rs
@@ -0,0 +1,693 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::marker::PhantomData;
+use std::sync::Arc;
+
+use arrow::array::{ArrayRef, AsArray, new_null_array};
+use arrow::datatypes::SchemaRef;
+use arrow::record_batch::RecordBatch;
+use datafusion_common::{Result, internal_err};
+use datafusion_execution::memory_pool::proxy::VecAllocExt;
+use datafusion_expr::{EmitTo, GroupsAccumulator};
+use datafusion_physical_expr::aggregate::AggregateFunctionExpr;
+
+use crate::PhysicalExpr;
+use crate::aggregates::group_values::{GroupByMetrics, GroupValues, new_group_values};
+use crate::aggregates::grouped_hash_stream::create_group_accumulator;
+use crate::aggregates::order::GroupOrdering;
+use crate::aggregates::{
+ AggregateExec, PhysicalGroupBy, aggregate_expressions, evaluate_group_by,
+};
+
+/// Marker for raw rows -> partial state aggregation.
+pub(in crate::aggregates) struct PartialMarker;
+/// Marker for raw rows -> final value aggregation.
+pub(in crate::aggregates) struct SingleMarker;
+/// Marker for partial state -> partial state aggregation.
+pub(in crate::aggregates) struct PartialReduceMarker;
+/// Marker for raw rows -> partial state conversion without aggregation.
+pub(in crate::aggregates) struct PartialSkipMarker;
+/// Marker for partial state -> final value aggregation.
+pub(in crate::aggregates) struct FinalMarker;
+
+/// Grouped hash table shared by the partial and final paths.
+///
+/// While building, it consumes input batches and updates group / accumulator
+/// state. While outputting, it incrementally drains that state into output
+/// batches.
+///
+/// # Logical and Physical Model
+///
+/// Logically, this is a hash table that maps { group keys -> accumulator states }
+/// For example, `AVG(v) GROUP BY k` stores one entry per `k`, where each
+/// entry owns the `sum(v)` and `count(v)` state needed to compute the final
+/// average.
+///
+/// Physically, the group keys and accumulators are backed by [`GroupValues`] and
+/// [`GroupsAccumulator`]. Both use columnar storage so aggregation can stay
+/// vectorized.
+///
+/// # Marker Type
+/// `AggrMode` selects the aggregate semantics.
+///
+/// e.g. `AggregateHashTable::<PartialMarker>::new(...)` creates an aggregate hash table
+/// for the partial hash aggregate stage, the input schema is raw rows and output
+/// schema is intermediate states.
+///
+/// It is a zero-sized compile-time marker, so each stage keeps its update logic
+/// in a separate impl block, to make the behavior difference explicit.
+pub(in crate::aggregates) struct AggregateHashTable<AggrMode> {
+ /// Grouping and accumulator-specific timing metrics.
+ pub(super) group_by_metrics: GroupByMetrics,
+
+ /// Raw input schema, used to evaluate expressions and synthesize empty
+ /// grouping-set rows.
+ pub(super) input_schema: SchemaRef,
+
+ /// Output schema: group columns followed by aggregate state or final values.
+ pub(super) output_schema: SchemaRef,
+
+ /// Intermediate-state schema used when memory pressure requires the table
+ /// to spill its current state.
+ pub(super) state_schema: SchemaRef,
+
+ /// Maximum rows per emitted output batch, from config `batch_size`.
+ pub(super) batch_size: usize,
+
+ /// Lifecycle-specific state: building stage / outputting stage.
+ pub(super) state: AggregateHashTableState,
+
+ pub(super) _mode: PhantomData<AggrMode>,
+}
+
+/// Methods shared by all aggregate hash table modes.
+impl<AggrMode> AggregateHashTable<AggrMode> {
+ pub(super) fn new_with_filters(
+ agg: &AggregateExec,
+ partition: usize,
+ output_schema: SchemaRef,
+ state_schema: SchemaRef,
+ batch_size: usize,
+ filters: Vec<Option<Arc<dyn PhysicalExpr>>>,
+ ) -> Result<Self> {
+ if batch_size == 0 {
+ return internal_err!("AggregateHashTable requires config batch_size >= 1");
+ }
+
+ let input_schema = agg.input().schema();
+ let aggregate_arguments = aggregate_expressions(
+ &agg.aggr_expr,
+ &agg.mode,
+ agg.group_by.num_group_exprs(),
+ )?;
+ let accumulators: Vec<_> = agg
+ .aggr_expr
+ .iter()
+ .zip(aggregate_arguments)
+ .zip(filters)
+ .map(|((agg_expr, arguments), filter)| {
+ let accumulator = create_group_accumulator(agg_expr)?;
+ Ok(HashAggregateAccumulator::new(
+ Arc::clone(agg_expr),
+ arguments,
+ filter,
+ accumulator,
+ ))
+ })
+ .collect::<Result<_>>()?;
+
+ let group_schema = agg.group_by.group_schema(&input_schema)?;
+ let group_values = new_group_values(group_schema, &GroupOrdering::None)?;
+
+ Ok(Self {
+ group_by_metrics: GroupByMetrics::new(&agg.metrics, partition),
+ input_schema,
+ output_schema,
+ state_schema,
+ batch_size,
+ state: AggregateHashTableState::Building(AggregateHashTableBuffer {
+ group_by: Arc::clone(&agg.group_by),
+ group_values,
+ batch_group_indices: Default::default(),
+ accumulators,
+ }),
+ _mode: PhantomData,
+ })
+ }
+
+ /// See comments in [`EvaluatedAggregateBatch`]
+ pub(super) fn evaluate_batch(
+ &self,
+ batch: &RecordBatch,
+ ) -> Result<EvaluatedAggregateBatch> {
+ let state = self.state.building();
+ let timer = self.group_by_metrics.time_calculating_group_ids.timer();
+ // outer vec: one per each grouping set
+ // inner vec: all group by exprs for the current grouping set
+ let grouping_set_args = evaluate_group_by(&state.group_by, batch)?;
+ drop(timer);
+
+ let timer = self.group_by_metrics.aggregate_arguments_time.timer();
+ // The evaluated args for each accumulator
+ let accumulator_args = self
+ .state
+ .building()
+ .accumulators
+ .iter()
+ .map(|acc| acc.evaluate_acc_args(batch))
+ .collect::<Result<Vec<_>>>()?;
+ drop(timer);
+
+ Ok(EvaluatedAggregateBatch {
+ grouping_set_args,
+ accumulator_args,
+ })
+ }
+
+ /// Aggregates one input batch after selecting the mode-specific accumulator
+ /// operation.
+ ///
+ /// Each aggregation mode chooses a different `aggregate_fn` according to its
+ /// semantics. For example, partial aggregation takes raw inputs, and update them
+ /// into stored partial states, so [`GroupsAccumulator::update_batch`] is used.
+ pub(super) fn aggregate_batch_inner(
+ &mut self,
+ batch: &RecordBatch,
+ aggregate_fn: AggregateBatchFn,
+ ) -> Result<()> {
+ let evaluated_batch = self.evaluate_batch(batch)?;
+ let state = self.state.building_mut();
+
+ let _timer = self.group_by_metrics.aggregation_time.timer();
+ for group_values in &evaluated_batch.grouping_set_args {
+ state
+ .group_values
+ .intern(group_values, &mut state.batch_group_indices)?;
+ let group_indices = &state.batch_group_indices;
+ let total_num_groups = state.group_values.len();
+
+ for (acc, values) in state
+ .accumulators
+ .iter_mut()
+ .zip(evaluated_batch.accumulator_args.iter())
+ {
+ aggregate_fn(acc, values, group_indices, total_num_groups)?;
+ }
+ }
+
+ Ok(())
+ }
+
+ /// Materializes the full output once, then returns it downstream incrementally
+ /// by slicing it into `batch_size` chunks.
+ ///
+ /// Each aggregation mode chooses a different `materialize_accumulator_fn`
+ /// according to its semantics. For example, partial aggregation emits
+ /// partial states to feed the final stage, so it uses [`GroupsAccumulator::state`].
+ ///
+ /// This is a temporary solution until blocked state management is implemented:
+ /// Issue: <https://github.com/apache/datafusion/issues/7065>
+ pub(super) fn next_output_batch_inner(
+ &mut self,
+ materialize_accumulator_fn: MaterializeAccumulatorFn,
+ ) -> Result<Option<RecordBatch>> {
+ let output_schema = Arc::clone(&self.output_schema);
+ let batch_size = self.batch_size;
+
+ let mut output =
+ match std::mem::replace(&mut self.state, AggregateHashTableState::Done) {
+ AggregateHashTableState::Outputting(mut state) => {
+ if state.group_values.is_empty() {
+ return Ok(None);
+ }
+
+ // Accumulator output consumes internal state. Materialize all
+ // groups once, then slice the materialized batch on later polls.
+ let emit_to = EmitTo::All;
+ let timer = self.group_by_metrics.emitting_time.timer();
+ let mut columns = state.group_values.emit(emit_to)?;
+ for acc in state.accumulators.iter_mut() {
+ columns.extend(materialize_accumulator_fn(acc, emit_to)?);
+ }
+ drop(timer);
+
+ let batch = RecordBatch::try_new(output_schema, columns)?;
+ debug_assert!(batch.num_rows() > 0);
+ MaterializedAggregateOutput::new(batch)
+ }
+ AggregateHashTableState::OutputtingMaterialized(output) => output,
+ AggregateHashTableState::Done => return Ok(None),
+ AggregateHashTableState::Building(_) => {
+ return internal_err!(
+ "next_output_batch must be called in the outputting state"
+ );
+ }
+ };
+
+ let batch = output.next_batch(batch_size);
+ if output.is_exhausted() {
+ self.state = AggregateHashTableState::Done;
+ } else {
+ self.state = AggregateHashTableState::OutputtingMaterialized(output);
+ }
+ Ok(batch)
+ }
+
+ pub(in crate::aggregates) fn memory_size(&self) -> usize {
+ match &self.state {
+ AggregateHashTableState::Building(state)
+ | AggregateHashTableState::Outputting(state) => {
+ let acc = state
+ .accumulators
+ .iter()
+ .map(|acc| acc.accumulator.size())
+ .sum::<usize>();
+
+ acc + state.group_values.size()
+ + state.batch_group_indices.allocated_size()
+ }
+ AggregateHashTableState::OutputtingMaterialized(output) => {
+ output.memory_size()
+ }
+ AggregateHashTableState::Done => 0,
+ }
+ }
+
+ pub(in crate::aggregates) fn group_by_metrics(&self) -> &GroupByMetrics {
+ &self.group_by_metrics
+ }
+
+ /// Returns the number of distinct groups accumulated so far.
+ pub(in crate::aggregates) fn building_group_count(&self) -> usize {
+ self.state.building().group_values.len()
+ }
+
+ /// Takes every intermediate aggregate state and resets the table so it can
+ /// continue accumulating raw input.
+ ///
+ /// Unlike normal single aggregation output, this materializes intermediate
+ /// states rather than final values. The states can therefore be merged after
+ /// spilling without finalizing the same group more than once.
+ pub(in crate::aggregates) fn take_state_batch(
+ &mut self,
+ ) -> Result<Option<RecordBatch>> {
+ let state_schema = Arc::clone(&self.state_schema);
+ let state = self.state.building_mut();
+ if state.group_values.is_empty() {
+ return Ok(None);
+ }
+
+ let mut output = state.group_values.emit(EmitTo::All)?;
+ for acc in &mut state.accumulators {
+ output.extend(acc.state(EmitTo::All)?);
+ }
+
+ let batch = RecordBatch::try_new(state_schema, output)?;
+ debug_assert!(batch.num_rows() > 0);
+
+ // `emit(EmitTo::All)` resets accumulator state. Explicitly shrink the
+ // key/index buffers too so the memory reservation can be released
+ // before the batch is sorted for spilling.
+ state.group_values.clear_shrink(0);
+ state.batch_group_indices.clear();
+ state.batch_group_indices.shrink_to_fit();
+
+ Ok(Some(batch))
+ }
+
+ pub(in crate::aggregates) fn is_building(&self) -> bool {
+ matches!(self.state, AggregateHashTableState::Building(_))
+ }
+
+ pub(in crate::aggregates) fn is_done(&self) -> bool {
+ matches!(self.state, AggregateHashTableState::Done)
+ }
+
+ pub(super) fn start_outputting(&mut self) {
+ let AggregateHashTableState::Building(mut state) =
+ std::mem::replace(&mut self.state, AggregateHashTableState::Done)
+ else {
+ unreachable!("hash aggregate table is not building")
+ };
+
+ state.batch_group_indices = Vec::new();
+ self.state = AggregateHashTableState::Outputting(state);
+ }
+}
+
+/// State and argument information for a single Aggregate
+///
+/// For example, for `SELECT COUNT(x), SUM(y WHERE z > 10) ...` there would be two
+/// `HashAggregateAccumulator`, one each for `COUNT(x)` and `SUM(y WHERE z > 10)`
+pub(super) struct HashAggregateAccumulator {
+ /// Aggregate expression used to create a fresh accumulator for related
+ /// hash tables, such as the partial-skip table.
+ aggregate_expr: Arc<AggregateFunctionExpr>,
+
+ /// Arguments to pass to this accumulator.
+ ///
+ /// Example: `CORR(x, y)` stores two expressions here, while `SUM(x)` stores one.
+ arguments: Vec<Arc<dyn PhysicalExpr>>,
+
+ /// Optional `FILTER` expression for this accumulator.
+ ///
+ /// Example: `SUM(x) FILTER (WHERE x > 10)` stores the `x > 10` predicate.
+ filter: Option<Arc<dyn PhysicalExpr>>,
+
+ /// Accumulator state for all groups for one aggregate expression.
+ accumulator: Box<dyn GroupsAccumulator>,
+}
+
+pub(super) type AggregateAccumulator = HashAggregateAccumulator;
+
+/// Function used by [`AggregateHashTable::aggregate_batch_inner`] to update one
+/// accumulator with one evaluated input batch.
+///
+/// Arguments:
+/// * accumulator to update.
+/// * accumulator's evaluated arguments and optional filter.
+/// * one group index per input row, mapping each row to its interned group.
+/// * total number of groups currently interned in that buffer, including newly
+/// interned groups.
+pub(super) type AggregateBatchFn = fn(
+ &mut AggregateAccumulator,
+ &EvaluatedAccumulatorArgs,
+ &[usize],
+ usize,
+) -> Result<()>;
+
+/// Function used by [`AggregateHashTable::next_output_batch_inner`] to
+/// materialize one accumulator's output columns.
+///
+/// Arguments:
+/// * accumulator to materialize.
+/// * group range to emit from the accumulator.
+pub(super) type MaterializeAccumulatorFn =
+ fn(&mut AggregateAccumulator, EmitTo) -> Result<Vec<ArrayRef>>;
+
+/// Evaluated aggregate arguments and filter for one input batch.
+///
+/// For example, `AVG(x + 1) FILTER (WHERE x > 0)` evaluates both `x + 1`
+/// and `x > 0`.
+///
+/// These arrays can be passed directly to [`GroupsAccumulator`].
+pub(super) struct EvaluatedAccumulatorArgs {
+ /// Evaluated argument arrays. Some aggregate functions take multiple arguments.
+ pub(super) arguments: Vec<ArrayRef>,
+ /// Evaluated filter array, `Some` if the aggregate has a `FILTER` expression.
+ pub(super) filter: Option<ArrayRef>,
+}
+
+/// Evaluated all group by keys and accumulator args.
+///
+/// e.g., `select k+1, sum(v*v) from t group by (k+1)`, this function evaluates
+/// `k+1`, `v*v`
+pub(super) struct EvaluatedAggregateBatch {
+ /// One entry per grouping set; each entry contains all evaluated group key
+ /// arrays for the current input batch.
+ pub(super) grouping_set_args: Vec<Vec<ArrayRef>>,
+
+ /// Evaluated arguments and filters, one entry per aggregate expression.
+ pub(super) accumulator_args: Vec<EvaluatedAccumulatorArgs>,
+}
+
+/// Buffer for the aggregate hash table's group keys and accumulator states.
+///
+/// It accumulates input during aggregation and emits final results during the
+/// outputting stage.
+///
+/// [`GroupValues`] stores the physical group-key layout, while
+/// [`GroupsAccumulator`] stores per-group aggregate state.
+pub(super) struct AggregateHashTableBuffer {
+ /// GROUP BY expressions evaluated for each input batch.
+ pub(super) group_by: Arc<PhysicalGroupBy>,
+
+ /// Interned group keys. Accumulator state is stored separately by group index.
+ pub(super) group_values: Box<dyn GroupValues>,
+
+ /// Group index for each row in the current input batch.
+ ///
+ /// Each value indexes into `group_values`, and the same index is used by every
+ /// accumulator to update that group's aggregate state.
+ pub(super) batch_group_indices: Vec<usize>,
+
+ /// One item per aggregate expression.
+ ///
+ /// Example: `COUNT(x), SUM(y)` creates two items. Each item owns the input
+ /// expressions, optional filter, and accumulator state for all groups.
+ pub(super) accumulators: Vec<HashAggregateAccumulator>,
+}
+
+pub(super) enum AggregateHashTableState {
+ /// Accumulating input rows into group keys and aggregate state.
+ Building(AggregateHashTableBuffer),
+ /// Emitting results directly from group keys and aggregate state.
+ Outputting(AggregateHashTableBuffer),
+ /// Materialize all the output results, and then incrementally output in the `OutputtingMaterialized` state.
+ ///
+ /// Note this is a temporary solution until the `GroupValues` issue is solved:
+ /// Issue: <https://github.com/apache/datafusion/issues/23178>
+ OutputtingMaterialized(MaterializedAggregateOutput),
+ Done,
+}
+
+/// Fully evaluated aggregate output and the next row offset to emit.
+///
+/// Final aggregate evaluation consumes accumulator state, and partial terminal
+/// output should not repeatedly renumber group values with `EmitTo::First`.
+/// Materialize once and then slice to honor `batch_size` across output polls.
+pub(super) struct MaterializedAggregateOutput {
+ batch: RecordBatch,
+ offset: usize,
+}
+
+impl MaterializedAggregateOutput {
+ pub(super) fn new(batch: RecordBatch) -> Self {
+ Self { batch, offset: 0 }
+ }
+
+ pub(super) fn next_batch(&mut self, batch_size: usize) -> Option<RecordBatch> {
+ debug_assert!(batch_size > 0);
+ if self.is_exhausted() {
+ return None;
+ }
+
+ let length = batch_size.min(self.batch.num_rows() - self.offset);
+ let batch = self.batch.slice(self.offset, length);
+ self.offset += length;
+ Some(batch)
+ }
+
+ pub(super) fn is_exhausted(&self) -> bool {
+ self.offset >= self.batch.num_rows()
+ }
+
+ pub(super) fn memory_size(&self) -> usize {
+ self.batch.get_array_memory_size()
+ }
+}
+
+impl HashAggregateAccumulator {
+ pub(super) fn new(
+ aggregate_expr: Arc<AggregateFunctionExpr>,
+ arguments: Vec<Arc<dyn PhysicalExpr>>,
+ filter: Option<Arc<dyn PhysicalExpr>>,
+ accumulator: Box<dyn GroupsAccumulator>,
+ ) -> Self {
+ Self {
+ aggregate_expr,
+ arguments,
+ filter,
+ accumulator,
+ }
+ }
+
+ /// Construct a new accumulator with the same definition, but with empty internal
+ /// state buffers (empty [`GroupsAccumulator`]).
+ pub(super) fn empty_like(&self) -> Result<Self> {
+ let accumulator = create_group_accumulator(&self.aggregate_expr)?;
+ Ok(Self::new(
+ Arc::clone(&self.aggregate_expr),
+ self.arguments.clone(),
+ self.filter.clone(),
+ accumulator,
+ ))
+ }
+
+ /// Evaluate aggregate arguments and filter for one input batch.
+ ///
+ /// For example, `AVG(x + 1) FILTER (WHERE x > 0)` evaluates both `x + 1`
+ /// and `x > 0`.
+ ///
+ /// These arrays can be passed directly to [`GroupsAccumulator`] next.
+ pub(super) fn evaluate_acc_args(
+ &self,
+ batch: &RecordBatch,
+ ) -> Result<EvaluatedAccumulatorArgs> {
+ let arguments = self
+ .arguments
+ .iter()
+ .map(|expr| {
+ expr.evaluate(batch)
+ .and_then(|value| value.into_array(batch.num_rows()))
+ })
+ .collect::<Result<_>>()?;
+
+ let filter = self
+ .filter
+ .as_ref()
+ .map(|filter| {
+ filter
+ .evaluate(batch)
+ .and_then(|value| value.into_array(batch.num_rows()))
+ })
+ .transpose()?;
+
+ Ok(EvaluatedAccumulatorArgs { arguments, filter })
+ }
+
+ pub(super) fn size(&self) -> usize {
+ self.accumulator.size()
+ }
+
+ pub(super) fn update_batch(
+ &mut self,
+ values: &EvaluatedAccumulatorArgs,
+ group_indices: &[usize],
+ total_num_groups: usize,
+ ) -> Result<()> {
+ let filter = values.filter.as_ref().map(|filter| filter.as_boolean());
+ self.accumulator.update_batch(
+ &values.arguments,
+ group_indices,
+ filter,
+ total_num_groups,
+ )
+ }
+
+ pub(super) fn merge_batch(
+ &mut self,
+ values: &EvaluatedAccumulatorArgs,
+ group_indices: &[usize],
+ total_num_groups: usize,
+ ) -> Result<()> {
+ debug_assert!(values.filter.is_none());
+ self.accumulator
+ .merge_batch(&values.arguments, group_indices, total_num_groups)
+ }
+
+ /// Evaluating final aggregate results according to `EmitTo`, and reset inner
+ /// states. (e.g. after `evaluate(EmitTo::All)`, it returns all accumulated groups
+ /// , and clear the inner buffers)
+ pub(super) fn evaluate(&mut self, emit_to: EmitTo) -> Result<ArrayRef> {
+ self.accumulator.evaluate(emit_to)
+ }
+
+ pub(super) fn evaluate_to_columns(
+ &mut self,
+ emit_to: EmitTo,
+ ) -> Result<Vec<ArrayRef>> {
+ Ok(vec![self.evaluate(emit_to)?])
+ }
+
+ /// Evaluating partial aggregate results according to `EmitTo`, and reset inner
+ /// states. (e.g. after `state(EmitTo::All)`, it returns all accumulated groups
+ /// , and clear the inner buffers)
+ pub(super) fn state(&mut self, emit_to: EmitTo) -> Result<Vec<ArrayRef>> {
+ self.accumulator.state(emit_to)
+ }
+
+ pub(super) fn convert_to_state(
+ &mut self,
+ values: &EvaluatedAccumulatorArgs,
+ ) -> Result<Vec<ArrayRef>> {
+ let opt_filter = values.filter.as_ref().map(|filter| filter.as_boolean());
+ self.accumulator
+ .convert_to_state(&values.arguments, opt_filter)
+ }
+
+ pub(super) fn null_arguments(
+ &self,
+ input_schema: &SchemaRef,
+ ) -> Result<Vec<ArrayRef>> {
+ self.arguments
+ .iter()
+ .map(|expr| {
+ let data_type = expr.data_type(input_schema)?;
+ Ok(new_null_array(&data_type, 1))
+ })
+ .collect()
+ }
+}
+
+impl AggregateHashTableState {
+ pub(super) fn building(&self) -> &AggregateHashTableBuffer {
+ let Self::Building(state) = self else {
+ unreachable!("hash aggregate table is not building")
+ };
+ state
+ }
+
+ pub(super) fn building_mut(&mut self) -> &mut AggregateHashTableBuffer {
+ let Self::Building(state) = self else {
+ unreachable!("hash aggregate table is not building")
+ };
+ state
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use std::sync::Arc;
+
+ use arrow::array::{Array, Int32Array};
+ use arrow::datatypes::{DataType, Field, Schema};
+
+ use super::*;
+
+ #[test]
+ fn materialized_aggregate_output_slices_batches_until_exhausted() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![Field::new(
+ "group_col",
+ DataType::Int32,
+ false,
+ )]));
+ let batch = RecordBatch::try_new(
+ schema,
+ vec![Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5]))],
+ )?;
+ let mut output = MaterializedAggregateOutput::new(batch);
+
+ assert_eq!(int32_values(&output.next_batch(2).unwrap(), 0), vec![1, 2]);
+ assert_eq!(int32_values(&output.next_batch(2).unwrap(), 0), vec![3, 4]);
+ assert_eq!(int32_values(&output.next_batch(2).unwrap(), 0), vec![5]);
+ assert!(output.next_batch(2).is_none());
+ assert!(output.is_exhausted());
+
+ Ok(())
+ }
+
+ fn int32_values(batch: &RecordBatch, column: usize) -> Vec<i32> {
+ let array = batch
+ .column(column)
+ .as_any()
+ .downcast_ref::<Int32Array>()
+ .unwrap();
+ (0..array.len()).map(|idx| array.value(idx)).collect()
+ }
+}
diff --git a/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/common_ordered.rs b/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/common_ordered.rs
new file mode 100644
index 0000000000..2293e7b1b8
--- /dev/null
+++ b/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/common_ordered.rs
@@ -0,0 +1,410 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Common utilities for aggregate tables used in aggregations that inputs are ordered
+//! by the groups.
+
+use std::marker::PhantomData;
+use std::sync::Arc;
+
+use arrow::datatypes::SchemaRef;
+use arrow::record_batch::RecordBatch;
+use datafusion_common::Result;
+use datafusion_common::assert_or_internal_err;
+use datafusion_execution::memory_pool::proxy::VecAllocExt;
+use datafusion_expr::EmitTo;
+
+use crate::InputOrderMode;
+use crate::PhysicalExpr;
+use crate::aggregates::group_values::{GroupByMetrics, GroupValues, new_group_values};
+use crate::aggregates::grouped_hash_stream::create_group_accumulator;
+use crate::aggregates::order::GroupOrdering;
+use crate::aggregates::{
+ AggregateExec, AggregateMode, PhysicalGroupBy, aggregate_expressions,
+ evaluate_group_by,
+};
+
+use super::common::{AggregateAccumulator, EvaluatedAggregateBatch};
+
+/// Aggregate table shared by the ordered partial and final paths.
+///
+/// # Ordering optimization
+///
+/// The table consumes input batches while `GroupOrdering` tracks which groups
+/// are proven complete. Completed groups can be emitted before the input stream
+/// ends, which keeps memory bounded by the active ordered key range.
+///
+/// # Partial and final variant difference
+///
+/// The partial and final aggregate tables implement the two stages of grouped
+/// aggregation. See
+/// [`OrderedPartialAggregateStream`](crate::aggregates::ordered_partial_stream::OrderedPartialAggregateStream)
+/// for the high-level plan shape.
+///
+/// Example: `AVG(v) FILTER (WHERE v>0) GROUP BY k`
+///
+/// Partial table ([`AggregateMode::Partial`], with optional filter from query):
+/// - Input rows: `k, v`
+/// - Table stores: `k, sum(v), count(v)`
+/// - Output schema: `k, sum(v), count(v)`
+///
+/// Final table ([`AggregateMode::Final`], no filters):
+/// - Input rows: `k, sum(v), count(v)`
+/// - Table stores: `k, sum(v), count(v)`
+/// - Output schema: `k, avg(v)`
+///
+/// # Marker Type
+///
+/// `OrderedAggrMode` selects the aggregate semantics. For example,
+/// `OrderedAggregateTable::<PartialMarker>::new(...)` consumes raw rows
+/// and emits partial states, while
+/// `OrderedAggregateTable::<FinalMarker>::new_with_input_order(...)`
+/// consumes partial states and emits final values.
+///
+/// Shared methods live on `impl<T>`; partial/final behavior lives on
+/// marker-specific impls.
+pub(in crate::aggregates) struct OrderedAggregateTable<OrderedAggrMode> {
+ /// Output schema: group columns followed by aggregate state or final values.
+ pub(super) output_schema: SchemaRef,
+
+ /// Intermediate-state schema used when memory pressure requires the table
+ /// to pass through or spill its current state.
+ pub(super) state_schema: SchemaRef,
+
+ /// Maximum rows per emitted output batch, from config `batch_size`.
+ pub(super) batch_size: usize,
+
+ /// Grouping and accumulator-specific timing metrics.
+ pub(super) group_by_metrics: GroupByMetrics,
+
+ /// Group keys, ordering state, and accumulator states.
+ pub(super) buffer: OrderedAggregateTableBuffer,
+
+ _mode: PhantomData<OrderedAggrMode>,
+}
+
+/// Buffer for the ordered aggregate table's group keys and accumulator states.
+///
+/// It accumulates input during aggregation and emits output rows as soon as the
+/// input ordering proves those groups are complete.
+///
+/// [`GroupOrdering`] tracks when and how to do early emit.
+/// [`GroupValues`] stores the physical group-key layout, while
+/// [`datafusion_expr::GroupsAccumulator`] stores per-group aggregate state.
+pub(super) struct OrderedAggregateTableBuffer {
+ /// GROUP BY expressions evaluated against input batches.
+ pub(super) group_by: Arc<PhysicalGroupBy>,
+
+ /// Tracks how far ordered input allows this table to drain safely.
+ pub(super) group_ordering: GroupOrdering,
+
+ /// Interned group keys, in the same group-id order used by accumulators.
+ pub(super) group_values: Box<dyn GroupValues>,
+
+ /// Scratch group id vector for the current input batch.
+ pub(super) group_indices: Vec<usize>,
+
+ /// One item per aggregate expression.
+ ///
+ /// Example: `COUNT(x), SUM(y)` creates two items. Each item owns the input
+ /// expressions, optional filter, and accumulator state for all groups.
+ pub(super) accumulators: Vec<AggregateAccumulator>,
+}
+
+/// Methods shared by all aggregate modes
+impl<AggrMode> OrderedAggregateTable<AggrMode> {
+ #[expect(
+ clippy::too_many_arguments,
+ reason = "keeps ordered partial and final table construction explicit"
+ )]
+ pub(super) fn new_for_mode(
+ agg: &AggregateExec,
+ input_schema: &SchemaRef,
+ output_schema: SchemaRef,
+ state_schema: SchemaRef,
+ batch_size: usize,
+ input_order_mode: &InputOrderMode,
+ aggregate_mode: &AggregateMode,
+ filters: Vec<Option<Arc<dyn PhysicalExpr>>>,
+ group_by_metrics: GroupByMetrics,
+ ) -> Result<Self> {
+ assert_or_internal_err!(
+ batch_size > 0,
+ "OrderedAggregateTable requires config batch_size >= 1"
+ );
+
+ let group_ordering = GroupOrdering::try_new(input_order_mode)?;
+ let group_schema = agg.group_by.group_schema(input_schema)?;
+ let group_values = new_group_values(group_schema, &group_ordering)?;
+ let aggregate_arguments = aggregate_expressions(
+ &agg.aggr_expr,
+ aggregate_mode,
+ agg.group_by.num_group_exprs(),
+ )?;
+ let accumulators = agg
+ .aggr_expr
+ .iter()
+ .zip(aggregate_arguments)
+ .zip(filters)
+ .map(|((agg_expr, arguments), filter)| {
+ let accumulator = create_group_accumulator(agg_expr)?;
+ Ok(AggregateAccumulator::new(
+ Arc::clone(agg_expr),
+ arguments,
+ filter,
+ accumulator,
+ ))
+ })
+ .collect::<Result<_>>()?;
+
+ Ok(Self {
+ output_schema,
+ state_schema,
+ batch_size,
+ group_by_metrics,
+ buffer: OrderedAggregateTableBuffer {
+ group_by: Arc::clone(&agg.group_by),
+ group_ordering,
+ group_values,
+ group_indices: vec![],
+ accumulators,
+ },
+ _mode: PhantomData,
+ })
+ }
+
+ /// Evaluates all group by keys and accumulator args.
+ ///
+ /// e.g., `select k+1, sum(v*v) from t group by (k+1)`, this function
+ /// evaluates `k+1`, `v*v`.
+ pub(super) fn evaluate_batch(
+ &self,
+ batch: &RecordBatch,
+ ) -> Result<EvaluatedAggregateBatch> {
+ let timer = self.group_by_metrics.time_calculating_group_ids.timer();
+ let grouping_set_args = evaluate_group_by(&self.buffer.group_by, batch)?;
+ drop(timer);
+
+ let timer = self.group_by_metrics.aggregate_arguments_time.timer();
+ let accumulator_args = self
+ .buffer
+ .accumulators
+ .iter()
+ .map(|acc| acc.evaluate_acc_args(batch))
+ .collect::<Result<Vec<_>>>()?;
+ drop(timer);
+
+ Ok(EvaluatedAggregateBatch {
+ grouping_set_args,
+ accumulator_args,
+ })
+ }
+
+ /// Called after the input stream is exhausted and the last batch has been
+ /// aggregated.
+ ///
+ /// Updates the internal `GroupOrdering` so it can continue emitting until
+ /// the buffer is empty.
+ pub(in crate::aggregates) fn input_done(&mut self) {
+ self.buffer.group_ordering.input_done();
+ }
+
+ /// Returns the ordering state used to decide how memory pressure is handled.
+ pub(in crate::aggregates) fn group_ordering(&self) -> &GroupOrdering {
+ &self.buffer.group_ordering
+ }
+
+ /// Number of groups currently buffered.
+ pub(in crate::aggregates) fn num_groups(&self) -> usize {
+ self.buffer.group_values.len()
+ }
+
+ /// Check if there is zero groups accumulated so far.
+ pub(in crate::aggregates) fn is_empty(&self) -> bool {
+ self.num_groups() == 0
+ }
+
+ /// All internal buffer's memory size.
+ pub(in crate::aggregates) fn memory_size(&self) -> usize {
+ self.buffer
+ .accumulators
+ .iter()
+ .map(|acc| acc.size())
+ .sum::<usize>()
+ + self.buffer.group_values.size()
+ + self.buffer.group_ordering.size()
+ + self.buffer.group_indices.allocated_size()
+ }
+
+ pub(in crate::aggregates) fn group_by_metrics(&self) -> GroupByMetrics {
+ self.group_by_metrics.clone()
+ }
+
+ /// Takes every intermediate aggregate state and resets the table so it can
+ /// continue with a new ordered input segment.
+ ///
+ /// Unlike normal ordered emission, this operation is allowed to take the
+ /// active (incomplete) groups. Partial aggregation can pass those states to
+ /// its final stage, while final aggregation sorts and spills them before
+ /// replay.
+ pub(in crate::aggregates) fn take_state_batch(
+ &mut self,
+ ) -> Result<Option<RecordBatch>> {
+ if self.buffer.group_values.is_empty() {
+ return Ok(None);
+ }
+
+ let mut output = self.buffer.group_values.emit(EmitTo::All)?;
+ for acc in &mut self.buffer.accumulators {
+ output.extend(acc.state(EmitTo::All)?);
+ }
+
+ let batch = RecordBatch::try_new(Arc::clone(&self.state_schema), output)?;
+ debug_assert!(batch.num_rows() > 0);
+
+ // `emit(EmitTo::All)` resets accumulator state. Explicitly shrink the
+ // key/index buffers too so the memory reservation can be released
+ // before the batch is passed downstream or sorted for spilling.
+ self.buffer.group_values.clear_shrink(0);
+ self.buffer.group_indices.clear();
+ self.buffer.group_indices.shrink_to_fit();
+ self.buffer.group_ordering.reset();
+
+ Ok(Some(batch))
+ }
+
+ /// Returns the [`EmitTo`], clamped to the specified batch size
+ ///
+ /// Returns `(emit_to, should_remove_groups)`, where `emit_to` is the number
+ /// of groups to emit from `GroupValues` / accumulators, and
+ /// `should_remove_groups` indicates whether `GroupOrdering` must also shift
+ /// its tracked indexes.
+ pub(super) fn clamp_emit_to(
+ &self,
+ group_count: usize,
+ emit_to: EmitTo,
+ ) -> (EmitTo, bool) {
+ match emit_to {
+ EmitTo::First(n) => (EmitTo::First(n.min(self.batch_size)), true),
+ EmitTo::All if group_count <= self.batch_size => (EmitTo::All, false),
+ EmitTo::All => (EmitTo::First(self.batch_size), false),
+ }
+ }
+ /// Aggregates one evaluated input batch.
+ ///
+ /// This common utility is used by ordered partial and ordered final aggregation.
+ ///
+ /// # Argument: `is_final`
+ ///
+ /// - `true`: merge partial aggregate states for final aggregation.
+ /// - `false`: update aggregate states from raw input for partial aggregation.
+ pub(super) fn aggregate_evaluated_batch(
+ &mut self,
+ evaluated_batch: &EvaluatedAggregateBatch,
+ is_final: bool,
+ ) -> Result<()> {
+ for group_values in &evaluated_batch.grouping_set_args {
+ let starting_num_groups = self.buffer.group_values.len();
+ self.buffer
+ .group_values
+ .intern(group_values, &mut self.buffer.group_indices)?;
+ let total_num_groups = self.buffer.group_values.len();
+ if total_num_groups > starting_num_groups {
+ self.buffer.group_ordering.new_groups(
+ group_values,
+ &self.buffer.group_indices,
+ total_num_groups,
+ )?;
+ }
+
+ let timer = self.group_by_metrics.aggregation_time.timer();
+ for (acc, values) in self
+ .buffer
+ .accumulators
+ .iter_mut()
+ .zip(evaluated_batch.accumulator_args.iter())
+ {
+ if is_final {
+ acc.merge_batch(
+ values,
+ &self.buffer.group_indices,
+ total_num_groups,
+ )?;
+ } else {
+ acc.update_batch(
+ values,
+ &self.buffer.group_indices,
+ total_num_groups,
+ )?;
+ }
+ }
+ drop(timer);
+ }
+
+ Ok(())
+ }
+
+ /// Emits groups allowed by `GroupOrdering`, leaving only the current
+ /// unfinished ordered-key range buffered.
+ ///
+ /// This common utility is used by ordered partial and ordered final aggregation.
+ ///
+ /// # Argument: `is_final`
+ ///
+ /// - `true`: output final aggregate values.
+ /// - `false`: output partial accumulator states.
+ pub(super) fn next_output_batch_for_mode(
+ &mut self,
+ is_final: bool,
+ ) -> Result<Option<RecordBatch>> {
+ if self.buffer.group_values.is_empty() {
+ return Ok(None);
+ }
+
+ let Some(emit_to) = self.buffer.group_ordering.emit_to() else {
+ return Ok(None);
+ };
+ let (emit_to, should_remove_groups) =
+ self.clamp_emit_to(self.buffer.group_values.len(), emit_to);
+
+ let timer = self.group_by_metrics.emitting_time.timer();
+ let mut output = self.buffer.group_values.emit(emit_to)?;
+ if should_remove_groups {
+ match emit_to {
+ EmitTo::First(n) => self.buffer.group_ordering.remove_groups(n),
+ // `EmitTo::All` is only used after `input_done`, when all
+ // buffered groups are known complete and the ordering state is
+ // no longer needed.
+ EmitTo::All => {}
+ }
+ }
+
+ for acc in &mut self.buffer.accumulators {
+ if is_final {
+ output.push(acc.evaluate(emit_to)?);
+ } else {
+ output.extend(acc.state(emit_to)?);
+ }
+ }
+ drop(timer);
+
+ let batch = RecordBatch::try_new(Arc::clone(&self.output_schema), output)?;
+ debug_assert!(batch.num_rows() > 0);
+
+ Ok(Some(batch))
+ }
+}
diff --git a/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/final_table.rs b/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/final_table.rs
new file mode 100644
index 0000000000..b80e15d7f8
--- /dev/null
+++ b/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/final_table.rs
@@ -0,0 +1,77 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use arrow::datatypes::SchemaRef;
+use arrow::record_batch::RecordBatch;
+use datafusion_common::Result;
+
+use crate::aggregates::AggregateExec;
+
+use super::common::{AggregateHashTable, FinalMarker, HashAggregateAccumulator};
+
+/// Implementation specific to final aggregation, where the table stores partial
+/// aggregate states and the input rows are also partial states.
+///
+/// Example: `AVG(x) GROUP BY k`
+///
+/// - Aggregate table stores: `k, sum(x), count(x)`
+/// - Input rows: `k, sum(x), count(x)`
+impl AggregateHashTable<FinalMarker> {
+ pub(in crate::aggregates) fn new(
+ agg: &AggregateExec,
+ partition: usize,
+ output_schema: SchemaRef,
+ batch_size: usize,
+ ) -> Result<Self> {
+ Self::new_with_filters(
+ agg,
+ partition,
+ output_schema,
+ Arc::clone(&agg.input().schema()),
+ batch_size,
+ vec![None; agg.aggr_expr.len()],
+ )
+ }
+
+ /// Emits the next batch of aggregated group keys and final aggregate values.
+ ///
+ /// The output batch size is determined by `self.batch_size`.
+ ///
+ /// Returns `Some(batch)` for each emitted batch, `None` when output is
+ /// exhausted, and an internal error if polled in the `Building` state.
+ pub(in crate::aggregates) fn next_output_batch(
+ &mut self,
+ ) -> Result<Option<RecordBatch>> {
+ self.next_output_batch_inner(HashAggregateAccumulator::evaluate_to_columns)
+ }
+
+ /// Final aggregation consumes partial aggregate states and merges them into
+ /// the table's partial-state accumulators.
+ pub(in crate::aggregates) fn aggregate_batch(
+ &mut self,
+ batch: &RecordBatch,
+ ) -> Result<()> {
+ self.aggregate_batch_inner(batch, HashAggregateAccumulator::merge_batch)
+ }
+
+ pub(in crate::aggregates) fn start_output(&mut self) -> Result<()> {
+ self.start_outputting();
+ Ok(())
+ }
+}
diff --git a/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/mod.rs b/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/mod.rs
new file mode 100644
index 0000000000..2c7ec01654
--- /dev/null
+++ b/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/mod.rs
@@ -0,0 +1,31 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+mod common;
+mod common_ordered;
+mod final_table;
+mod ordered_final_table;
+mod ordered_partial_table;
+mod partial_reduce_table;
+mod partial_table;
+mod single_table;
+
+pub(super) use common::{
+ AggregateHashTable, FinalMarker, PartialMarker, PartialReduceMarker,
+ PartialSkipMarker, SingleMarker,
+};
+pub(super) use common_ordered::OrderedAggregateTable;
diff --git a/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/ordered_final_table.rs b/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/ordered_final_table.rs
new file mode 100644
index 0000000000..fd064ebffe
--- /dev/null
+++ b/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/ordered_final_table.rs
@@ -0,0 +1,85 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Aggregate table for final aggregation when partial-state input is ordered.
+//!
+//! See comments in [`super::ordered_partial_table`] for details.
+
+use std::sync::Arc;
+
+use arrow::datatypes::SchemaRef;
+use arrow::record_batch::RecordBatch;
+use datafusion_common::Result;
+
+use crate::InputOrderMode;
+use crate::aggregates::aggregate_hash_table::FinalMarker;
+use crate::aggregates::group_values::GroupByMetrics;
+use crate::aggregates::{AggregateExec, AggregateMode};
+
+use super::common_ordered::OrderedAggregateTable;
+
+/// Implementation specific to final aggregation, where the table stores partial
+/// aggregate states and the input rows are also partial states.
+///
+/// Example: `AVG(x) GROUP BY k`
+///
+/// - Aggregate table stores: `k, sum(x), count(x)`
+/// - Input rows: `k, sum(x), count(x)`
+///
+/// See comments at [`OrderedAggregateTable`] for details.
+impl OrderedAggregateTable<FinalMarker> {
+ pub(in crate::aggregates) fn new_with_input_order(
+ agg: &AggregateExec,
+ input_schema: &SchemaRef,
+ output_schema: SchemaRef,
+ batch_size: usize,
+ input_order_mode: &InputOrderMode,
+ group_by_metrics: GroupByMetrics,
+ ) -> Result<Self> {
+ Self::new_for_mode(
+ agg,
+ input_schema,
+ output_schema,
+ Arc::clone(input_schema),
+ batch_size,
+ input_order_mode,
+ &AggregateMode::Final,
+ vec![None; agg.aggr_expr.len()],
+ group_by_metrics,
+ )
+ }
+
+ /// Merges one partial-state input batch and updates ordering information for
+ /// any newly observed groups.
+ pub(in crate::aggregates) fn aggregate_batch(
+ &mut self,
+ batch: &RecordBatch,
+ ) -> Result<()> {
+ let evaluated_batch = self.evaluate_batch(batch)?;
+ // `PhysicalGroupBy::as_final()` removes grouping sets while planning
+ // final aggregation, so final ordered aggregation sees one grouping.
+ debug_assert_eq!(evaluated_batch.grouping_set_args.len(), 1);
+ self.aggregate_evaluated_batch(&evaluated_batch, true)
+ }
+
+ /// See comments in `ordered_partial_stream::next_output_batch`
+ pub(in crate::aggregates) fn next_output_batch(
+ &mut self,
+ ) -> Result<Option<RecordBatch>> {
+ self.next_output_batch_for_mode(true)
+ }
+}
diff --git a/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/ordered_partial_table.rs b/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/ordered_partial_table.rs
new file mode 100644
index 0000000000..a04e4dda8f
--- /dev/null
+++ b/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/ordered_partial_table.rs
@@ -0,0 +1,106 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Aggregate table for partial aggregation when input is ordered by group keys.
+//!
+//! See the [`super::common_ordered`] comments for the high-level ideas.
+//!
+//! This operator handles input that is ordered by group keys:
+//! - Fully ordered: `GROUP BY a, b`, input is `ORDER BY a, b`
+//! - Partially ordered: `GROUP BY a, b`, input is `ORDER BY a`
+//!
+//! When a group key combination is exhausted, this table eagerly flushes the
+//! completed groups to improve memory efficiency.
+//!
+//! The implementation is separated from other aggregate tables because this
+//! execution path is likely to be optimized further in the future.
+
+use std::sync::Arc;
+
+use arrow::datatypes::SchemaRef;
+use arrow::record_batch::RecordBatch;
+use datafusion_common::Result;
+
+use crate::aggregates::{
+ AggregateExec, AggregateMode, aggregate_hash_table::PartialMarker,
+ group_values::GroupByMetrics,
+};
+
+use super::common_ordered::OrderedAggregateTable;
+
+/// Implementation specific to partial aggregation, where the table stores
+/// partial aggregate states and the input rows are raw rows.
+///
+/// Example: `AVG(x) GROUP BY k`
+///
+/// - Aggregate table stores: `k, sum(x), count(x)`
+/// - Input rows: `k, x`
+///
+/// See comments at [`OrderedAggregateTable`] for details.
+impl OrderedAggregateTable<PartialMarker> {
+ pub(in crate::aggregates) fn new(
+ agg: &AggregateExec,
+ partition: usize,
+ output_schema: SchemaRef,
+ batch_size: usize,
+ ) -> Result<Self> {
+ let input_schema = agg.input().schema();
+ let state_schema = Arc::clone(&output_schema);
+ let group_by_metrics = GroupByMetrics::new(&agg.metrics, partition);
+ Self::new_for_mode(
+ agg,
+ &input_schema,
+ output_schema,
+ state_schema,
+ batch_size,
+ &agg.input_order_mode,
+ &AggregateMode::Partial,
+ agg.filter_expr.iter().cloned().collect(),
+ group_by_metrics,
+ )
+ }
+
+ /// Aggregates one raw input batch and updates ordering information for any
+ /// newly observed groups.
+ pub(in crate::aggregates) fn aggregate_batch(
+ &mut self,
+ batch: &RecordBatch,
+ ) -> Result<()> {
+ let evaluated_batch = self.evaluate_batch(batch)?;
+ self.aggregate_evaluated_batch(&evaluated_batch, false)
+ }
+
+ /// Emits the next batch of partial state rows for groups proven complete by
+ /// the input ordering.
+ ///
+ /// For example, when the query is `GROUP BY a` and the input is ordered by
+ /// `a`, seeing a latest input row with `a = 3` means all groups with `a < 3`
+ /// are complete and safe to emit.
+ ///
+ /// Key steps:
+ /// 1. Ask `group_ordering` to decide how many groups can be emitted eagerly.
+ /// 2. Remove the emitted groups from `group_ordering`, `GroupValues`, and
+ /// all `GroupsAccumulator`s.
+ ///
+ /// This may output small batches. Avoiding tiny batches is left to future
+ /// ordered-aggregation optimizations.
+ pub(in crate::aggregates) fn next_output_batch(
+ &mut self,
+ ) -> Result<Option<RecordBatch>> {
+ self.next_output_batch_for_mode(false)
+ }
+}
diff --git a/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/partial_reduce_table.rs b/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/partial_reduce_table.rs
new file mode 100644
index 0000000000..4dfd6a74d1
--- /dev/null
+++ b/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/partial_reduce_table.rs
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use arrow::datatypes::SchemaRef;
+use arrow::record_batch::RecordBatch;
+use datafusion_common::Result;
+
+use crate::aggregates::AggregateExec;
+
+use super::common::{AggregateHashTable, HashAggregateAccumulator, PartialReduceMarker};
+
+/// Methods specific to the aggregate hash table used in the partial-reduce stage.
+impl AggregateHashTable<PartialReduceMarker> {
+ pub(in crate::aggregates) fn new(
+ agg: &AggregateExec,
+ partition: usize,
+ output_schema: SchemaRef,
+ batch_size: usize,
+ ) -> Result<Self> {
+ Self::new_with_filters(
+ agg,
+ partition,
+ Arc::clone(&output_schema),
+ output_schema,
+ batch_size,
+ vec![None; agg.aggr_expr.len()],
+ )
+ }
+
+ /// Emits the next batch of aggregated group keys and aggregate states.
+ ///
+ /// The output batch size is determined by `self.batch_size`.
+ ///
+ /// Returns `Some(batch)` for each emitted batch, `None` when output is
+ /// exhausted, and an internal error if polled in the `Building` state.
+ pub(in crate::aggregates) fn next_output_batch(
+ &mut self,
+ ) -> Result<Option<RecordBatch>> {
+ self.next_output_batch_inner(HashAggregateAccumulator::state)
+ }
+
+ /// Partial-reduce aggregation consumes partial aggregate states and merges
+ /// them into the table's partial-state accumulators.
+ pub(in crate::aggregates) fn aggregate_batch(
+ &mut self,
+ batch: &RecordBatch,
+ ) -> Result<()> {
+ self.aggregate_batch_inner(batch, HashAggregateAccumulator::merge_batch)
+ }
+
+ pub(in crate::aggregates) fn start_output(&mut self) -> Result<()> {
+ self.start_outputting();
+ Ok(())
+ }
+}
diff --git a/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/partial_table.rs b/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/partial_table.rs
new file mode 100644
index 0000000000..a64fd32536
--- /dev/null
+++ b/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/partial_table.rs
@@ -0,0 +1,219 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::collections::HashMap;
+use std::marker::PhantomData;
+use std::sync::Arc;
+
+use arrow::array::{ArrayRef, BooleanArray, new_null_array};
+use arrow::datatypes::SchemaRef;
+use arrow::record_batch::RecordBatch;
+use datafusion_common::{Result, assert_eq_or_internal_err};
+
+use crate::aggregates::group_values::new_group_values;
+use crate::aggregates::order::GroupOrdering;
+use crate::aggregates::{AggregateExec, group_id_array, max_duplicate_ordinal};
+
+use super::common::{
+ AggregateHashTable, AggregateHashTableBuffer, AggregateHashTableState,
+ EvaluatedAccumulatorArgs, HashAggregateAccumulator, PartialMarker, PartialSkipMarker,
+};
+
+/// Implementation specific to partial aggregation, where the table stores
+/// partial aggregate states and the input rows are raw rows.
+///
+/// Example: `AVG(x) GROUP BY k`
+///
+/// - Aggregate table stores: `k, sum(x), count(x)`
+/// - Input rows: `k, x`
+impl AggregateHashTable<PartialMarker> {
+ pub(in crate::aggregates) fn new(
+ agg: &AggregateExec,
+ partition: usize,
+ output_schema: SchemaRef,
+ batch_size: usize,
+ ) -> Result<Self> {
+ Self::new_with_filters(
+ agg,
+ partition,
+ Arc::clone(&output_schema),
+ output_schema,
+ batch_size,
+ agg.filter_expr.iter().cloned().collect(),
+ )
+ }
+
+ /// Emits the next batch of aggregated group keys and aggregate states.
+ ///
+ /// The output batch size is determined by `self.batch_size`.
+ ///
+ /// Returns `Some(batch)` for each emitted batch, `None` when output is
+ /// exhausted, and an internal error if polled in the `Building` state.
+ pub(in crate::aggregates) fn next_output_batch(
+ &mut self,
+ ) -> Result<Option<RecordBatch>> {
+ self.next_output_batch_inner(HashAggregateAccumulator::state)
+ }
+
+ /// In skip-partial-aggregation optimization, when a decision has been made to skip
+ /// partial stage, build a typed hash table only for aggregation state conversion
+ /// row-by-row.
+ pub(in crate::aggregates) fn partial_skip_table(
+ &self,
+ ) -> Result<AggregateHashTable<PartialSkipMarker>> {
+ let state = self.state.building();
+ let group_schema = state.group_by.group_schema(&self.input_schema)?;
+ let group_values = new_group_values(group_schema, &GroupOrdering::None)?;
+ let accumulators = state
+ .accumulators
+ .iter()
+ .map(HashAggregateAccumulator::empty_like)
+ .collect::<Result<Vec<_>>>()?;
+
+ Ok(AggregateHashTable {
+ group_by_metrics: self.group_by_metrics.clone(),
+ input_schema: Arc::clone(&self.input_schema),
+ output_schema: Arc::clone(&self.output_schema),
+ state_schema: Arc::clone(&self.state_schema),
+ batch_size: self.batch_size,
+ state: AggregateHashTableState::Building(AggregateHashTableBuffer {
+ group_by: Arc::clone(&state.group_by),
+ group_values,
+ batch_group_indices: Default::default(),
+ accumulators,
+ }),
+ _mode: PhantomData,
+ })
+ }
+
+ /// Partial aggregation consumes raw input rows and updates the table's
+ /// partial-state accumulators.
+ pub(in crate::aggregates) fn aggregate_batch(
+ &mut self,
+ batch: &RecordBatch,
+ ) -> Result<()> {
+ self.aggregate_batch_inner(batch, HashAggregateAccumulator::update_batch)
+ }
+
+ pub(in crate::aggregates) fn start_output(&mut self) -> Result<()> {
+ self.init_empty_grouping_sets()?;
+ self.start_outputting();
+ Ok(())
+ }
+
+ /// Creates the required empty grouping-set rows when the input is empty.
+ ///
+ /// For example, this query must still produce one grand-total group even if
+ /// `t` has no rows:
+ ///
+ /// ```sql
+ /// SELECT COUNT(v)
+ /// FROM t
+ /// GROUP BY GROUPING SETS (());
+ /// ```
+ ///
+ /// The synthetic row is filtered out before accumulator update so aggregates
+ /// see the same state they would see for an empty input, rather than a real
+ /// null-valued row.
+ fn init_empty_grouping_sets(&mut self) -> Result<()> {
+ let state = self.state.building_mut();
+ if !state.group_by.has_grouping_set() || !state.group_values.is_empty() {
+ return Ok(());
+ }
+
+ let max_ordinal = max_duplicate_ordinal(state.group_by.groups());
+ let mut ordinals: HashMap<&[bool], usize> = HashMap::new();
+ let group_schema = state.group_by.group_schema(&self.input_schema)?;
+ let n_expr = state.group_by.expr().len();
+ let mut any_interned = false;
+
+ for group in state.group_by.groups() {
+ let ordinal = {
+ let entry = ordinals.entry(group.as_slice()).or_insert(0);
+ let ordinal = *entry;
+ *entry += 1;
+ ordinal
+ };
+
+ if !group.iter().all(|&is_null| is_null) {
+ continue;
+ }
+
+ let mut cols: Vec<ArrayRef> = group_schema
+ .fields()
+ .iter()
+ .take(n_expr)
+ .map(|field| new_null_array(field.data_type(), 1))
+ .collect();
+ cols.push(group_id_array(group, ordinal, max_ordinal, 1)?);
+
+ state
+ .group_values
+ .intern(&cols, &mut state.batch_group_indices)?;
+ any_interned = true;
+ }
+
+ if any_interned {
+ let total_groups = state.group_values.len();
+ let false_filter = BooleanArray::from(vec![false]);
+ for acc in state.accumulators.iter_mut() {
+ let null_args = acc.null_arguments(&self.input_schema)?;
+ let values = EvaluatedAccumulatorArgs {
+ arguments: null_args,
+ filter: Some(Arc::new(false_filter.clone())),
+ };
+ acc.update_batch(&values, &[0], total_groups)?;
+ }
+ }
+
+ Ok(())
+ }
+}
+
+impl AggregateHashTable<PartialSkipMarker> {
+ pub(in crate::aggregates) fn convert_batch_to_state(
+ &mut self,
+ batch: &RecordBatch,
+ ) -> Result<RecordBatch> {
+ let evaluated_batch = self.evaluate_batch(batch)?;
+
+ assert_eq_or_internal_err!(
+ evaluated_batch.grouping_set_args.len(),
+ 1,
+ "group_values expected to have single element"
+ );
+ let mut output = evaluated_batch
+ .grouping_set_args
+ .into_iter()
+ .next()
+ .unwrap_or_default();
+
+ let state = self.state.building_mut();
+ for (acc, values) in state
+ .accumulators
+ .iter_mut()
+ .zip(evaluated_batch.accumulator_args.iter())
+ {
+ output.extend(acc.convert_to_state(values)?);
+ }
+
+ Ok(RecordBatch::try_new(
+ Arc::clone(&self.output_schema),
+ output,
+ )?)
+ }
+}
diff --git a/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/single_table.rs b/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/single_table.rs
new file mode 100644
index 0000000000..56d601c793
--- /dev/null
+++ b/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/single_table.rs
@@ -0,0 +1,76 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use arrow::datatypes::SchemaRef;
+use arrow::record_batch::RecordBatch;
+use datafusion_common::Result;
+
+use crate::aggregates::AggregateExec;
+
+use super::common::{AggregateHashTable, HashAggregateAccumulator, SingleMarker};
+
+/// Implementation specific to single aggregation, where the table stores final
+/// aggregate values and the input rows are raw rows.
+///
+/// Example: `AVG(x) GROUP BY k`
+///
+/// - Aggregate table stores: `k, avg(x)`
+/// - Input rows: `k, x`
+impl AggregateHashTable<SingleMarker> {
+ pub(in crate::aggregates) fn new(
+ agg: &AggregateExec,
+ partition: usize,
+ output_schema: SchemaRef,
+ state_schema: SchemaRef,
+ batch_size: usize,
+ ) -> Result<Self> {
+ Self::new_with_filters(
+ agg,
+ partition,
+ output_schema,
+ state_schema,
+ batch_size,
+ agg.filter_expr.iter().cloned().collect(),
+ )
+ }
+
+ /// Emits the next batch of aggregated group keys and final aggregate values.
+ ///
+ /// The output batch size is determined by `self.batch_size`.
+ ///
+ /// Returns `Some(batch)` for each emitted batch, `None` when output is
+ /// exhausted, and an internal error if polled in the `Building` state.
+ pub(in crate::aggregates) fn next_output_batch(
+ &mut self,
+ ) -> Result<Option<RecordBatch>> {
+ self.next_output_batch_inner(HashAggregateAccumulator::evaluate_to_columns)
+ }
+
+ /// Single aggregation consumes raw input rows and updates the table's
+ /// final-value accumulators.
+ pub(in crate::aggregates) fn aggregate_batch(
+ &mut self,
+ batch: &RecordBatch,
+ ) -> Result<()> {
+ self.aggregate_batch_inner(batch, HashAggregateAccumulator::update_batch)
+ }
+
+ pub(in crate::aggregates) fn start_output(&mut self) -> Result<()> {
+ self.start_outputting();
+ Ok(())
+ }
+}
diff --git a/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_stream.rs b/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_stream.rs
new file mode 100644
index 0000000000..ac7727b459
--- /dev/null
+++ b/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_stream.rs
@@ -0,0 +1,478 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Aggregate without grouping columns
+
+use crate::aggregates::{
+ AccumulatorItem, AggrDynFilter, AggregateInputMode, AggregateMode,
+ DynamicFilterAggregateType, aggregate_expressions, create_accumulators,
+ finalize_aggregation,
+};
+use crate::metrics::{BaselineMetrics, RecordOutput};
+use crate::stream::EmptyRecordBatchStream;
+use crate::{RecordBatchStream, SendableRecordBatchStream};
+use arrow::datatypes::SchemaRef;
+use arrow::record_batch::RecordBatch;
+use datafusion_common::{Result, ScalarValue, internal_datafusion_err, internal_err};
+use datafusion_execution::TaskContext;
+use datafusion_expr::Operator;
+use datafusion_physical_expr::PhysicalExpr;
+use datafusion_physical_expr::expressions::{BinaryExpr, lit};
+use futures::stream::BoxStream;
+use std::borrow::Cow;
+use std::cmp::Ordering;
+use std::sync::Arc;
+use std::task::{Context, Poll};
+
+use super::AggregateExec;
+use crate::filter::batch_filter;
+use datafusion_execution::memory_pool::{MemoryConsumer, MemoryReservation};
+use datafusion_physical_expr_common::utils::evaluate_expressions_to_arrays;
+use futures::stream::{Stream, StreamExt};
+
+/// stream struct for aggregation without grouping columns
+pub(crate) struct AggregateStream {
+ stream: BoxStream<'static, Result<RecordBatch>>,
+ schema: SchemaRef,
+}
+
+/// Actual implementation of [`AggregateStream`].
+///
+/// This is wrapped into yet another struct because we need to interact with the async memory management subsystem
+/// during poll. To have as little code "weirdness" as possible, we chose to just use [`BoxStream`] together with
+/// [`futures::stream::unfold`].
+///
+/// The latter requires a state object, which is [`AggregateStreamInner`].
+struct AggregateStreamInner {
+ // ==== Properties ====
+ schema: SchemaRef,
+ mode: AggregateMode,
+ input: SendableRecordBatchStream,
+ aggregate_expressions: Vec<Vec<Arc<dyn PhysicalExpr>>>,
+ filter_expressions: Arc<[Option<Arc<dyn PhysicalExpr>>]>,
+
+ // ==== Runtime States/Buffers ====
+ accumulators: Vec<AccumulatorItem>,
+ // None if the dynamic filter is not applicable. See details in `AggrDynFilter`.
+ agg_dyn_filter_state: Option<Arc<AggrDynFilter>>,
+ finished: bool,
+
+ // ==== Execution Resources ====
+ baseline_metrics: BaselineMetrics,
+ reservation: MemoryReservation,
+}
+
+impl AggregateStreamInner {
+ // TODO: check if we get Null handling correct
+ /// # Examples
+ /// - Example 1
+ /// Accumulators: min(c1)
+ /// Current Bounds: min(c1)=10
+ /// --> dynamic filter PhysicalExpr: c1 < 10
+ ///
+ /// - Example 2
+ /// Accumulators: min(c1), max(c1), min(c2)
+ /// Current Bounds: min(c1)=10, max(c1)=100, min(c2)=20
+ /// --> dynamic filter PhysicalExpr: (c1 < 10) OR (c1>100) OR (c2 < 20)
+ ///
+ /// # Errors
+ /// Returns internal errors if the dynamic filter is not enabled, or other
+ /// invariant check fails.
+ fn build_dynamic_filter_from_accumulator_bounds(
+ &self,
+ ) -> Result<Arc<dyn PhysicalExpr>> {
+ let Some(filter_state) = self.agg_dyn_filter_state.as_ref() else {
+ return internal_err!(
+ "`build_dynamic_filter_from_accumulator_bounds()` is only called when dynamic filter is enabled"
+ );
+ };
+
+ let mut predicates: Vec<Arc<dyn PhysicalExpr>> =
+ Vec::with_capacity(filter_state.supported_accumulators_info.len());
+
+ for acc_info in &filter_state.supported_accumulators_info {
+ // Skip if we don't yet have a meaningful bound
+ let bound = {
+ let guard = acc_info.shared_bound.lock();
+ if (*guard).is_null() {
+ continue;
+ }
+ guard.clone()
+ };
+
+ let agg_exprs = self
+ .aggregate_expressions
+ .get(acc_info.aggr_index)
+ .ok_or_else(|| {
+ internal_datafusion_err!(
+ "Invalid aggregate expression index {} for dynamic filter",
+ acc_info.aggr_index
+ )
+ })?;
+ // Only aggregates with a single argument are supported.
+ let column_expr = agg_exprs.first().ok_or_else(|| {
+ internal_datafusion_err!(
+ "Aggregate expression at index {} expected a single argument",
+ acc_info.aggr_index
+ )
+ })?;
+
+ let literal = lit(bound);
+ let predicate: Arc<dyn PhysicalExpr> = match acc_info.aggr_type {
+ DynamicFilterAggregateType::Min => Arc::new(BinaryExpr::new(
+ Arc::clone(column_expr),
+ Operator::Lt,
+ literal,
+ )),
+ DynamicFilterAggregateType::Max => Arc::new(BinaryExpr::new(
+ Arc::clone(column_expr),
+ Operator::Gt,
+ literal,
+ )),
+ };
+ predicates.push(predicate);
+ }
+
+ let combined = predicates.into_iter().reduce(|acc, pred| {
+ Arc::new(BinaryExpr::new(acc, Operator::Or, pred)) as Arc<dyn PhysicalExpr>
+ });
+
+ Ok(combined.unwrap_or_else(|| lit(true)))
+ }
+
+ // If the dynamic filter is enabled, update it using the current accumulator's
+ // values
+ fn maybe_update_dyn_filter(&mut self) -> Result<()> {
+ // Step 1: Update each partition's current bound
+ let Some(filter_state) = self.agg_dyn_filter_state.as_ref() else {
+ return Ok(());
+ };
+
+ let mut bounds_changed = false;
+
+ for acc_info in &filter_state.supported_accumulators_info {
+ let acc =
+ self.accumulators
+ .get_mut(acc_info.aggr_index)
+ .ok_or_else(|| {
+ internal_datafusion_err!(
+ "Invalid accumulator index {} for dynamic filter",
+ acc_info.aggr_index
+ )
+ })?;
+ // First get current partition's bound, then update the shared bound among
+ // all partitions.
+ let current_bound = acc.evaluate()?;
+ {
+ let mut bound = acc_info.shared_bound.lock();
+ let new_bound = match acc_info.aggr_type {
+ DynamicFilterAggregateType::Max => {
+ scalar_max(&bound, ¤t_bound)?
+ }
+ DynamicFilterAggregateType::Min => {
+ scalar_min(&bound, ¤t_bound)?
+ }
+ };
+ if new_bound != *bound {
+ *bound = new_bound;
+ bounds_changed = true;
+ }
+ }
+ }
+
+ // Step 2: Sync the dynamic filter physical expression with reader,
+ // but only if any bound actually changed.
+ if bounds_changed {
+ let predicate = self.build_dynamic_filter_from_accumulator_bounds()?;
+ filter_state.filter.update(predicate)?;
+ }
+
+ Ok(())
+ }
+}
+
+/// Returns the element-wise minimum of two `ScalarValue`s.
+///
+/// # Null semantics
+/// - `min(NULL, NULL) = NULL`
+/// - `min(NULL, x) = x`
+/// - `min(x, NULL) = x`
+///
+/// # Errors
+/// Returns internal error if v1 and v2 has incompatible types.
+fn scalar_min(v1: &ScalarValue, v2: &ScalarValue) -> Result<ScalarValue> {
+ if let Some(result) = scalar_cmp_null_short_circuit(v1, v2) {
+ return Ok(result);
+ }
+
+ match v1.partial_cmp(v2) {
+ Some(Ordering::Less | Ordering::Equal) => Ok(v1.clone()),
+ Some(Ordering::Greater) => Ok(v2.clone()),
+ None => datafusion_common::internal_err!(
+ "cannot compare values of different or incompatible types: {v1:?} vs {v2:?}"
+ ),
+ }
+}
+
+/// Returns the element-wise maximum of two `ScalarValue`s.
+///
+/// # Null semantics
+/// - `max(NULL, NULL) = NULL`
+/// - `max(NULL, x) = x`
+/// - `max(x, NULL) = x`
+///
+/// # Errors
+/// Returns internal error if v1 and v2 has incompatible types.
+fn scalar_max(v1: &ScalarValue, v2: &ScalarValue) -> Result<ScalarValue> {
+ if let Some(result) = scalar_cmp_null_short_circuit(v1, v2) {
+ return Ok(result);
+ }
+
+ match v1.partial_cmp(v2) {
+ Some(Ordering::Greater | Ordering::Equal) => Ok(v1.clone()),
+ Some(Ordering::Less) => Ok(v2.clone()),
+ None => datafusion_common::internal_err!(
+ "cannot compare values of different or incompatible types: {v1:?} vs {v2:?}"
+ ),
+ }
+}
+
+fn scalar_cmp_null_short_circuit(
+ v1: &ScalarValue,
+ v2: &ScalarValue,
+) -> Option<ScalarValue> {
+ match (v1, v2) {
+ (ScalarValue::Null, ScalarValue::Null) => Some(ScalarValue::Null),
+ (ScalarValue::Null, other) | (other, ScalarValue::Null) => Some(other.clone()),
+ _ => None,
+ }
+}
+
+/// Prepend the grouping ID column to the output columns if present.
+///
+/// For GROUPING SETS with no GROUP BY expressions, the schema includes a `__grouping_id`
+/// column that must be present in the output. This function inserts it at the beginning
+/// of the columns array to maintain schema alignment.
+fn prepend_grouping_id_column(
+ mut columns: Vec<Arc<dyn arrow::array::Array>>,
+ grouping_id: Option<&ScalarValue>,
+) -> Result<Vec<Arc<dyn arrow::array::Array>>> {
+ if let Some(id) = grouping_id {
+ let num_rows = columns.first().map(|array| array.len()).unwrap_or(1);
+ let grouping_ids = id.to_array_of_size(num_rows)?;
+ columns.insert(0, grouping_ids);
+ }
+ Ok(columns)
+}
+
+impl AggregateStream {
+ /// Create a new AggregateStream
+ pub fn new(
+ agg: &AggregateExec,
+ context: &Arc<TaskContext>,
+ partition: usize,
+ ) -> Result<Self> {
+ let agg_schema = Arc::clone(&agg.schema);
+ let agg_filter_expr = Arc::clone(&agg.filter_expr);
+
+ let baseline_metrics = BaselineMetrics::new(&agg.metrics, partition);
+ let input = agg.input.execute(partition, Arc::clone(context))?;
+
+ let aggregate_expressions = aggregate_expressions(&agg.aggr_expr, &agg.mode, 0)?;
+ let filter_expressions = match agg.mode.input_mode() {
+ AggregateInputMode::Raw => agg_filter_expr,
+ AggregateInputMode::Partial => vec![None; agg.aggr_expr.len()].into(),
+ };
+ let accumulators = create_accumulators(&agg.aggr_expr)?;
+
+ let reservation = MemoryConsumer::new(format!("AggregateStream[{partition}]"))
+ .register(context.memory_pool());
+
+ // Enable dynamic filter if:
+ // 1. AggregateExec did the check and ensure it supports the dynamic filter
+ // (its dynamic_filter field will be Some(..))
+ // 2. Aggregate dynamic filter is enabled from the config
+ let mut maybe_dynamic_filter = match agg.dynamic_filter.as_ref() {
+ Some(filter) => Some(Arc::clone(filter)),
+ _ => None,
+ };
+
+ if !context
+ .session_config()
+ .options()
+ .optimizer
+ .enable_aggregate_dynamic_filter_pushdown
+ {
+ maybe_dynamic_filter = None;
+ }
+
+ let inner = AggregateStreamInner {
+ schema: Arc::clone(&agg.schema),
+ mode: agg.mode,
+ input,
+ baseline_metrics,
+ aggregate_expressions,
+ filter_expressions,
+ accumulators,
+ reservation,
+ finished: false,
+ agg_dyn_filter_state: maybe_dynamic_filter,
+ };
+
+ let stream = futures::stream::unfold(inner, |mut this| async move {
+ if this.finished {
+ return None;
+ }
+
+ loop {
+ let result = match this.input.next().await {
+ Some(Ok(batch)) => {
+ let result = {
+ let elapsed_compute = this.baseline_metrics.elapsed_compute();
+ let _timer = elapsed_compute.timer(); // Stops on drop
+ aggregate_batch(
+ &this.mode,
+ &batch,
+ &mut this.accumulators,
+ &this.aggregate_expressions,
+ &this.filter_expressions,
+ )
+ };
+
+ let result = result.and_then(|allocated| {
+ this.maybe_update_dyn_filter()?;
+ Ok(allocated)
+ });
+
+ // allocate memory
+ // This happens AFTER we actually used the memory, but simplifies the whole accounting and we are OK with
+ // overshooting a bit. Also this means we either store the whole record batch or not.
+ match result
+ .and_then(|allocated| this.reservation.try_grow(allocated))
+ {
+ Ok(_) => continue,
+ Err(e) => Err(e),
+ }
+ }
+ Some(Err(e)) => Err(e),
+ None => {
+ this.finished = true;
+ // Release the input pipeline's resources before finalization.
+ let input_schema = this.input.schema();
+ this.input = Box::pin(EmptyRecordBatchStream::new(input_schema));
+ let timer = this.baseline_metrics.elapsed_compute().timer();
+ let result =
+ finalize_aggregation(&mut this.accumulators, &this.mode)
+ .and_then(|columns| {
+ prepend_grouping_id_column(columns, None)
+ })
+ .and_then(|columns| {
+ RecordBatch::try_new(
+ Arc::clone(&this.schema),
+ columns,
+ )
+ .map_err(Into::into)
+ })
+ .record_output(&this.baseline_metrics);
+
+ timer.done();
+
+ result
+ }
+ };
+
+ this.finished = true;
+ return Some((result, this));
+ }
+ });
+
+ // seems like some consumers call this stream even after it returned `None`, so let's fuse the stream.
+ let stream = stream.fuse();
+ let stream = Box::pin(stream);
+
+ Ok(Self {
+ schema: agg_schema,
+ stream,
+ })
+ }
+}
+
+impl Stream for AggregateStream {
+ type Item = Result<RecordBatch>;
+
+ fn poll_next(
+ mut self: std::pin::Pin<&mut Self>,
+ cx: &mut Context<'_>,
+ ) -> Poll<Option<Self::Item>> {
+ let this = &mut *self;
+ this.stream.poll_next_unpin(cx)
+ }
+}
+
+impl RecordBatchStream for AggregateStream {
+ fn schema(&self) -> SchemaRef {
+ Arc::clone(&self.schema)
+ }
+}
+
+/// Perform group-by aggregation for the given [`RecordBatch`].
+///
+/// If successful, this returns the additional number of bytes that were allocated during this process.
+///
+/// TODO: Make this a member function
+fn aggregate_batch(
+ mode: &AggregateMode,
+ batch: &RecordBatch,
+ accumulators: &mut [AccumulatorItem],
+ expressions: &[Vec<Arc<dyn PhysicalExpr>>],
+ filters: &[Option<Arc<dyn PhysicalExpr>>],
+) -> Result<usize> {
+ let mut allocated = 0usize;
+
+ // 1.1 iterate accumulators and respective expressions together
+ // 1.2 filter the batch if necessary
+ // 1.3 evaluate expressions
+ // 1.4 update / merge accumulators with the expressions' values
+
+ // 1.1
+ accumulators
+ .iter_mut()
+ .zip(expressions)
+ .zip(filters)
+ .try_for_each(|((accum, expr), filter)| {
+ // 1.2
+ let batch = match filter {
+ Some(filter) => Cow::Owned(batch_filter(batch, filter)?),
+ None => Cow::Borrowed(batch),
+ };
+
+ // 1.3
+ let values = evaluate_expressions_to_arrays(expr, batch.as_ref())?;
+
+ // 1.4
+ let size_pre = accum.size();
+ let res = match mode.input_mode() {
+ AggregateInputMode::Raw => accum.update_batch(&values),
+ AggregateInputMode::Partial => accum.merge_batch(&values),
+ };
+ let size_post = accum.size();
+ allocated += size_post.saturating_sub(size_pre);
+ res
+ })?;
+
+ Ok(allocated)
+}
diff --git a/native/vendor/datafusion-physical-plan/src/aggregates/group_values/metrics.rs b/native/vendor/datafusion-physical-plan/src/aggregates/group_values/metrics.rs
new file mode 100644
index 0000000000..1c6285d793
--- /dev/null
+++ b/native/vendor/datafusion-physical-plan/src/aggregates/group_values/metrics.rs
@@ -0,0 +1,222 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Metrics for the various group-by implementations.
+
+use crate::metrics::{ExecutionPlanMetricsSet, MetricBuilder, Time};
+
+#[derive(Clone)]
+pub(crate) struct GroupByMetrics {
+ /// Time spent calculating the group IDs from the evaluated grouping columns.
+ pub(crate) time_calculating_group_ids: Time,
+ /// Time spent evaluating the inputs to the aggregate functions.
+ pub(crate) aggregate_arguments_time: Time,
+ /// Time spent evaluating the aggregate expressions themselves
+ /// (e.g. summing all elements and counting number of elements for `avg` aggregate).
+ pub(crate) aggregation_time: Time,
+ /// Time spent emitting the final results and constructing the record batch
+ /// which includes finalizing the grouping expressions
+ /// (e.g. emit from the hash table in case of hash aggregation) and the accumulators
+ pub(crate) emitting_time: Time,
+}
+
+impl GroupByMetrics {
+ pub(crate) fn new(metrics: &ExecutionPlanMetricsSet, partition: usize) -> Self {
+ Self {
+ time_calculating_group_ids: MetricBuilder::new(metrics)
+ .subset_time("time_calculating_group_ids", partition),
+ aggregate_arguments_time: MetricBuilder::new(metrics)
+ .subset_time("aggregate_arguments_time", partition),
+ aggregation_time: MetricBuilder::new(metrics)
+ .subset_time("aggregation_time", partition),
+ emitting_time: MetricBuilder::new(metrics)
+ .subset_time("emitting_time", partition),
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use crate::aggregates::{AggregateExec, AggregateMode, PhysicalGroupBy};
+ use crate::metrics::MetricsSet;
+ use crate::test::TestMemoryExec;
+ use crate::{ExecutionPlan, collect};
+ use arrow::array::{Float64Array, UInt32Array};
+ use arrow::datatypes::{DataType, Field, Schema};
+ use arrow::record_batch::RecordBatch;
+ use datafusion_common::Result;
+ use datafusion_execution::TaskContext;
+ use datafusion_execution::runtime_env::RuntimeEnvBuilder;
+ use datafusion_functions_aggregate::count::count_udaf;
+ use datafusion_functions_aggregate::sum::sum_udaf;
+ use datafusion_physical_expr::aggregate::AggregateExprBuilder;
+ use datafusion_physical_expr::expressions::col;
+ use std::sync::Arc;
+
+ /// Helper function to verify all three GroupBy metrics exist and have non-zero values
+ fn assert_groupby_metrics(metrics: &MetricsSet) {
+ let agg_arguments_time = metrics.sum_by_name("aggregate_arguments_time");
+ assert!(agg_arguments_time.is_some());
+ assert!(agg_arguments_time.unwrap().as_usize() > 0);
+
+ let aggregation_time = metrics.sum_by_name("aggregation_time");
+ assert!(aggregation_time.is_some());
+ assert!(aggregation_time.unwrap().as_usize() > 0);
+
+ let emitting_time = metrics.sum_by_name("emitting_time");
+ assert!(emitting_time.is_some());
+ assert!(emitting_time.unwrap().as_usize() > 0);
+ }
+
+ #[tokio::test]
+ async fn test_groupby_metrics_partial_mode() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("a", DataType::UInt32, false),
+ Field::new("b", DataType::Float64, false),
+ ]));
+
+ // Create multiple batches to ensure metrics accumulate
+ let batches = (0..5)
+ .map(|i| {
+ RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![
+ Arc::new(UInt32Array::from(vec![1, 2, 3, 4])),
+ Arc::new(Float64Array::from(vec![
+ i as f64,
+ (i + 1) as f64,
+ (i + 2) as f64,
+ (i + 3) as f64,
+ ])),
+ ],
+ )
+ .unwrap()
+ })
+ .collect::<Vec<_>>();
+
+ let input = TestMemoryExec::try_new_exec(&[batches], Arc::clone(&schema), None)?;
+
+ let group_by =
+ PhysicalGroupBy::new_single(vec![(col("a", &schema)?, "a".to_string())]);
+
+ let aggregates = vec![
+ Arc::new(
+ AggregateExprBuilder::new(sum_udaf(), vec![col("b", &schema)?])
+ .schema(Arc::clone(&schema))
+ .alias("SUM(b)")
+ .build()?,
+ ),
+ Arc::new(
+ AggregateExprBuilder::new(count_udaf(), vec![col("b", &schema)?])
+ .schema(Arc::clone(&schema))
+ .alias("COUNT(b)")
+ .build()?,
+ ),
+ ];
+
+ let aggregate_exec = Arc::new(AggregateExec::try_new(
+ AggregateMode::Partial,
+ group_by,
+ aggregates,
+ vec![None, None],
+ input,
+ schema,
+ )?);
+
+ // This test is for `GroupByMetrics`, which are maintained by
+ // `GroupedHashAggregateStream`. Use a finite memory pool so the partial
+ // aggregate does not take the initial-partial stream path.
+ let runtime = RuntimeEnvBuilder::new()
+ .with_memory_limit(10 * 1024 * 1024, 1.0)
+ .build_arc()?;
+ let task_ctx = Arc::new(TaskContext::default().with_runtime(runtime));
+ let _result =
+ collect(Arc::clone(&aggregate_exec) as _, Arc::clone(&task_ctx)).await?;
+
+ let metrics = aggregate_exec.metrics().unwrap();
+ assert_groupby_metrics(&metrics);
+
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn test_groupby_metrics_final_mode() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("a", DataType::UInt32, false),
+ Field::new("b", DataType::Float64, false),
+ ]));
+
+ let batches = (0..3)
+ .map(|i| {
+ RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![
+ Arc::new(UInt32Array::from(vec![1, 2, 3])),
+ Arc::new(Float64Array::from(vec![
+ i as f64,
+ (i + 1) as f64,
+ (i + 2) as f64,
+ ])),
+ ],
+ )
+ .unwrap()
+ })
+ .collect::<Vec<_>>();
+
+ let partial_input =
+ TestMemoryExec::try_new_exec(&[batches], Arc::clone(&schema), None)?;
+
+ let group_by =
+ PhysicalGroupBy::new_single(vec![(col("a", &schema)?, "a".to_string())]);
+
+ let aggregates = vec![Arc::new(
+ AggregateExprBuilder::new(sum_udaf(), vec![col("b", &schema)?])
+ .schema(Arc::clone(&schema))
+ .alias("SUM(b)")
+ .build()?,
+ )];
+
+ // Create partial aggregate
+ let partial_aggregate = Arc::new(AggregateExec::try_new(
+ AggregateMode::Partial,
+ group_by.clone(),
+ aggregates.clone(),
+ vec![None],
+ partial_input,
+ Arc::clone(&schema),
+ )?);
+
+ // Create final aggregate
+ let final_aggregate = Arc::new(AggregateExec::try_new(
+ AggregateMode::Final,
+ group_by.as_final(),
+ aggregates,
+ vec![None],
+ partial_aggregate,
+ schema,
+ )?);
+
+ let task_ctx = Arc::new(TaskContext::default());
+ let _result =
+ collect(Arc::clone(&final_aggregate) as _, Arc::clone(&task_ctx)).await?;
+
+ let metrics = final_aggregate.metrics().unwrap();
+ assert_groupby_metrics(&metrics);
+
+ Ok(())
+ }
+}
diff --git a/native/vendor/datafusion-physical-plan/src/aggregates/group_values/mod.rs b/native/vendor/datafusion-physical-plan/src/aggregates/group_values/mod.rs
new file mode 100644
index 0000000000..1101d53531
--- /dev/null
+++ b/native/vendor/datafusion-physical-plan/src/aggregates/group_values/mod.rs
@@ -0,0 +1,214 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! [`GroupValues`] trait for storing and interning group keys
+
+use arrow::array::types::{
+ Date32Type, Date64Type, Decimal128Type, Time32MillisecondType, Time32SecondType,
+ Time64MicrosecondType, Time64NanosecondType, TimestampMicrosecondType,
+ TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType,
+};
+use arrow::array::{ArrayRef, downcast_primitive};
+use arrow::datatypes::{DataType, SchemaRef, TimeUnit};
+use datafusion_common::Result;
+
+use datafusion_expr::EmitTo;
+
+pub mod multi_group_by;
+
+mod row;
+pub use row::GroupValuesRows;
+mod single_group_by;
+use datafusion_physical_expr::binary_map::OutputType;
+use multi_group_by::GroupValuesColumn;
+
+pub(crate) use single_group_by::primitive::HashValue;
+
+use crate::aggregates::{
+ group_values::single_group_by::{
+ boolean::GroupValuesBoolean, bytes::GroupValuesBytes,
+ bytes_view::GroupValuesBytesView, primitive::GroupValuesPrimitive,
+ },
+ order::GroupOrdering,
+};
+
+mod metrics;
+mod null_builder;
+
+pub(crate) use metrics::GroupByMetrics;
+
+/// Stores the group values during hash aggregation.
+///
+/// # Background
+///
+/// In a query such as `SELECT a, b, count(*) FROM t GROUP BY a, b`, the group values
+/// identify each group, and correspond to all the distinct values of `(a,b)`.
+///
+/// ```sql
+/// -- Input has 4 rows with 3 distinct combinations of (a,b) ("groups")
+/// create table t(a int, b varchar)
+/// as values (1, 'a'), (2, 'b'), (1, 'a'), (3, 'c');
+///
+/// select a, b, count(*) from t group by a, b;
+/// ----
+/// 1 a 2
+/// 2 b 1
+/// 3 c 1
+/// ```
+///
+/// # Design
+///
+/// Managing group values is a performance critical operation in hash
+/// aggregation. The major operations are:
+///
+/// 1. Intern: Quickly finding existing and adding new group values
+/// 2. Emit: Returning the group values as an array
+///
+/// There are multiple specialized implementations of this trait optimized for
+/// different data types and number of columns, optimized for these operations.
+/// See [`new_group_values`] for details.
+///
+/// # Group Ids
+///
+/// Each distinct group in a hash aggregation is identified by a unique group id
+/// (usize) which is assigned by instances of this trait. Group ids are
+/// continuous without gaps, starting from 0.
+pub trait GroupValues: Send {
+ /// Calculates the group id for each input row of `cols`, assigning new
+ /// group ids as necessary.
+ ///
+ /// When the function returns, `groups` must contain the group id for each
+ /// row in `cols`.
+ ///
+ /// If a row has the same value as a previous row, the same group id is
+ /// assigned. If a row has a new value, the next available group id is
+ /// assigned.
+ fn intern(&mut self, cols: &[ArrayRef], groups: &mut Vec<usize>) -> Result<()>;
+
+ /// Returns the number of bytes of memory used by this [`GroupValues`].
+ ///
+ /// May be expensive; check the implementation before calling on hot paths.
+ fn size(&self) -> usize;
+
+ /// Returns true if this [`GroupValues`] is empty
+ fn is_empty(&self) -> bool;
+
+ /// The number of values (distinct group values) stored in this [`GroupValues`]
+ fn len(&self) -> usize;
+
+ /// Emits the group values
+ fn emit(&mut self, emit_to: EmitTo) -> Result<Vec<ArrayRef>>;
+
+ /// Clear the contents and shrink the capacity to the size of the batch (free up memory usage)
+ fn clear_shrink(&mut self, num_rows: usize);
+}
+
+/// Return a specialized implementation of [`GroupValues`] for the given schema.
+///
+/// [`GroupValues`] implementations choosing logic:
+///
+/// - If group by single column, and type of this column has
+/// the specific [`GroupValues`] implementation, such implementation
+/// will be chosen.
+///
+/// - If group by multiple columns, and all column types have the specific
+/// `GroupColumn` implementations, `GroupValuesColumn` will be chosen.
+///
+/// - Otherwise, the general implementation `GroupValuesRows` will be chosen.
+///
+/// `GroupColumn`: crate::aggregates::group_values::multi_group_by::GroupColumn
+/// `GroupValuesColumn`: crate::aggregates::group_values::multi_group_by::GroupValuesColumn
+/// `GroupValuesRows`: crate::aggregates::group_values::GroupValuesRows
+pub fn new_group_values(
+ schema: SchemaRef,
+ group_ordering: &GroupOrdering,
+) -> Result<Box<dyn GroupValues>> {
+ if schema.fields.len() == 1 {
+ let d = schema.fields[0].data_type();
+
+ macro_rules! downcast_helper {
+ ($t:ty, $d:ident) => {
+ return Ok(Box::new(GroupValuesPrimitive::<$t>::new($d.clone())))
+ };
+ }
+
+ downcast_primitive! {
+ d => (downcast_helper, d),
+ _ => {}
+ }
+
+ match d {
+ DataType::Date32 => {
+ downcast_helper!(Date32Type, d);
+ }
+ DataType::Date64 => {
+ downcast_helper!(Date64Type, d);
+ }
+ DataType::Time32(t) => match t {
+ TimeUnit::Second => downcast_helper!(Time32SecondType, d),
+ TimeUnit::Millisecond => downcast_helper!(Time32MillisecondType, d),
+ _ => {}
+ },
+ DataType::Time64(t) => match t {
+ TimeUnit::Microsecond => downcast_helper!(Time64MicrosecondType, d),
+ TimeUnit::Nanosecond => downcast_helper!(Time64NanosecondType, d),
+ _ => {}
+ },
+ DataType::Timestamp(t, _tz) => match t {
+ TimeUnit::Second => downcast_helper!(TimestampSecondType, d),
+ TimeUnit::Millisecond => downcast_helper!(TimestampMillisecondType, d),
+ TimeUnit::Microsecond => downcast_helper!(TimestampMicrosecondType, d),
+ TimeUnit::Nanosecond => downcast_helper!(TimestampNanosecondType, d),
+ },
+ DataType::Decimal128(_, _) => {
+ downcast_helper!(Decimal128Type, d);
+ }
+ DataType::Utf8 => {
+ return Ok(Box::new(GroupValuesBytes::<i32>::new(OutputType::Utf8)));
+ }
+ DataType::LargeUtf8 => {
+ return Ok(Box::new(GroupValuesBytes::<i64>::new(OutputType::Utf8)));
+ }
+ DataType::Utf8View => {
+ return Ok(Box::new(GroupValuesBytesView::new(OutputType::Utf8View)));
+ }
+ DataType::Binary => {
+ return Ok(Box::new(GroupValuesBytes::<i32>::new(OutputType::Binary)));
+ }
+ DataType::LargeBinary => {
+ return Ok(Box::new(GroupValuesBytes::<i64>::new(OutputType::Binary)));
+ }
+ DataType::BinaryView => {
+ return Ok(Box::new(GroupValuesBytesView::new(OutputType::BinaryView)));
+ }
+ DataType::Boolean => {
+ return Ok(Box::new(GroupValuesBoolean::new()));
+ }
+ _ => {}
+ }
+ }
+
+ if multi_group_by::supported_schema(schema.as_ref()) {
+ if matches!(group_ordering, GroupOrdering::None) {
+ Ok(Box::new(GroupValuesColumn::<false>::try_new(schema)?))
+ } else {
+ Ok(Box::new(GroupValuesColumn::<true>::try_new(schema)?))
+ }
+ } else {
+ Ok(Box::new(GroupValuesRows::try_new(schema)?))
+ }
+}
diff --git a/native/vendor/datafusion-physical-plan/src/aggregates/group_values/multi_group_by/boolean.rs b/native/vendor/datafusion-physical-plan/src/aggregates/group_values/multi_group_by/boolean.rs
new file mode 100644
index 0000000000..5fdbe434f9
--- /dev/null
+++ b/native/vendor/datafusion-physical-plan/src/aggregates/group_values/multi_group_by/boolean.rs
@@ -0,0 +1,493 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use crate::aggregates::group_values::multi_group_by::Nulls;
+use crate::aggregates::group_values::multi_group_by::{GroupColumn, nulls_equal_to};
+use crate::aggregates::group_values::null_builder::MaybeNullBufferBuilder;
+use arrow::array::{Array as _, ArrayRef, AsArray, BooleanArray, BooleanBufferBuilder};
+use datafusion_common::Result;
+
+/// An implementation of [`GroupColumn`] for booleans
+///
+/// Optimized to skip null buffer construction if the input is known to be non nullable
+///
+/// # Template parameters
+///
+/// `NULLABLE`: if the data can contain any nulls
+#[derive(Debug)]
+pub struct BooleanGroupValueBuilder<const NULLABLE: bool> {
+ buffer: BooleanBufferBuilder,
+ nulls: MaybeNullBufferBuilder,
+}
+
+impl<const NULLABLE: bool> BooleanGroupValueBuilder<NULLABLE> {
+ /// Create a new `BooleanGroupValueBuilder`
+ pub fn new() -> Self {
+ Self {
+ buffer: BooleanBufferBuilder::new(0),
+ nulls: MaybeNullBufferBuilder::new(),
+ }
+ }
+}
+
+impl<const NULLABLE: bool> GroupColumn for BooleanGroupValueBuilder<NULLABLE> {
+ fn equal_to(&self, lhs_row: usize, array: &ArrayRef, rhs_row: usize) -> bool {
+ if NULLABLE {
+ let exist_null = self.nulls.is_null(lhs_row);
+ let input_null = array.is_null(rhs_row);
+ if let Some(result) = nulls_equal_to(exist_null, input_null) {
+ return result;
+ }
+ }
+
+ self.buffer.get_bit(lhs_row) == array.as_boolean().value(rhs_row)
+ }
+
+ fn append_val(&mut self, array: &ArrayRef, row: usize) -> Result<()> {
+ if NULLABLE {
+ if array.is_null(row) {
+ self.nulls.append(true);
+ self.buffer.append(bool::default());
+ } else {
+ self.nulls.append(false);
+ self.buffer.append(array.as_boolean().value(row));
+ }
+ } else {
+ self.buffer.append(array.as_boolean().value(row));
+ }
+
+ Ok(())
+ }
+
+ fn vectorized_equal_to(
+ &self,
+ lhs_rows: &[usize],
+ array: &ArrayRef,
+ rhs_rows: &[usize],
+ equal_to_results: &mut BooleanBufferBuilder,
+ ) {
+ let array = array.as_boolean();
+
+ for (idx, (&lhs_row, &rhs_row)) in
+ lhs_rows.iter().zip(rhs_rows.iter()).enumerate()
+ {
+ if !equal_to_results.get_bit(idx) {
+ continue;
+ }
+
+ if NULLABLE {
+ let exist_null = self.nulls.is_null(lhs_row);
+ let input_null = array.is_null(rhs_row);
+ if let Some(result) = nulls_equal_to(exist_null, input_null) {
+ if !result {
+ equal_to_results.set_bit(idx, false);
+ }
+ continue;
+ }
+ }
+
+ if self.buffer.get_bit(lhs_row) != array.value(rhs_row) {
+ equal_to_results.set_bit(idx, false);
+ }
+ }
+ }
+
+ fn vectorized_append(&mut self, array: &ArrayRef, rows: &[usize]) -> Result<()> {
+ let arr = array.as_boolean();
+
+ let null_count = array.null_count();
+ let num_rows = array.len();
+ let all_null_or_non_null = if null_count == 0 {
+ Nulls::None
+ } else if null_count == num_rows {
+ Nulls::All
+ } else {
+ Nulls::Some
+ };
+
+ match (NULLABLE, all_null_or_non_null) {
+ (true, Nulls::Some) => {
+ for &row in rows {
+ if array.is_null(row) {
+ self.nulls.append(true);
+ self.buffer.append(bool::default());
+ } else {
+ self.nulls.append(false);
+ self.buffer.append(arr.value(row));
+ }
+ }
+ }
+
+ (true, Nulls::None) => {
+ self.nulls.append_n(rows.len(), false);
+ for &row in rows {
+ self.buffer.append(arr.value(row));
+ }
+ }
+
+ (true, Nulls::All) => {
+ self.nulls.append_n(rows.len(), true);
+ self.buffer.append_n(rows.len(), bool::default());
+ }
+
+ (false, _) => {
+ for &row in rows {
+ self.buffer.append(arr.value(row));
+ }
+ }
+ }
+
+ Ok(())
+ }
+
+ fn len(&self) -> usize {
+ self.buffer.len()
+ }
+
+ fn size(&self) -> usize {
+ self.buffer.capacity() / 8 + self.nulls.allocated_size()
+ }
+
+ fn build(self: Box<Self>) -> ArrayRef {
+ let Self { mut buffer, nulls } = *self;
+
+ let nulls = nulls.build();
+ if !NULLABLE {
+ assert!(nulls.is_none(), "unexpected nulls in non nullable input");
+ }
+
+ let arr = BooleanArray::new(buffer.finish(), nulls);
+
+ Arc::new(arr)
+ }
+
+ fn take_n(&mut self, n: usize) -> ArrayRef {
+ let first_n_nulls = if NULLABLE { self.nulls.take_n(n) } else { None };
+
+ let mut new_builder = BooleanBufferBuilder::new(self.buffer.len());
+ new_builder.append_packed_range(n..self.buffer.len(), self.buffer.as_slice());
+ std::mem::swap(&mut new_builder, &mut self.buffer);
+
+ // take only first n values from the original builder
+ new_builder.truncate(n);
+
+ Arc::new(BooleanArray::new(new_builder.finish(), first_n_nulls))
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use arrow::array::{BooleanBufferBuilder, NullBufferBuilder};
+
+ use super::*;
+
+ fn make_true_buffer(n: usize) -> BooleanBufferBuilder {
+ let mut buf = BooleanBufferBuilder::new(n);
+ buf.append_n(n, true);
+ buf
+ }
+
+ fn to_vec(buf: &BooleanBufferBuilder) -> Vec<bool> {
+ (0..buf.len()).map(|i| buf.get_bit(i)).collect()
+ }
+
+ #[test]
+ fn test_nullable_boolean_equal_to() {
+ let append = |builder: &mut BooleanGroupValueBuilder<true>,
+ builder_array: &ArrayRef,
+ append_rows: &[usize]| {
+ for &index in append_rows {
+ builder.append_val(builder_array, index).unwrap();
+ }
+ };
+
+ let equal_to =
+ |builder: &BooleanGroupValueBuilder<true>,
+ lhs_rows: &[usize],
+ input_array: &ArrayRef,
+ rhs_rows: &[usize],
+ equal_to_results: &mut BooleanBufferBuilder| {
+ let iter = lhs_rows.iter().zip(rhs_rows.iter());
+ for (idx, (&lhs_row, &rhs_row)) in iter.enumerate() {
+ equal_to_results
+ .set_bit(idx, builder.equal_to(lhs_row, input_array, rhs_row));
+ }
+ };
+
+ test_nullable_boolean_equal_to_internal(append, equal_to);
+ }
+
+ #[test]
+ fn test_nullable_primitive_vectorized_equal_to() {
+ let append = |builder: &mut BooleanGroupValueBuilder<true>,
+ builder_array: &ArrayRef,
+ append_rows: &[usize]| {
+ builder
+ .vectorized_append(builder_array, append_rows)
+ .unwrap();
+ };
+
+ let equal_to =
+ |builder: &BooleanGroupValueBuilder<true>,
+ lhs_rows: &[usize],
+ input_array: &ArrayRef,
+ rhs_rows: &[usize],
+ equal_to_results: &mut BooleanBufferBuilder| {
+ builder.vectorized_equal_to(
+ lhs_rows,
+ input_array,
+ rhs_rows,
+ equal_to_results,
+ );
+ };
+
+ test_nullable_boolean_equal_to_internal(append, equal_to);
+ }
+
+ fn test_nullable_boolean_equal_to_internal<A, E>(mut append: A, mut equal_to: E)
+ where
+ A: FnMut(&mut BooleanGroupValueBuilder<true>, &ArrayRef, &[usize]),
+ E: FnMut(
+ &BooleanGroupValueBuilder<true>,
+ &[usize],
+ &ArrayRef,
+ &[usize],
+ &mut BooleanBufferBuilder,
+ ),
+ {
+ // Will cover such cases:
+ // - exist null, input not null
+ // - exist null, input null; values not equal
+ // - exist null, input null; values equal
+ // - exist not null, input null
+ // - exist not null, input not null; values not equal
+ // - exist not null, input not null; values equal
+
+ // Define BooleanGroupValueBuilder
+ let mut builder = BooleanGroupValueBuilder::<true>::new();
+ let builder_array = Arc::new(BooleanArray::from(vec![
+ None,
+ None,
+ None,
+ Some(true),
+ Some(false),
+ Some(true),
+ ])) as ArrayRef;
+ append(&mut builder, &builder_array, &[0, 1, 2, 3, 4, 5]);
+
+ // Define input array
+ let (values, _nulls) = BooleanArray::from(vec![
+ Some(true),
+ Some(false),
+ None,
+ None,
+ Some(true),
+ Some(true),
+ ])
+ .into_parts();
+
+ // explicitly build a null buffer where one of the null values also happens to match
+ let mut nulls = NullBufferBuilder::new(6);
+ nulls.append_non_null();
+ nulls.append_null();
+ nulls.append_null();
+ nulls.append_null();
+ nulls.append_non_null();
+ nulls.append_non_null();
+ let input_array = Arc::new(BooleanArray::new(values, nulls.finish())) as ArrayRef;
+
+ // Check
+ let mut equal_to_results = make_true_buffer(builder.len());
+ equal_to(
+ &builder,
+ &[0, 1, 2, 3, 4, 5],
+ &input_array,
+ &[0, 1, 2, 3, 4, 5],
+ &mut equal_to_results,
+ );
+ let results = to_vec(&equal_to_results);
+
+ assert!(!results[0]);
+ assert!(results[1]);
+ assert!(results[2]);
+ assert!(!results[3]);
+ assert!(!results[4]);
+ assert!(results[5]);
+ }
+
+ #[test]
+ fn test_not_nullable_primitive_equal_to() {
+ let append = |builder: &mut BooleanGroupValueBuilder<false>,
+ builder_array: &ArrayRef,
+ append_rows: &[usize]| {
+ for &index in append_rows {
+ builder.append_val(builder_array, index).unwrap();
+ }
+ };
+
+ let equal_to =
+ |builder: &BooleanGroupValueBuilder<false>,
+ lhs_rows: &[usize],
+ input_array: &ArrayRef,
+ rhs_rows: &[usize],
+ equal_to_results: &mut BooleanBufferBuilder| {
+ let iter = lhs_rows.iter().zip(rhs_rows.iter());
+ for (idx, (&lhs_row, &rhs_row)) in iter.enumerate() {
+ equal_to_results
+ .set_bit(idx, builder.equal_to(lhs_row, input_array, rhs_row));
+ }
+ };
+
+ test_not_nullable_boolean_equal_to_internal(append, equal_to);
+ }
+
+ #[test]
+ fn test_not_nullable_primitive_vectorized_equal_to() {
+ let append = |builder: &mut BooleanGroupValueBuilder<false>,
+ builder_array: &ArrayRef,
+ append_rows: &[usize]| {
+ builder
+ .vectorized_append(builder_array, append_rows)
+ .unwrap();
+ };
+
+ let equal_to =
+ |builder: &BooleanGroupValueBuilder<false>,
+ lhs_rows: &[usize],
+ input_array: &ArrayRef,
+ rhs_rows: &[usize],
+ equal_to_results: &mut BooleanBufferBuilder| {
+ builder.vectorized_equal_to(
+ lhs_rows,
+ input_array,
+ rhs_rows,
+ equal_to_results,
+ );
+ };
+
+ test_not_nullable_boolean_equal_to_internal(append, equal_to);
+ }
+
+ fn test_not_nullable_boolean_equal_to_internal<A, E>(mut append: A, mut equal_to: E)
+ where
+ A: FnMut(&mut BooleanGroupValueBuilder<false>, &ArrayRef, &[usize]),
+ E: FnMut(
+ &BooleanGroupValueBuilder<false>,
+ &[usize],
+ &ArrayRef,
+ &[usize],
+ &mut BooleanBufferBuilder,
+ ),
+ {
+ // Will cover such cases:
+ // - values equal
+ // - values not equal
+
+ // Define BooleanGroupValueBuilder
+ let mut builder = BooleanGroupValueBuilder::<false>::new();
+ let builder_array = Arc::new(BooleanArray::from(vec![
+ Some(false),
+ Some(true),
+ Some(false),
+ Some(true),
+ ])) as ArrayRef;
+ append(&mut builder, &builder_array, &[0, 1, 2, 3]);
+
+ // Define input array
+ let input_array = Arc::new(BooleanArray::from(vec![
+ Some(false),
+ Some(false),
+ Some(true),
+ Some(true),
+ ])) as ArrayRef;
+
+ // Check
+ let mut equal_to_results = make_true_buffer(builder.len());
+ equal_to(
+ &builder,
+ &[0, 1, 2, 3],
+ &input_array,
+ &[0, 1, 2, 3],
+ &mut equal_to_results,
+ );
+ let results = to_vec(&equal_to_results);
+
+ assert!(results[0]);
+ assert!(!results[1]);
+ assert!(!results[2]);
+ assert!(results[3]);
+ }
+
+ #[test]
+ fn test_nullable_boolean_vectorized_operation_special_case() {
+ // Test the special `all nulls` or `not nulls` input array case
+ // for vectorized append and equal to
+
+ let mut builder = BooleanGroupValueBuilder::<true>::new();
+
+ // All nulls input array
+ let all_nulls_input_array =
+ Arc::new(BooleanArray::from(vec![None, None, None, None, None])) as _;
+ builder
+ .vectorized_append(&all_nulls_input_array, &[0, 1, 2, 3, 4])
+ .unwrap();
+
+ let mut equal_to_results = make_true_buffer(all_nulls_input_array.len());
+ builder.vectorized_equal_to(
+ &[0, 1, 2, 3, 4],
+ &all_nulls_input_array,
+ &[0, 1, 2, 3, 4],
+ &mut equal_to_results,
+ );
+ let results = to_vec(&equal_to_results);
+
+ assert!(results[0]);
+ assert!(results[1]);
+ assert!(results[2]);
+ assert!(results[3]);
+ assert!(results[4]);
+
+ // All not nulls input array
+ let all_not_nulls_input_array = Arc::new(BooleanArray::from(vec![
+ Some(false),
+ Some(true),
+ Some(false),
+ Some(true),
+ Some(true),
+ ])) as _;
+ builder
+ .vectorized_append(&all_not_nulls_input_array, &[0, 1, 2, 3, 4])
+ .unwrap();
+
+ let mut equal_to_results = make_true_buffer(all_not_nulls_input_array.len());
+ builder.vectorized_equal_to(
+ &[5, 6, 7, 8, 9],
+ &all_not_nulls_input_array,
+ &[0, 1, 2, 3, 4],
+ &mut equal_to_results,
+ );
+ let results = to_vec(&equal_to_results);
+
+ assert!(results[0]);
+ assert!(results[1]);
+ assert!(results[2]);
+ assert!(results[3]);
+ assert!(results[4]);
+ }
+}
diff --git a/native/vendor/datafusion-physical-plan/src/aggregates/group_values/multi_group_by/bytes.rs b/native/vendor/datafusion-physical-plan/src/aggregates/group_values/multi_group_by/bytes.rs
new file mode 100644
index 0000000000..c83b1da404
--- /dev/null
+++ b/native/vendor/datafusion-physical-plan/src/aggregates/group_values/multi_group_by/bytes.rs
@@ -0,0 +1,701 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::aggregates::group_values::multi_group_by::{
+ GroupColumn, Nulls, nulls_equal_to,
+};
+use crate::aggregates::group_values::null_builder::MaybeNullBufferBuilder;
+use arrow::array::{
+ Array, ArrayRef, AsArray, BooleanBufferBuilder, BufferBuilder, GenericBinaryArray,
+ GenericByteArray, GenericStringArray, OffsetSizeTrait, types::GenericStringType,
+};
+use arrow::buffer::{OffsetBuffer, ScalarBuffer};
+use arrow::datatypes::{ByteArrayType, DataType, GenericBinaryType};
+use datafusion_common::utils::proxy::VecAllocExt;
+use datafusion_common::utils::split_vec_min_alloc;
+use datafusion_common::{Result, exec_datafusion_err};
+use datafusion_physical_expr_common::binary_map::{INITIAL_BUFFER_CAPACITY, OutputType};
+use std::mem::size_of;
+use std::sync::Arc;
+use std::vec;
+
+/// An implementation of [`GroupColumn`] for binary and utf8 types.
+///
+/// Stores a collection of binary or utf8 group values in a single buffer
+/// in a way that allows:
+///
+/// 1. Efficient comparison of incoming rows to existing rows
+/// 2. Efficient construction of the final output array
+pub struct ByteGroupValueBuilder<O>
+where
+ O: OffsetSizeTrait,
+{
+ output_type: OutputType,
+ buffer: BufferBuilder<u8>,
+ /// Offsets into `buffer` for each distinct value. These offsets as used
+ /// directly to create the final `GenericBinaryArray`. The `i`th string is
+ /// stored in the range `offsets[i]..offsets[i+1]` in `buffer`. Null values
+ /// are stored as a zero length string.
+ offsets: Vec<O>,
+ /// Nulls
+ nulls: MaybeNullBufferBuilder,
+ /// The maximum size of the buffer for `0`
+ max_buffer_size: usize,
+}
+
+impl<O> ByteGroupValueBuilder<O>
+where
+ O: OffsetSizeTrait,
+{
+ pub fn new(output_type: OutputType) -> Self {
+ Self {
+ output_type,
+ buffer: BufferBuilder::new(INITIAL_BUFFER_CAPACITY),
+ offsets: vec![O::default()],
+ nulls: MaybeNullBufferBuilder::new(),
+ max_buffer_size: if O::IS_LARGE {
+ i64::MAX as usize
+ } else {
+ i32::MAX as usize
+ },
+ }
+ }
+
+ fn equal_to_inner<B>(&self, lhs_row: usize, array: &ArrayRef, rhs_row: usize) -> bool
+ where
+ B: ByteArrayType,
+ {
+ let array = array.as_bytes::<B>();
+ self.do_equal_to_inner(lhs_row, array, rhs_row)
+ }
+
+ fn append_val_inner<B>(&mut self, array: &ArrayRef, row: usize) -> Result<()>
+ where
+ B: ByteArrayType,
+ {
+ let arr = array.as_bytes::<B>();
+ if arr.is_null(row) {
+ self.nulls.append(true);
+ // nulls need a zero length in the offset buffer
+ let offset = self.buffer.len();
+ self.offsets.push(O::usize_as(offset));
+ } else {
+ self.nulls.append(false);
+ self.do_append_val_inner(arr, row)?;
+ }
+
+ Ok(())
+ }
+
+ fn vectorized_equal_to_inner<B>(
+ &self,
+ lhs_rows: &[usize],
+ array: &ArrayRef,
+ rhs_rows: &[usize],
+ equal_to_results: &mut BooleanBufferBuilder,
+ ) where
+ B: ByteArrayType,
+ {
+ let array = array.as_bytes::<B>();
+
+ for (idx, (&lhs_row, &rhs_row)) in
+ lhs_rows.iter().zip(rhs_rows.iter()).enumerate()
+ {
+ if !equal_to_results.get_bit(idx) {
+ continue;
+ }
+
+ if !self.do_equal_to_inner(lhs_row, array, rhs_row) {
+ equal_to_results.set_bit(idx, false);
+ }
+ }
+ }
+
+ fn vectorized_append_inner<B>(
+ &mut self,
+ array: &ArrayRef,
+ rows: &[usize],
+ ) -> Result<()>
+ where
+ B: ByteArrayType,
+ {
+ let arr = array.as_bytes::<B>();
+ let null_count = array.null_count();
+ let num_rows = array.len();
+ let all_null_or_non_null = if null_count == 0 {
+ Nulls::None
+ } else if null_count == num_rows {
+ Nulls::All
+ } else {
+ Nulls::Some
+ };
+
+ match all_null_or_non_null {
+ Nulls::Some => {
+ for &row in rows {
+ self.append_val_inner::<B>(array, row)?
+ }
+ }
+
+ Nulls::None => {
+ self.nulls.append_n(rows.len(), false);
+ for &row in rows {
+ self.do_append_val_inner(arr, row)?;
+ }
+ }
+
+ Nulls::All => {
+ self.nulls.append_n(rows.len(), true);
+
+ let new_len = self.offsets.len() + rows.len();
+ let offset = self.buffer.len();
+ self.offsets.resize(new_len, O::usize_as(offset));
+ }
+ }
+
+ Ok(())
+ }
+
+ fn do_equal_to_inner<B>(
+ &self,
+ lhs_row: usize,
+ array: &GenericByteArray<B>,
+ rhs_row: usize,
+ ) -> bool
+ where
+ B: ByteArrayType,
+ {
+ let exist_null = self.nulls.is_null(lhs_row);
+ let input_null = array.is_null(rhs_row);
+ if let Some(result) = nulls_equal_to(exist_null, input_null) {
+ return result;
+ }
+ // Otherwise, we need to check their values
+ self.value(lhs_row) == (array.value(rhs_row).as_ref() as &[u8])
+ }
+
+ fn do_append_val_inner<B>(
+ &mut self,
+ array: &GenericByteArray<B>,
+ row: usize,
+ ) -> Result<()>
+ where
+ B: ByteArrayType,
+ {
+ let value: &[u8] = array.value(row).as_ref();
+ self.buffer.append_slice(value);
+
+ if self.buffer.len() > self.max_buffer_size {
+ return Err(exec_datafusion_err!(
+ "offset overflow, buffer size > {}",
+ self.max_buffer_size
+ ));
+ }
+
+ self.offsets.push(O::usize_as(self.buffer.len()));
+ Ok(())
+ }
+
+ /// return the current value of the specified row irrespective of null
+ pub fn value(&self, row: usize) -> &[u8] {
+ let l = self.offsets[row].as_usize();
+ let r = self.offsets[row + 1].as_usize();
+ // Safety: the offsets are constructed correctly and never decrease
+ unsafe { self.buffer.as_slice().get_unchecked(l..r) }
+ }
+}
+
+impl<O> GroupColumn for ByteGroupValueBuilder<O>
+where
+ O: OffsetSizeTrait,
+{
+ fn equal_to(&self, lhs_row: usize, column: &ArrayRef, rhs_row: usize) -> bool {
+ // Sanity array type
+ match self.output_type {
+ OutputType::Binary => {
+ debug_assert!(matches!(
+ column.data_type(),
+ DataType::Binary | DataType::LargeBinary
+ ));
+ self.equal_to_inner::<GenericBinaryType<O>>(lhs_row, column, rhs_row)
+ }
+ OutputType::Utf8 => {
+ debug_assert!(matches!(
+ column.data_type(),
+ DataType::Utf8 | DataType::LargeUtf8
+ ));
+ self.equal_to_inner::<GenericStringType<O>>(lhs_row, column, rhs_row)
+ }
+ _ => unreachable!("View types should use `ArrowBytesViewMap`"),
+ }
+ }
+
+ fn append_val(&mut self, column: &ArrayRef, row: usize) -> Result<()> {
+ // Sanity array type
+ match self.output_type {
+ OutputType::Binary => {
+ debug_assert!(matches!(
+ column.data_type(),
+ DataType::Binary | DataType::LargeBinary
+ ));
+ self.append_val_inner::<GenericBinaryType<O>>(column, row)?
+ }
+ OutputType::Utf8 => {
+ debug_assert!(matches!(
+ column.data_type(),
+ DataType::Utf8 | DataType::LargeUtf8
+ ));
+ self.append_val_inner::<GenericStringType<O>>(column, row)?
+ }
+ _ => unreachable!("View types should use `ArrowBytesViewMap`"),
+ };
+
+ Ok(())
+ }
+
+ fn vectorized_equal_to(
+ &self,
+ lhs_rows: &[usize],
+ array: &ArrayRef,
+ rhs_rows: &[usize],
+ equal_to_results: &mut BooleanBufferBuilder,
+ ) {
+ // Sanity array type
+ match self.output_type {
+ OutputType::Binary => {
+ debug_assert!(matches!(
+ array.data_type(),
+ DataType::Binary | DataType::LargeBinary
+ ));
+ self.vectorized_equal_to_inner::<GenericBinaryType<O>>(
+ lhs_rows,
+ array,
+ rhs_rows,
+ equal_to_results,
+ );
+ }
+ OutputType::Utf8 => {
+ debug_assert!(matches!(
+ array.data_type(),
+ DataType::Utf8 | DataType::LargeUtf8
+ ));
+ self.vectorized_equal_to_inner::<GenericStringType<O>>(
+ lhs_rows,
+ array,
+ rhs_rows,
+ equal_to_results,
+ );
+ }
+ _ => unreachable!("View types should use `ArrowBytesViewMap`"),
+ }
+ }
+
+ fn vectorized_append(&mut self, column: &ArrayRef, rows: &[usize]) -> Result<()> {
+ match self.output_type {
+ OutputType::Binary => {
+ debug_assert!(matches!(
+ column.data_type(),
+ DataType::Binary | DataType::LargeBinary
+ ));
+ self.vectorized_append_inner::<GenericBinaryType<O>>(column, rows)?
+ }
+ OutputType::Utf8 => {
+ debug_assert!(matches!(
+ column.data_type(),
+ DataType::Utf8 | DataType::LargeUtf8
+ ));
+ self.vectorized_append_inner::<GenericStringType<O>>(column, rows)?
+ }
+ _ => unreachable!("View types should use `ArrowBytesViewMap`"),
+ };
+
+ Ok(())
+ }
+
+ fn len(&self) -> usize {
+ self.offsets.len() - 1
+ }
+
+ fn size(&self) -> usize {
+ self.buffer.capacity() * size_of::<u8>()
+ + self.offsets.allocated_size()
+ + self.nulls.allocated_size()
+ }
+
+ fn build(self: Box<Self>) -> ArrayRef {
+ let Self {
+ output_type,
+ mut buffer,
+ offsets,
+ nulls,
+ ..
+ } = *self;
+
+ let null_buffer = nulls.build();
+
+ // SAFETY: the offsets were constructed correctly in `insert_if_new` --
+ // monotonically increasing, overflows were checked.
+ let offsets = unsafe { OffsetBuffer::new_unchecked(ScalarBuffer::from(offsets)) };
+ let values = buffer.finish();
+ match output_type {
+ OutputType::Binary => {
+ // SAFETY: the offsets were constructed correctly
+ Arc::new(unsafe {
+ GenericBinaryArray::new_unchecked(offsets, values, null_buffer)
+ })
+ }
+ OutputType::Utf8 => {
+ // SAFETY:
+ // 1. the offsets were constructed safely
+ //
+ // 2. the input arrays were all the correct type and thus since
+ // all the values that went in were valid (e.g. utf8) so are all
+ // the values that come out
+ Arc::new(unsafe {
+ GenericStringArray::new_unchecked(offsets, values, null_buffer)
+ })
+ }
+ _ => unreachable!("View types should use `ArrowBytesViewMap`"),
+ }
+ }
+
+ fn take_n(&mut self, n: usize) -> ArrayRef {
+ debug_assert!(self.len() >= n);
+ let null_buffer = self.nulls.take_n(n);
+ let first_remaining_offset = O::as_usize(self.offsets[n]);
+
+ // Given offsets like [0, 2, 4, 5] and n = 1, we expect to get
+ // offsets [0, 2, 3]. We first create two offsets for first_n as [0, 2] and the remaining as [2, 4, 5].
+ // And we shift the offset starting from 0 for the remaining one, [2, 4, 5] -> [0, 2, 3].
+ let offset_n = self.offsets[n];
+ let mut first_n_offsets = split_vec_min_alloc(&mut self.offsets, n);
+ // After the split, self.offsets[0] == offset_n in both branches; normalize in-place.
+ self.offsets.iter_mut().for_each(|o| *o = o.sub(offset_n));
+ first_n_offsets.push(offset_n);
+
+ // SAFETY: the offsets were constructed correctly in `insert_if_new` --
+ // monotonically increasing, overflows were checked.
+ let offsets =
+ unsafe { OffsetBuffer::new_unchecked(ScalarBuffer::from(first_n_offsets)) };
+
+ let mut remaining_buffer =
+ BufferBuilder::new(self.buffer.len() - first_remaining_offset);
+ // TODO: Current approach copy the remaining and truncate the original one
+ // Find out a way to avoid copying buffer but split the original one into two.
+ remaining_buffer.append_slice(&self.buffer.as_slice()[first_remaining_offset..]);
+ self.buffer.truncate(first_remaining_offset);
+ let values = self.buffer.finish();
+ self.buffer = remaining_buffer;
+
+ match self.output_type {
+ OutputType::Binary => {
+ // SAFETY: the offsets were constructed correctly
+ Arc::new(unsafe {
+ GenericBinaryArray::new_unchecked(offsets, values, null_buffer)
+ })
+ }
+ OutputType::Utf8 => {
+ // SAFETY:
+ // 1. the offsets were constructed safely
+ //
+ // 2. we asserted the input arrays were all the correct type and
+ // thus since all the values that went in were valid (e.g. utf8)
+ // so are all the values that come out
+ Arc::new(unsafe {
+ GenericStringArray::new_unchecked(offsets, values, null_buffer)
+ })
+ }
+ _ => unreachable!("View types should use `ArrowBytesViewMap`"),
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use std::sync::Arc;
+
+ use crate::aggregates::group_values::multi_group_by::bytes::ByteGroupValueBuilder;
+ use arrow::array::{ArrayRef, BooleanBufferBuilder, NullBufferBuilder, StringArray};
+ use datafusion_common::DataFusionError;
+ use datafusion_physical_expr::binary_map::OutputType;
+
+ use super::GroupColumn;
... 136335 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org