You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/06/02 10:47:52 UTC
(datafusion) branch gh-readonly-queue/main/pr-22700-e34912f2b4a924a282e6570f9ea9958f1b878983 created (now 766096a8aa)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a change to branch gh-readonly-queue/main/pr-22700-e34912f2b4a924a282e6570f9ea9958f1b878983
in repository https://gitbox.apache.org/repos/asf/datafusion.git
at 766096a8aa fix: correct cross join byte size statistics (#22700)
This branch includes the following new commits:
new 766096a8aa fix: correct cross join byte size statistics (#22700)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org
(datafusion) 01/01: fix: correct cross join byte size statistics (#22700)
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-22700-e34912f2b4a924a282e6570f9ea9958f1b878983
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 766096a8aa344c303787f9ea232aedb24b4f2338
Author: Neil Conway <ne...@gmail.com>
AuthorDate: Tue Jun 2 06:47:29 2026 -0400
fix: correct cross join byte size statistics (#22700)
## Which issue does this PR close?
- Closes #22699
## Rationale for this change
`stats_cartesian_product` computes the total byte size of a cross join
as:
```rust
let total_byte_size = left_stats
.total_byte_size
.multiply(&right_stats.total_byte_size)
.multiply(&Precision::Exact(2));
```
This is wrong (e.g., it multiplies two byte-size values together). The
correct formula is "left-num-rows * right-size-in-bytes + right-num-rows
* left-size-in-bytes", since the left side is repeated once per row on
the right, and vice versa.
## What changes are included in this PR?
* Fix total byte size formula for cross join
* Update expected SLT results
## Are these changes tested?
Yes; covered by existing tests.
## Are there any user-facing changes?
No.
---
.../tests/physical_optimizer/partition_statistics.rs | 6 +++---
datafusion/physical-plan/src/joins/cross_join.rs | 19 +++++++++++--------
2 files changed, 14 insertions(+), 11 deletions(-)
diff --git a/datafusion/core/tests/physical_optimizer/partition_statistics.rs b/datafusion/core/tests/physical_optimizer/partition_statistics.rs
index d06e506abf..181b7de7d9 100644
--- a/datafusion/core/tests/physical_optimizer/partition_statistics.rs
+++ b/datafusion/core/tests/physical_optimizer/partition_statistics.rs
@@ -556,10 +556,10 @@ mod test {
// Check that we have 2 partitions
assert_eq!(statistics.len(), 2);
// Cross join output schema: [left.id, left.date, right.id]
- // Cross join doesn't propagate Column's byte_size
let expected_statistic_partition_1 = Statistics {
num_rows: Precision::Exact(8),
- total_byte_size: Precision::Exact(512),
+ total_byte_size: Precision::Exact(96),
+ // Cross join doesn't propagate Column's byte_size
column_statistics: vec![
// column 0: left.id (Int32, file column from t1)
ColumnStatistics {
@@ -593,7 +593,7 @@ mod test {
};
let expected_statistic_partition_2 = Statistics {
num_rows: Precision::Exact(8),
- total_byte_size: Precision::Exact(512),
+ total_byte_size: Precision::Exact(96),
column_statistics: vec![
// column 0: left.id (Int32, file column from t1)
ColumnStatistics {
diff --git a/datafusion/physical-plan/src/joins/cross_join.rs b/datafusion/physical-plan/src/joins/cross_join.rs
index 6661d2782b..45b34692ab 100644
--- a/datafusion/physical-plan/src/joins/cross_join.rs
+++ b/datafusion/physical-plan/src/joins/cross_join.rs
@@ -430,13 +430,14 @@ fn stats_cartesian_product(
let left_row_count = left_stats.num_rows;
let right_row_count = right_stats.num_rows;
- // calculate global stats
+ // Calculate global stats
let num_rows = left_row_count.multiply(&right_row_count);
- // the result size is two times a*b because you have the columns of both left and right
- let total_byte_size = left_stats
- .total_byte_size
- .multiply(&right_stats.total_byte_size)
- .multiply(&Precision::Exact(2));
+
+ // Each output row includes every left and right column, so the left side is
+ // repeated once per right row and the right side once per left row.
+ let left_byte_size = left_stats.total_byte_size.multiply(&right_row_count);
+ let right_byte_size = right_stats.total_byte_size.multiply(&left_row_count);
+ let total_byte_size = left_byte_size.add(&right_byte_size);
let left_col_stats = left_stats.column_statistics;
let right_col_stats = right_stats.column_statistics;
@@ -494,7 +495,7 @@ fn stats_cartesian_product(
}
}
-/// A stream that issues [RecordBatch]es as they arrive from the right of the join.
+/// A stream that issues [RecordBatch]es as they arrive from the right of the join.
struct CrossJoinStream<T> {
/// Input schema
schema: Arc<Schema>,
@@ -755,7 +756,9 @@ mod tests {
let expected = Statistics {
num_rows: Precision::Exact(left_row_count * right_row_count),
- total_byte_size: Precision::Exact(2 * left_bytes * right_bytes),
+ total_byte_size: Precision::Exact(
+ left_bytes * right_row_count + right_bytes * left_row_count,
+ ),
column_statistics: vec![
ColumnStatistics {
distinct_count: Precision::Exact(5),
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org