You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/10/05 06:11:27 UTC

(datafusion) 01/01: Add skewness metric to nodes that execute in partitioned mode (#25977)

This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-25977-62f29430d262f951730dfd5bff1723f8d44f0a0e
in repository https://gitbox.apache.org/repos/asf/datafusion.git

commit c2baac54aff394a88b8ac98471967b3a7295e65c
Author: Lía Adriana <li...@datadoghq.com>
AuthorDate: Mon Oct 5 06:10:57 2026 +0000

    Add skewness metric to nodes that execute in partitioned mode (#25977)
    
    ## Which issue does this PR close?
    
    <!--
    We generally require a GitHub issue to be filed for all bug fixes and
    enhancements and this helps us generate change logs for our releases.
    You can link an issue to this PR using the GitHub syntax. For example
    `Closes #123` indicates that this PR will close issue #123.
    -->
    
    Related to https://github.com/apache/datafusion/issues/23237 and
    [feat(metric): Add output skewness metric to detect skewed plans
    easier](https://github.com/apache/datafusion/pull/21211)
    
    
    - Closes #.
    
    ## Rationale for this change
    
    I think it would be useful to add the skewness metric for nodes that
    execute in partitioned mdoe -- right now the only node that includes
    that metric in the explain analyze is DataSourceExec for parquet.
    
    This is specially useful to check how many partitions were idle during
    Aggregations or HashJoins for example.
    
    ## What changes are included in this PR?
    
    The PR adds a new metric `output_rows_skew` in the Explain Analyze
    output for the following nodes:
    
    - RepartitionExec
    - HashJoinExec
    - AggregateExec
    - BoundedWindowAggExec
    - WindowAggExec
    
    Note that for a plan that has a Parquet DataSource the skewness might
    not be the same as for the rest of the nodes, for example we might have
    the following scenario:
    
    ```
    DataSourceExec       [100, 100, 100, 100]  → 0% skewness, data is even
      FilterExec         [100,   0,   0,   0]  → 100%   (filter only matches in partition 0)
        AggregateExec    [  3,   0,   0,   0]  → 100%   (Partial: groups, not rows)
          RepartitionExec Hash  [1, 1, 1, 0]   → ~11%   (3 groups re-spread by key)
    
    ```
    
    ## What is the testing strategy for this PR?
    
    I added a Explain Analyze test
    
    ## Are there any user-facing changes?
    
    Yes, a new metric will show in the explain analyze plan for the nodes
    mentioned above.
---
 datafusion/core/tests/sql/explain_analyze.rs       | 78 ++++++++++++++++++++++
 datafusion/physical-expr-common/src/metrics/mod.rs | 16 +++++
 datafusion/physical-plan/src/aggregates/mod.rs     |  2 +-
 .../physical-plan/src/joins/hash_join/exec.rs      |  2 +-
 datafusion/physical-plan/src/repartition/mod.rs    |  2 +-
 .../src/windows/bounded_window_agg_exec.rs         |  2 +-
 .../physical-plan/src/windows/window_agg_exec.rs   |  2 +-
 .../test_files/dynamic_row_group_pruning.slt       |  2 +-
 datafusion/sqllogictest/test_files/joins.slt       |  2 +-
 9 files changed, 101 insertions(+), 7 deletions(-)

diff --git a/datafusion/core/tests/sql/explain_analyze.rs b/datafusion/core/tests/sql/explain_analyze.rs
index 67614cfbde..3efe084bae 100644
--- a/datafusion/core/tests/sql/explain_analyze.rs
+++ b/datafusion/core/tests/sql/explain_analyze.rs
@@ -19,6 +19,7 @@ use super::*;
 use insta::assert_snapshot;
 use rstest::rstest;
 
+use datafusion::catalog::MemTable;
 use datafusion::config::ConfigOptions;
 use datafusion::physical_plan::display::DisplayableExecutionPlan;
 use datafusion::physical_plan::metrics::Timestamp;
@@ -908,6 +909,83 @@ async fn parquet_explain_analyze() {
     );
 }
 
+/// Operators that run once per partition report `output_rows_skew`. A single
+/// join/group/window key hashes every row to one partition, so each
+/// hash-partitioned operator must report the maximum skew.
+#[tokio::test]
+#[cfg_attr(coverage, ignore)]
+async fn explain_analyze_output_rows_skew_on_hash_partitioned_operators() {
+    let mut config = SessionConfig::new().with_target_partitions(4);
+    // Force a partitioned (hash repartitioned) join instead of `CollectLeft`
+    let options = config.options_mut();
+    options.optimizer.hash_join_single_partition_threshold = 0;
+    options.optimizer.hash_join_single_partition_threshold_rows = 0;
+    let ctx = SessionContext::new_with_config(config);
+    // Spread the rows over 4 input partitions so they get hash repartitioned
+    // (a single input partition already satisfies the hash distribution)
+    let schema = Arc::new(Schema::new(vec![
+        Field::new("k", DataType::Int64, false),
+        Field::new("v", DataType::Int64, false),
+    ]));
+    let partitions = (0..4)
+        .map(|p| {
+            let v = (p * 25..(p + 1) * 25).collect::<Vec<i64>>();
+            let batch = RecordBatch::try_new(
+                Arc::clone(&schema),
+                vec![
+                    Arc::new(Int64Array::from(vec![1; v.len()])),
+                    Arc::new(Int64Array::from(v)),
+                ],
+            )
+            .unwrap();
+            vec![batch]
+        })
+        .collect();
+    let table = MemTable::try_new(schema, partitions).unwrap();
+    ctx.register_table("t", Arc::new(table)).unwrap();
+
+    let sql = "EXPLAIN ANALYZE \
+               SELECT a.k, count(*) FROM t a JOIN t b ON a.k = b.k GROUP BY a.k";
+    let formatted = explain_analyze_output(&ctx, sql).await;
+    for operator in [
+        "RepartitionExec: partitioning=Hash",
+        "HashJoinExec: mode=Partitioned",
+        "AggregateExec: mode=SinglePartitioned",
+    ] {
+        assert_operator_metric(&formatted, operator, "output_rows_skew=100%");
+    }
+
+    let sql = "EXPLAIN ANALYZE \
+               SELECT k, \
+                 row_number() OVER (PARTITION BY k ORDER BY v), \
+                 sum(v) OVER (PARTITION BY k) \
+               FROM t";
+    let formatted = explain_analyze_output(&ctx, sql).await;
+    for operator in ["BoundedWindowAggExec", "WindowAggExec"] {
+        assert_operator_metric(&formatted, operator, "output_rows_skew=100%");
+    }
+}
+
+async fn explain_analyze_output(ctx: &SessionContext, sql: &str) -> String {
+    let actual = execute_to_batches(ctx, sql).await;
+    arrow::util::pretty::pretty_format_batches(&actual)
+        .unwrap()
+        .to_string()
+}
+
+fn assert_operator_metric(formatted: &str, operator: &str, metric: &str) {
+    // Leading space so `WindowAggExec` does not match `BoundedWindowAggExec`
+    let needle = format!(" {operator}");
+    let line = formatted
+        .lines()
+        .find(|line| line.contains(&needle))
+        .unwrap_or_else(|| panic!("{operator} not found in plan:\n{formatted}"));
+    assert!(
+        line.contains(metric),
+        "expected {metric} on {operator}, got:\n{line}"
+    );
+}
+
 /// The Parquet scan's `elapsed_compute` must cover decoding, which dominates a
 /// full scan, and not only the projection of batches that are already decoded.
 /// See <https://github.com/apache/datafusion/issues/18195>.
diff --git a/datafusion/physical-expr-common/src/metrics/mod.rs b/datafusion/physical-expr-common/src/metrics/mod.rs
index 9a1c17872e..262550f594 100644
--- a/datafusion/physical-expr-common/src/metrics/mod.rs
+++ b/datafusion/physical-expr-common/src/metrics/mod.rs
@@ -250,6 +250,22 @@ impl MetricsSet {
         self.metrics.push(metric)
     }
 
+    /// Returns this set with the derived `output_rows_skew` metric appended,
+    /// or unchanged if no partition reported `output_rows` yet.
+    ///
+    /// This is typically used in `ExecutionPlan::metrics` of operators that
+    /// execute in multiple partitions (e.g. repartitions, partitioned joins and
+    /// aggregations), where it shows how evenly their output rows are spread
+    /// across partitions. With a single partition the skew is always `0%`.
+    ///
+    /// See [`BaselineMetrics::output_rows_skew_metric`] for how skew is computed.
+    pub fn with_output_rows_skew(mut self) -> Self {
+        if let Some(output_rows_skew) = BaselineMetrics::output_rows_skew_metric(&self) {
+            self.push(output_rows_skew);
+        }
+        self
+    }
+
     /// Return the metrics whose partition ID equals `partition`.
     ///
     /// Unpartitioned metrics are excluded; an unknown partition yields an empty
diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rs
index 6552bd95e0..d098272688 100644
--- a/datafusion/physical-plan/src/aggregates/mod.rs
+++ b/datafusion/physical-plan/src/aggregates/mod.rs
@@ -2600,7 +2600,7 @@ impl ExecutionPlan for AggregateExec {
     }
 
     fn metrics(&self) -> Option<MetricsSet> {
-        Some(self.metrics.clone_inner())
+        Some(self.metrics.clone_inner().with_output_rows_skew())
     }
 
     fn child_stats_requests(&self, partition: Option<usize>) -> Vec<ChildStats> {
diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs
index ce8ee72493..10a9fa70a8 100644
--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs
+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs
@@ -1863,7 +1863,7 @@ impl ExecutionPlan for HashJoinExec {
     }
 
     fn metrics(&self) -> Option<MetricsSet> {
-        Some(self.metrics.clone_inner())
+        Some(self.metrics.clone_inner().with_output_rows_skew())
     }
 
     fn child_stats_requests(&self, partition: Option<usize>) -> Vec<ChildStats> {
diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rs
index 3df29b577c..51ce1345fc 100644
--- a/datafusion/physical-plan/src/repartition/mod.rs
+++ b/datafusion/physical-plan/src/repartition/mod.rs
@@ -1884,7 +1884,7 @@ impl ExecutionPlan for RepartitionExec {
     }
 
     fn metrics(&self) -> Option<MetricsSet> {
-        Some(self.metrics.clone_inner())
+        Some(self.metrics.clone_inner().with_output_rows_skew())
     }
 
     fn child_stats_requests(&self, _partition: Option<usize>) -> Vec<ChildStats> {
diff --git a/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs b/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs
index 0c6e77a1d6..724a722898 100644
--- a/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs
+++ b/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs
@@ -528,7 +528,7 @@ impl ExecutionPlan for BoundedWindowAggExec {
     }
 
     fn metrics(&self) -> Option<MetricsSet> {
-        Some(self.metrics.clone_inner())
+        Some(self.metrics.clone_inner().with_output_rows_skew())
     }
 
     fn child_stats_requests(&self, partition: Option<usize>) -> Vec<ChildStats> {
diff --git a/datafusion/physical-plan/src/windows/window_agg_exec.rs b/datafusion/physical-plan/src/windows/window_agg_exec.rs
index dba4c063d1..e00aaa21bb 100644
--- a/datafusion/physical-plan/src/windows/window_agg_exec.rs
+++ b/datafusion/physical-plan/src/windows/window_agg_exec.rs
@@ -320,7 +320,7 @@ impl ExecutionPlan for WindowAggExec {
     }
 
     fn metrics(&self) -> Option<MetricsSet> {
-        Some(self.metrics.clone_inner())
+        Some(self.metrics.clone_inner().with_output_rows_skew())
     }
 
     fn child_stats_requests(&self, partition: Option<usize>) -> Vec<ChildStats> {
diff --git a/datafusion/sqllogictest/test_files/dynamic_row_group_pruning.slt b/datafusion/sqllogictest/test_files/dynamic_row_group_pruning.slt
index 753bf6fdfd..10d25d7235 100644
--- a/datafusion/sqllogictest/test_files/dynamic_row_group_pruning.slt
+++ b/datafusion/sqllogictest/test_files/dynamic_row_group_pruning.slt
@@ -338,7 +338,7 @@ query TT
 explain analyze select rgsel.b from dim join rgsel on dim.column1 = rgsel.b;
 ----
 Plan with Metrics
-01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(column1@0, b@0)], projection=[b@1], metrics=[output_rows=3, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, output_batches=1, build_mem_used=<slt:ignore>, array_map_created_count=0, build_input_batches=1, build_input_rows=3, input_batches=2, input_rows=3, build_time=<slt:ignore>, join_time=<slt:ignore>, avg_fanout=100% (3/3), probe_hit_rate=100% (3/3)]
+01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(column1@0, b@0)], projection=[b@1], metrics=[output_rows=3, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, output_batches=1, build_mem_used=<slt:ignore>, array_map_created_count=0, build_input_batches=1, build_input_rows=3, input_batches=2, input_rows=3, build_time=<slt:ignore>, join_time=<slt:ignore>, avg_fanout=100% (3/3), output_rows_skew=0%, probe_hit_rate=100% (3/3)]
 02)--DataSourceExec: partitions=1, partition_sizes=[1], metrics=[]
 03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_row_group_pruning/rgsel.parquet]]}, projection=[b], file_type=parquet, predicate=DynamicFilter [ b@1 >= 1000 AND b@1 <= 3050 AND hash_lookup ], dynamic_rg_pruning=eligible, pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 >= 1000 AND b_null_count@1 != row_count@2 AND b_min@3 <= 3050 AND b_null_count@1 != row_count@2 AND KEY_RANGE_BITMAP(b_min@3, b_max@0, 3/4096 bucket [...]
 
diff --git a/datafusion/sqllogictest/test_files/joins.slt b/datafusion/sqllogictest/test_files/joins.slt
index f4dbd212eb..e922233fd8 100644
--- a/datafusion/sqllogictest/test_files/joins.slt
+++ b/datafusion/sqllogictest/test_files/joins.slt
@@ -5263,7 +5263,7 @@ LEFT ANTI JOIN (
 ) t2 ON t1.k = t2.k;
 ----
 Plan with Metrics
-01)HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(k@0, k@0)], metrics=[output_rows=2, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, output_batches=1, build_mem_used=<slt:ignore>, array_map_created_count=0, build_input_batches=0, build_input_rows=0, input_batches=1, input_rows=2, build_time=<slt:ignore>, join_time=<slt:ignore>, avg_fanout=N/A (0/0), probe_hit_rate=0% (0/2)]
+01)HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(k@0, k@0)], metrics=[output_rows=2, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, output_batches=1, build_mem_used=<slt:ignore>, array_map_created_count=0, build_input_batches=0, build_input_rows=0, input_batches=1, input_rows=2, build_time=<slt:ignore>, join_time=<slt:ignore>, avg_fanout=N/A (0/0), output_rows_skew=0%, probe_hit_rate=0% (0/2)]
 02)--ProjectionExec: expr=[column1@0 as k], metrics=[output_rows=0, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, output_batches=0, expr_0_eval_time=<slt:ignore>]
 03)----FilterExec: column1@0 != 1, metrics=[output_rows=0, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, output_batches=0, selectivity=0% (0/1)]
 04)------DataSourceExec: partitions=1, partition_sizes=[1], metrics=[]


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org