You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/04/30 18:53:27 UTC
(datafusion) 01/01: test(sqllogictest): stabilize parquet output_rows_skew with WITH ORDER (#21898)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-21898-e18b1cfee8eecf3e82985489758d517240e1ab30
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit c2824b5e26b274ec02b29bf78e2c733cdc0a32c4
Author: Ratul Dawar <ra...@gmail.com>
AuthorDate: Fri May 1 00:23:03 2026 +0530
test(sqllogictest): stabilize parquet output_rows_skew with WITH ORDER (#21898)
## Summary
Adds `WITH ORDER (x)` to `CREATE EXTERNAL TABLE skew_parquet` /
`skew_parquet_single` in `explain_analyze.slt` so `FileScanConfig`
preserves scan ordering (`preserve_order`), keeping per-partition
`output_rows` stable under dynamic file scheduling (PR #21351).
## Related
- Follow-up to flaky skew assertions discussed around #21866 / #21850.
## Testing
- `cargo test -p datafusion-sqllogictest --test sqllogictests --
explain_analyze` (recommended before merge)
Sqllogictest-only change.
Made with [Cursor](https://cursor.com)
---------
Co-authored-by: Yongting You <20...@gmail.com>
Co-authored-by: Raz Luvaton <16...@users.noreply.github.com>
---
datafusion/sqllogictest/test_files/explain_analyze.slt | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/datafusion/sqllogictest/test_files/explain_analyze.slt b/datafusion/sqllogictest/test_files/explain_analyze.slt
index 561c1a9dcb..b272a8cd9c 100644
--- a/datafusion/sqllogictest/test_files/explain_analyze.slt
+++ b/datafusion/sqllogictest/test_files/explain_analyze.slt
@@ -24,9 +24,12 @@ EXPLAIN ANALYZE SELECT * FROM generate_series(100);
Plan with Metrics LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=0, end=100, batch_size=8192], metrics=[output_rows=101, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>]
# --------------------------------------------
-# Test parquet-only output_rows_skew metric
+# Test parquet-only output_rows_skew metric (WITH ORDER(x) → preserve_order scan)
# --------------------------------------------
+# Parquet supports dynamic work scheduling by default; per-partition data
+# sources might steal work from siblings, making row counts per partition
+# non-deterministic. Forcing `WITH ORDER` disables this behavior.
statement ok
set datafusion.explain.analyze_level = dev;
@@ -48,6 +51,7 @@ STORED AS PARQUET;
statement ok
CREATE EXTERNAL TABLE skew_parquet
STORED AS PARQUET
+WITH ORDER (x)
LOCATION 'test_files/scratch/explain_analyze/output_rows_skew';
# All partition's output_rows: [4]
@@ -70,6 +74,7 @@ STORED AS PARQUET;
statement ok
CREATE EXTERNAL TABLE skew_parquet
STORED AS PARQUET
+WITH ORDER (x)
LOCATION 'test_files/scratch/explain_analyze/output_rows_skew';
query TT
@@ -98,6 +103,7 @@ STORED AS PARQUET;
statement ok
CREATE EXTERNAL TABLE skew_parquet
STORED AS PARQUET
+WITH ORDER (x)
LOCATION 'test_files/scratch/explain_analyze/output_rows_skew';
query TT
@@ -109,6 +115,7 @@ Plan with Metrics DataSourceExec: <slt:ignore>output_rows_skew=84.31%<slt:ignore
statement ok
CREATE EXTERNAL TABLE skew_parquet_single
STORED AS PARQUET
+WITH ORDER (x)
LOCATION 'test_files/scratch/explain_analyze/output_rows_skew/f4.parquet';
query TT
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org