You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by mn...@apache.org on 2023/11/03 09:19:29 UTC
(arrow-datafusion) branch main updated: Minor: remove uncessary #cfg test (#8036)
This is an automated email from the ASF dual-hosted git repository.
mneumann pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 8c42d94851 Minor: remove uncessary #cfg test (#8036)
8c42d94851 is described below
commit 8c42d94851daa6f43053cd903222adfd1d81cb82
Author: Andrew Lamb <an...@nerdnetworks.org>
AuthorDate: Fri Nov 3 05:19:24 2023 -0400
Minor: remove uncessary #cfg test (#8036)
* Minor: remove uncessary #cfg test
* fmt
* Update datafusion/core/src/datasource/file_format/arrow.rs
Co-authored-by: Ruihang Xia <wa...@gmail.com>
---------
Co-authored-by: Daniël Heres <da...@gmail.com>
Co-authored-by: Ruihang Xia <wa...@gmail.com>
---
datafusion/core/tests/fuzz_cases/aggregate_fuzz.rs | 60 +++----
datafusion/core/tests/fuzz_cases/window_fuzz.rs | 192 ++++++++++-----------
2 files changed, 120 insertions(+), 132 deletions(-)
diff --git a/datafusion/core/tests/fuzz_cases/aggregate_fuzz.rs b/datafusion/core/tests/fuzz_cases/aggregate_fuzz.rs
index 798f7ae8ba..821f236af8 100644
--- a/datafusion/core/tests/fuzz_cases/aggregate_fuzz.rs
+++ b/datafusion/core/tests/fuzz_cases/aggregate_fuzz.rs
@@ -35,39 +35,33 @@ use datafusion_physical_expr::expressions::{col, Sum};
use datafusion_physical_expr::{AggregateExpr, PhysicalSortExpr};
use test_utils::add_empty_batches;
-#[cfg(test)]
-#[allow(clippy::items_after_test_module)]
-mod tests {
- use super::*;
-
- #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
- async fn aggregate_test() {
- let test_cases = vec![
- vec!["a"],
- vec!["b", "a"],
- vec!["c", "a"],
- vec!["c", "b", "a"],
- vec!["d", "a"],
- vec!["d", "b", "a"],
- vec!["d", "c", "a"],
- vec!["d", "c", "b", "a"],
- ];
- let n = 300;
- let distincts = vec![10, 20];
- for distinct in distincts {
- let mut handles = Vec::new();
- for i in 0..n {
- let test_idx = i % test_cases.len();
- let group_by_columns = test_cases[test_idx].clone();
- let job = tokio::spawn(run_aggregate_test(
- make_staggered_batches::<true>(1000, distinct, i as u64),
- group_by_columns,
- ));
- handles.push(job);
- }
- for job in handles {
- job.await.unwrap();
- }
+#[tokio::test(flavor = "multi_thread", worker_threads = 8)]
+async fn aggregate_test() {
+ let test_cases = vec![
+ vec!["a"],
+ vec!["b", "a"],
+ vec!["c", "a"],
+ vec!["c", "b", "a"],
+ vec!["d", "a"],
+ vec!["d", "b", "a"],
+ vec!["d", "c", "a"],
+ vec!["d", "c", "b", "a"],
+ ];
+ let n = 300;
+ let distincts = vec![10, 20];
+ for distinct in distincts {
+ let mut handles = Vec::new();
+ for i in 0..n {
+ let test_idx = i % test_cases.len();
+ let group_by_columns = test_cases[test_idx].clone();
+ let job = tokio::spawn(run_aggregate_test(
+ make_staggered_batches::<true>(1000, distinct, i as u64),
+ group_by_columns,
+ ));
+ handles.push(job);
+ }
+ for job in handles {
+ job.await.unwrap();
}
}
}
diff --git a/datafusion/core/tests/fuzz_cases/window_fuzz.rs b/datafusion/core/tests/fuzz_cases/window_fuzz.rs
index 66f7374a51..9f70321318 100644
--- a/datafusion/core/tests/fuzz_cases/window_fuzz.rs
+++ b/datafusion/core/tests/fuzz_cases/window_fuzz.rs
@@ -44,108 +44,102 @@ use hashbrown::HashMap;
use rand::rngs::StdRng;
use rand::{Rng, SeedableRng};
-#[cfg(test)]
-#[allow(clippy::items_after_test_module)]
-mod tests {
- use super::*;
-
- use datafusion_physical_plan::windows::PartitionSearchMode::{
- Linear, PartiallySorted, Sorted,
- };
+use datafusion_physical_plan::windows::PartitionSearchMode::{
+ Linear, PartiallySorted, Sorted,
+};
- #[tokio::test(flavor = "multi_thread", worker_threads = 16)]
- async fn window_bounded_window_random_comparison() -> Result<()> {
- // make_staggered_batches gives result sorted according to a, b, c
- // In the test cases first entry represents partition by columns
- // Second entry represents order by columns.
- // Third entry represents search mode.
- // In sorted mode physical plans are in the form for WindowAggExec
- //```
- // WindowAggExec
- // MemoryExec]
- // ```
- // and in the form for BoundedWindowAggExec
- // ```
- // BoundedWindowAggExec
- // MemoryExec
- // ```
- // In Linear and PartiallySorted mode physical plans are in the form for WindowAggExec
- //```
- // WindowAggExec
- // SortExec(required by window function)
- // MemoryExec]
- // ```
- // and in the form for BoundedWindowAggExec
- // ```
- // BoundedWindowAggExec
- // MemoryExec
- // ```
- let test_cases = vec![
- (vec!["a"], vec!["a"], Sorted),
- (vec!["a"], vec!["b"], Sorted),
- (vec!["a"], vec!["a", "b"], Sorted),
- (vec!["a"], vec!["b", "c"], Sorted),
- (vec!["a"], vec!["a", "b", "c"], Sorted),
- (vec!["b"], vec!["a"], Linear),
- (vec!["b"], vec!["a", "b"], Linear),
- (vec!["b"], vec!["a", "c"], Linear),
- (vec!["b"], vec!["a", "b", "c"], Linear),
- (vec!["c"], vec!["a"], Linear),
- (vec!["c"], vec!["a", "b"], Linear),
- (vec!["c"], vec!["a", "c"], Linear),
- (vec!["c"], vec!["a", "b", "c"], Linear),
- (vec!["b", "a"], vec!["a"], Sorted),
- (vec!["b", "a"], vec!["b"], Sorted),
- (vec!["b", "a"], vec!["c"], Sorted),
- (vec!["b", "a"], vec!["a", "b"], Sorted),
- (vec!["b", "a"], vec!["b", "c"], Sorted),
- (vec!["b", "a"], vec!["a", "c"], Sorted),
- (vec!["b", "a"], vec!["a", "b", "c"], Sorted),
- (vec!["c", "b"], vec!["a"], Linear),
- (vec!["c", "b"], vec!["a", "b"], Linear),
- (vec!["c", "b"], vec!["a", "c"], Linear),
- (vec!["c", "b"], vec!["a", "b", "c"], Linear),
- (vec!["c", "a"], vec!["a"], PartiallySorted(vec![1])),
- (vec!["c", "a"], vec!["b"], PartiallySorted(vec![1])),
- (vec!["c", "a"], vec!["c"], PartiallySorted(vec![1])),
- (vec!["c", "a"], vec!["a", "b"], PartiallySorted(vec![1])),
- (vec!["c", "a"], vec!["b", "c"], PartiallySorted(vec![1])),
- (vec!["c", "a"], vec!["a", "c"], PartiallySorted(vec![1])),
- (
- vec!["c", "a"],
- vec!["a", "b", "c"],
- PartiallySorted(vec![1]),
- ),
- (vec!["c", "b", "a"], vec!["a"], Sorted),
- (vec!["c", "b", "a"], vec!["b"], Sorted),
- (vec!["c", "b", "a"], vec!["c"], Sorted),
- (vec!["c", "b", "a"], vec!["a", "b"], Sorted),
- (vec!["c", "b", "a"], vec!["b", "c"], Sorted),
- (vec!["c", "b", "a"], vec!["a", "c"], Sorted),
- (vec!["c", "b", "a"], vec!["a", "b", "c"], Sorted),
- ];
- let n = 300;
- let n_distincts = vec![10, 20];
- for n_distinct in n_distincts {
- let mut handles = Vec::new();
- for i in 0..n {
- let idx = i % test_cases.len();
- let (pb_cols, ob_cols, search_mode) = test_cases[idx].clone();
- let job = tokio::spawn(run_window_test(
- make_staggered_batches::<true>(1000, n_distinct, i as u64),
- i as u64,
- pb_cols,
- ob_cols,
- search_mode,
- ));
- handles.push(job);
- }
- for job in handles {
- job.await.unwrap()?;
- }
+#[tokio::test(flavor = "multi_thread", worker_threads = 16)]
+async fn window_bounded_window_random_comparison() -> Result<()> {
+ // make_staggered_batches gives result sorted according to a, b, c
+ // In the test cases first entry represents partition by columns
+ // Second entry represents order by columns.
+ // Third entry represents search mode.
+ // In sorted mode physical plans are in the form for WindowAggExec
+ //```
+ // WindowAggExec
+ // MemoryExec]
+ // ```
+ // and in the form for BoundedWindowAggExec
+ // ```
+ // BoundedWindowAggExec
+ // MemoryExec
+ // ```
+ // In Linear and PartiallySorted mode physical plans are in the form for WindowAggExec
+ //```
+ // WindowAggExec
+ // SortExec(required by window function)
+ // MemoryExec]
+ // ```
+ // and in the form for BoundedWindowAggExec
+ // ```
+ // BoundedWindowAggExec
+ // MemoryExec
+ // ```
+ let test_cases = vec![
+ (vec!["a"], vec!["a"], Sorted),
+ (vec!["a"], vec!["b"], Sorted),
+ (vec!["a"], vec!["a", "b"], Sorted),
+ (vec!["a"], vec!["b", "c"], Sorted),
+ (vec!["a"], vec!["a", "b", "c"], Sorted),
+ (vec!["b"], vec!["a"], Linear),
+ (vec!["b"], vec!["a", "b"], Linear),
+ (vec!["b"], vec!["a", "c"], Linear),
+ (vec!["b"], vec!["a", "b", "c"], Linear),
+ (vec!["c"], vec!["a"], Linear),
+ (vec!["c"], vec!["a", "b"], Linear),
+ (vec!["c"], vec!["a", "c"], Linear),
+ (vec!["c"], vec!["a", "b", "c"], Linear),
+ (vec!["b", "a"], vec!["a"], Sorted),
+ (vec!["b", "a"], vec!["b"], Sorted),
+ (vec!["b", "a"], vec!["c"], Sorted),
+ (vec!["b", "a"], vec!["a", "b"], Sorted),
+ (vec!["b", "a"], vec!["b", "c"], Sorted),
+ (vec!["b", "a"], vec!["a", "c"], Sorted),
+ (vec!["b", "a"], vec!["a", "b", "c"], Sorted),
+ (vec!["c", "b"], vec!["a"], Linear),
+ (vec!["c", "b"], vec!["a", "b"], Linear),
+ (vec!["c", "b"], vec!["a", "c"], Linear),
+ (vec!["c", "b"], vec!["a", "b", "c"], Linear),
+ (vec!["c", "a"], vec!["a"], PartiallySorted(vec![1])),
+ (vec!["c", "a"], vec!["b"], PartiallySorted(vec![1])),
+ (vec!["c", "a"], vec!["c"], PartiallySorted(vec![1])),
+ (vec!["c", "a"], vec!["a", "b"], PartiallySorted(vec![1])),
+ (vec!["c", "a"], vec!["b", "c"], PartiallySorted(vec![1])),
+ (vec!["c", "a"], vec!["a", "c"], PartiallySorted(vec![1])),
+ (
+ vec!["c", "a"],
+ vec!["a", "b", "c"],
+ PartiallySorted(vec![1]),
+ ),
+ (vec!["c", "b", "a"], vec!["a"], Sorted),
+ (vec!["c", "b", "a"], vec!["b"], Sorted),
+ (vec!["c", "b", "a"], vec!["c"], Sorted),
+ (vec!["c", "b", "a"], vec!["a", "b"], Sorted),
+ (vec!["c", "b", "a"], vec!["b", "c"], Sorted),
+ (vec!["c", "b", "a"], vec!["a", "c"], Sorted),
+ (vec!["c", "b", "a"], vec!["a", "b", "c"], Sorted),
+ ];
+ let n = 300;
+ let n_distincts = vec![10, 20];
+ for n_distinct in n_distincts {
+ let mut handles = Vec::new();
+ for i in 0..n {
+ let idx = i % test_cases.len();
+ let (pb_cols, ob_cols, search_mode) = test_cases[idx].clone();
+ let job = tokio::spawn(run_window_test(
+ make_staggered_batches::<true>(1000, n_distinct, i as u64),
+ i as u64,
+ pb_cols,
+ ob_cols,
+ search_mode,
+ ));
+ handles.push(job);
+ }
+ for job in handles {
+ job.await.unwrap()?;
}
- Ok(())
}
+ Ok(())
}
fn get_random_function(