You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by al...@apache.org on 2025/09/12 12:16:09 UTC

(datafusion) branch main updated: feat: Add binary to `join_fuzz` testing (#17497)

This is an automated email from the ASF dual-hosted git repository.

alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git


The following commit(s) were added to refs/heads/main by this push:
     new aada2e1e65 feat: Add binary to `join_fuzz` testing (#17497)
aada2e1e65 is described below

commit aada2e1e655b9e145d16ed0381d1f438d5e90eef
Author: Jonathan Chen <ch...@gmail.com>
AuthorDate: Fri Sep 12 08:16:05 2025 -0400

    feat: Add binary to `join_fuzz` testing (#17497)
---
 datafusion/core/tests/fuzz_cases/join_fuzz.rs | 366 +++++++++++++++++++++++---
 1 file changed, 323 insertions(+), 43 deletions(-)

diff --git a/datafusion/core/tests/fuzz_cases/join_fuzz.rs b/datafusion/core/tests/fuzz_cases/join_fuzz.rs
index 0d24a7f0e2..5a2f9e9733 100644
--- a/datafusion/core/tests/fuzz_cases/join_fuzz.rs
+++ b/datafusion/core/tests/fuzz_cases/join_fuzz.rs
@@ -20,7 +20,7 @@ use std::time::SystemTime;
 
 use crate::fuzz_cases::join_fuzz::JoinTestType::{HjSmj, NljHj};
 
-use arrow::array::{ArrayRef, Int32Array};
+use arrow::array::{ArrayRef, BinaryArray, Int32Array};
 use arrow::compute::SortOptions;
 use arrow::datatypes::Schema;
 use arrow::record_batch::RecordBatch;
@@ -92,8 +92,8 @@ fn col_lt_col_filter(schema1: Arc<Schema>, schema2: Arc<Schema>) -> JoinFilter {
 #[tokio::test]
 async fn test_inner_join_1k_filtered() {
     JoinFuzzTestCase::new(
-        make_staggered_batches(1000),
-        make_staggered_batches(1000),
+        make_staggered_batches_i32(1000),
+        make_staggered_batches_i32(1000),
         JoinType::Inner,
         Some(Box::new(col_lt_col_filter)),
     )
@@ -104,8 +104,8 @@ async fn test_inner_join_1k_filtered() {
 #[tokio::test]
 async fn test_inner_join_1k() {
     JoinFuzzTestCase::new(
-        make_staggered_batches(1000),
-        make_staggered_batches(1000),
+        make_staggered_batches_i32(1000),
+        make_staggered_batches_i32(1000),
         JoinType::Inner,
         None,
     )
@@ -116,8 +116,8 @@ async fn test_inner_join_1k() {
 #[tokio::test]
 async fn test_left_join_1k() {
     JoinFuzzTestCase::new(
-        make_staggered_batches(1000),
-        make_staggered_batches(1000),
+        make_staggered_batches_i32(1000),
+        make_staggered_batches_i32(1000),
         JoinType::Left,
         None,
     )
@@ -128,8 +128,8 @@ async fn test_left_join_1k() {
 #[tokio::test]
 async fn test_left_join_1k_filtered() {
     JoinFuzzTestCase::new(
-        make_staggered_batches(1000),
-        make_staggered_batches(1000),
+        make_staggered_batches_i32(1000),
+        make_staggered_batches_i32(1000),
         JoinType::Left,
         Some(Box::new(col_lt_col_filter)),
     )
@@ -140,8 +140,8 @@ async fn test_left_join_1k_filtered() {
 #[tokio::test]
 async fn test_right_join_1k() {
     JoinFuzzTestCase::new(
-        make_staggered_batches(1000),
-        make_staggered_batches(1000),
+        make_staggered_batches_i32(1000),
+        make_staggered_batches_i32(1000),
         JoinType::Right,
         None,
     )
@@ -152,8 +152,8 @@ async fn test_right_join_1k() {
 #[tokio::test]
 async fn test_right_join_1k_filtered() {
     JoinFuzzTestCase::new(
-        make_staggered_batches(1000),
-        make_staggered_batches(1000),
+        make_staggered_batches_i32(1000),
+        make_staggered_batches_i32(1000),
         JoinType::Right,
         Some(Box::new(col_lt_col_filter)),
     )
@@ -164,8 +164,8 @@ async fn test_right_join_1k_filtered() {
 #[tokio::test]
 async fn test_full_join_1k() {
     JoinFuzzTestCase::new(
-        make_staggered_batches(1000),
-        make_staggered_batches(1000),
+        make_staggered_batches_i32(1000),
+        make_staggered_batches_i32(1000),
         JoinType::Full,
         None,
     )
@@ -176,8 +176,8 @@ async fn test_full_join_1k() {
 #[tokio::test]
 async fn test_full_join_1k_filtered() {
     JoinFuzzTestCase::new(
-        make_staggered_batches(1000),
-        make_staggered_batches(1000),
+        make_staggered_batches_i32(1000),
+        make_staggered_batches_i32(1000),
         JoinType::Full,
         Some(Box::new(col_lt_col_filter)),
     )
@@ -188,8 +188,8 @@ async fn test_full_join_1k_filtered() {
 #[tokio::test]
 async fn test_left_semi_join_1k() {
     JoinFuzzTestCase::new(
-        make_staggered_batches(1000),
-        make_staggered_batches(1000),
+        make_staggered_batches_i32(1000),
+        make_staggered_batches_i32(1000),
         JoinType::LeftSemi,
         None,
     )
@@ -200,8 +200,8 @@ async fn test_left_semi_join_1k() {
 #[tokio::test]
 async fn test_left_semi_join_1k_filtered() {
     JoinFuzzTestCase::new(
-        make_staggered_batches(1000),
-        make_staggered_batches(1000),
+        make_staggered_batches_i32(1000),
+        make_staggered_batches_i32(1000),
         JoinType::LeftSemi,
         Some(Box::new(col_lt_col_filter)),
     )
@@ -212,8 +212,8 @@ async fn test_left_semi_join_1k_filtered() {
 #[tokio::test]
 async fn test_right_semi_join_1k() {
     JoinFuzzTestCase::new(
-        make_staggered_batches(1000),
-        make_staggered_batches(1000),
+        make_staggered_batches_i32(1000),
+        make_staggered_batches_i32(1000),
         JoinType::RightSemi,
         None,
     )
@@ -224,8 +224,8 @@ async fn test_right_semi_join_1k() {
 #[tokio::test]
 async fn test_right_semi_join_1k_filtered() {
     JoinFuzzTestCase::new(
-        make_staggered_batches(1000),
-        make_staggered_batches(1000),
+        make_staggered_batches_i32(1000),
+        make_staggered_batches_i32(1000),
         JoinType::RightSemi,
         Some(Box::new(col_lt_col_filter)),
     )
@@ -236,8 +236,8 @@ async fn test_right_semi_join_1k_filtered() {
 #[tokio::test]
 async fn test_left_anti_join_1k() {
     JoinFuzzTestCase::new(
-        make_staggered_batches(1000),
-        make_staggered_batches(1000),
+        make_staggered_batches_i32(1000),
+        make_staggered_batches_i32(1000),
         JoinType::LeftAnti,
         None,
     )
@@ -248,8 +248,8 @@ async fn test_left_anti_join_1k() {
 #[tokio::test]
 async fn test_left_anti_join_1k_filtered() {
     JoinFuzzTestCase::new(
-        make_staggered_batches(1000),
-        make_staggered_batches(1000),
+        make_staggered_batches_i32(1000),
+        make_staggered_batches_i32(1000),
         JoinType::LeftAnti,
         Some(Box::new(col_lt_col_filter)),
     )
@@ -260,8 +260,8 @@ async fn test_left_anti_join_1k_filtered() {
 #[tokio::test]
 async fn test_right_anti_join_1k() {
     JoinFuzzTestCase::new(
-        make_staggered_batches(1000),
-        make_staggered_batches(1000),
+        make_staggered_batches_i32(1000),
+        make_staggered_batches_i32(1000),
         JoinType::RightAnti,
         None,
     )
@@ -272,8 +272,8 @@ async fn test_right_anti_join_1k() {
 #[tokio::test]
 async fn test_right_anti_join_1k_filtered() {
     JoinFuzzTestCase::new(
-        make_staggered_batches(1000),
-        make_staggered_batches(1000),
+        make_staggered_batches_i32(1000),
+        make_staggered_batches_i32(1000),
         JoinType::RightAnti,
         Some(Box::new(col_lt_col_filter)),
     )
@@ -284,8 +284,8 @@ async fn test_right_anti_join_1k_filtered() {
 #[tokio::test]
 async fn test_left_mark_join_1k() {
     JoinFuzzTestCase::new(
-        make_staggered_batches(1000),
-        make_staggered_batches(1000),
+        make_staggered_batches_i32(1000),
+        make_staggered_batches_i32(1000),
         JoinType::LeftMark,
         None,
     )
@@ -296,8 +296,8 @@ async fn test_left_mark_join_1k() {
 #[tokio::test]
 async fn test_left_mark_join_1k_filtered() {
     JoinFuzzTestCase::new(
-        make_staggered_batches(1000),
-        make_staggered_batches(1000),
+        make_staggered_batches_i32(1000),
+        make_staggered_batches_i32(1000),
         JoinType::LeftMark,
         Some(Box::new(col_lt_col_filter)),
     )
@@ -309,8 +309,8 @@ async fn test_left_mark_join_1k_filtered() {
 #[tokio::test]
 async fn test_right_mark_join_1k() {
     JoinFuzzTestCase::new(
-        make_staggered_batches(1000),
-        make_staggered_batches(1000),
+        make_staggered_batches_i32(1000),
+        make_staggered_batches_i32(1000),
         JoinType::RightMark,
         None,
     )
@@ -321,8 +321,249 @@ async fn test_right_mark_join_1k() {
 #[tokio::test]
 async fn test_right_mark_join_1k_filtered() {
     JoinFuzzTestCase::new(
-        make_staggered_batches(1000),
-        make_staggered_batches(1000),
+        make_staggered_batches_i32(1000),
+        make_staggered_batches_i32(1000),
+        JoinType::RightMark,
+        Some(Box::new(col_lt_col_filter)),
+    )
+    .run_test(&[NljHj], false)
+    .await
+}
+
+#[tokio::test]
+async fn test_inner_join_1k_binary_filtered() {
+    JoinFuzzTestCase::new(
+        make_staggered_batches_binary(1000),
+        make_staggered_batches_binary(1000),
+        JoinType::Inner,
+        Some(Box::new(col_lt_col_filter)),
+    )
+    .run_test(&[HjSmj, NljHj], false)
+    .await
+}
+
+#[tokio::test]
+async fn test_inner_join_1k_binary() {
+    JoinFuzzTestCase::new(
+        make_staggered_batches_binary(1000),
+        make_staggered_batches_binary(1000),
+        JoinType::Inner,
+        None,
+    )
+    .run_test(&[HjSmj, NljHj], false)
+    .await
+}
+
+#[tokio::test]
+async fn test_left_join_1k_binary() {
+    JoinFuzzTestCase::new(
+        make_staggered_batches_binary(1000),
+        make_staggered_batches_binary(1000),
+        JoinType::Left,
+        None,
+    )
+    .run_test(&[HjSmj, NljHj], false)
+    .await
+}
+
+#[tokio::test]
+async fn test_left_join_1k_binary_filtered() {
+    JoinFuzzTestCase::new(
+        make_staggered_batches_binary(1000),
+        make_staggered_batches_binary(1000),
+        JoinType::Left,
+        Some(Box::new(col_lt_col_filter)),
+    )
+    .run_test(&[HjSmj, NljHj], false)
+    .await
+}
+
+#[tokio::test]
+async fn test_right_join_1k_binary() {
+    JoinFuzzTestCase::new(
+        make_staggered_batches_binary(1000),
+        make_staggered_batches_binary(1000),
+        JoinType::Right,
+        None,
+    )
+    .run_test(&[HjSmj, NljHj], false)
+    .await
+}
+
+#[tokio::test]
+async fn test_right_join_1k_binary_filtered() {
+    JoinFuzzTestCase::new(
+        make_staggered_batches_binary(1000),
+        make_staggered_batches_binary(1000),
+        JoinType::Right,
+        Some(Box::new(col_lt_col_filter)),
+    )
+    .run_test(&[HjSmj, NljHj], false)
+    .await
+}
+
+#[tokio::test]
+async fn test_full_join_1k_binary() {
+    JoinFuzzTestCase::new(
+        make_staggered_batches_binary(1000),
+        make_staggered_batches_binary(1000),
+        JoinType::Full,
+        None,
+    )
+    .run_test(&[HjSmj, NljHj], false)
+    .await
+}
+
+#[tokio::test]
+async fn test_full_join_1k_binary_filtered() {
+    JoinFuzzTestCase::new(
+        make_staggered_batches_binary(1000),
+        make_staggered_batches_binary(1000),
+        JoinType::Full,
+        Some(Box::new(col_lt_col_filter)),
+    )
+    .run_test(&[NljHj, HjSmj], false)
+    .await
+}
+
+#[tokio::test]
+async fn test_left_semi_join_1k_binary() {
+    JoinFuzzTestCase::new(
+        make_staggered_batches_binary(1000),
+        make_staggered_batches_binary(1000),
+        JoinType::LeftSemi,
+        None,
+    )
+    .run_test(&[HjSmj, NljHj], false)
+    .await
+}
+
+#[tokio::test]
+async fn test_left_semi_join_1k_binary_filtered() {
+    JoinFuzzTestCase::new(
+        make_staggered_batches_binary(1000),
+        make_staggered_batches_binary(1000),
+        JoinType::LeftSemi,
+        Some(Box::new(col_lt_col_filter)),
+    )
+    .run_test(&[HjSmj, NljHj], false)
+    .await
+}
+
+#[tokio::test]
+async fn test_right_semi_join_1k_binary() {
+    JoinFuzzTestCase::new(
+        make_staggered_batches_binary(1000),
+        make_staggered_batches_binary(1000),
+        JoinType::RightSemi,
+        None,
+    )
+    .run_test(&[HjSmj, NljHj], false)
+    .await
+}
+
+#[tokio::test]
+async fn test_right_semi_join_1k_binary_filtered() {
+    JoinFuzzTestCase::new(
+        make_staggered_batches_binary(1000),
+        make_staggered_batches_binary(1000),
+        JoinType::RightSemi,
+        Some(Box::new(col_lt_col_filter)),
+    )
+    .run_test(&[HjSmj, NljHj], false)
+    .await
+}
+
+#[tokio::test]
+async fn test_left_anti_join_1k_binary() {
+    JoinFuzzTestCase::new(
+        make_staggered_batches_binary(1000),
+        make_staggered_batches_binary(1000),
+        JoinType::LeftAnti,
+        None,
+    )
+    .run_test(&[HjSmj, NljHj], false)
+    .await
+}
+
+#[tokio::test]
+async fn test_left_anti_join_1k_binary_filtered() {
+    JoinFuzzTestCase::new(
+        make_staggered_batches_binary(1000),
+        make_staggered_batches_binary(1000),
+        JoinType::LeftAnti,
+        Some(Box::new(col_lt_col_filter)),
+    )
+    .run_test(&[HjSmj, NljHj], false)
+    .await
+}
+
+#[tokio::test]
+async fn test_right_anti_join_1k_binary() {
+    JoinFuzzTestCase::new(
+        make_staggered_batches_binary(1000),
+        make_staggered_batches_binary(1000),
+        JoinType::RightAnti,
+        None,
+    )
+    .run_test(&[HjSmj, NljHj], false)
+    .await
+}
+
+#[tokio::test]
+async fn test_right_anti_join_1k_binary_filtered() {
+    JoinFuzzTestCase::new(
+        make_staggered_batches_binary(1000),
+        make_staggered_batches_binary(1000),
+        JoinType::RightAnti,
+        Some(Box::new(col_lt_col_filter)),
+    )
+    .run_test(&[HjSmj, NljHj], false)
+    .await
+}
+
+#[tokio::test]
+async fn test_left_mark_join_1k_binary() {
+    JoinFuzzTestCase::new(
+        make_staggered_batches_binary(1000),
+        make_staggered_batches_binary(1000),
+        JoinType::LeftMark,
+        None,
+    )
+    .run_test(&[HjSmj, NljHj], false)
+    .await
+}
+
+#[tokio::test]
+async fn test_left_mark_join_1k_binary_filtered() {
+    JoinFuzzTestCase::new(
+        make_staggered_batches_binary(1000),
+        make_staggered_batches_binary(1000),
+        JoinType::LeftMark,
+        Some(Box::new(col_lt_col_filter)),
+    )
+    .run_test(&[HjSmj, NljHj], false)
+    .await
+}
+
+// todo: add JoinTestType::HjSmj after Right mark SortMergeJoin support
+#[tokio::test]
+async fn test_right_mark_join_1k_binary() {
+    JoinFuzzTestCase::new(
+        make_staggered_batches_binary(1000),
+        make_staggered_batches_binary(1000),
+        JoinType::RightMark,
+        None,
+    )
+    .run_test(&[NljHj], false)
+    .await
+}
+
+#[tokio::test]
+async fn test_right_mark_join_1k_binary_filtered() {
+    JoinFuzzTestCase::new(
+        make_staggered_batches_binary(1000),
+        make_staggered_batches_binary(1000),
         JoinType::RightMark,
         Some(Box::new(col_lt_col_filter)),
     )
@@ -619,7 +860,6 @@ impl JoinFuzzTestCase {
                     hj_formatted_sorted.iter().for_each(|s| println!("{s}"));
                     println!("=============== NestedLoopJoinExec ==================");
                     nlj_formatted_sorted.iter().for_each(|s| println!("{s}"));
-
                     Self::save_partitioned_batches_as_parquet(
                         &nlj_collected,
                         out_dir_name,
@@ -791,7 +1031,7 @@ impl JoinFuzzTestCase {
 /// Return randomly sized record batches with:
 /// two sorted int32 columns 'a', 'b' ranged from 0..99 as join columns
 /// two random int32 columns 'x', 'y' as other columns
-fn make_staggered_batches(len: usize) -> Vec<RecordBatch> {
+fn make_staggered_batches_i32(len: usize) -> Vec<RecordBatch> {
     let mut rng = rand::rng();
     let mut input12: Vec<(i32, i32)> = vec![(0, 0); len];
     let mut input3: Vec<i32> = vec![0; len];
@@ -819,3 +1059,43 @@ fn make_staggered_batches(len: usize) -> Vec<RecordBatch> {
     // use a random number generator to pick a random sized output
     stagger_batch_with_seed(batch, 42)
 }
+
+fn rand_bytes<R: Rng>(rng: &mut R, min: usize, max: usize) -> Vec<u8> {
+    let n = rng.random_range(min..=max);
+    let mut v = vec![0u8; n];
+    rng.fill(&mut v[..]);
+    v
+}
+
+/// Return randomly sized record batches with:
+/// two sorted binary columns 'a', 'b' (lexicographically) as join columns
+/// two random binary columns 'x', 'y' as other columns
+fn make_staggered_batches_binary(len: usize) -> Vec<RecordBatch> {
+    let mut rng = rand::rng();
+
+    // produce (a,b) pairs then sort lexicographically so SMJ has naturally sorted keys
+    let mut input12: Vec<(Vec<u8>, Vec<u8>)> = (0..len)
+        .map(|_| (rand_bytes(&mut rng, 4, 16), rand_bytes(&mut rng, 4, 16)))
+        .collect();
+    input12.sort_unstable(); // lexicographic on Vec<u8>
+
+    // payload cols (also binary so the existing x < x filter is well-typed)
+    let input3: Vec<Vec<u8>> = (0..len).map(|_| rand_bytes(&mut rng, 4, 24)).collect();
+    let input4: Vec<Vec<u8>> = (0..len).map(|_| rand_bytes(&mut rng, 4, 24)).collect();
+
+    let a = BinaryArray::from_iter_values(input12.iter().map(|k| &k.0));
+    let b = BinaryArray::from_iter_values(input12.iter().map(|k| &k.1));
+    let x = BinaryArray::from_iter_values(input3.iter());
+    let y = BinaryArray::from_iter_values(input4.iter());
+
+    let batch = RecordBatch::try_from_iter(vec![
+        ("a", Arc::new(a) as ArrayRef),
+        ("b", Arc::new(b) as ArrayRef),
+        ("x", Arc::new(x) as ArrayRef),
+        ("y", Arc::new(y) as ArrayRef),
+    ])
+    .unwrap();
+
+    // preserve your existing randomized partitioning
+    stagger_batch_with_seed(batch, 42)
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org