(datafusion) 01/01: test: add tests for spill file sizes to verify View GC (#21750)

[gi...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-21750-a311d149bf7ee345c9c9db5486b67ff98d8ce708
in repository https://gitbox.apache.org/repos/asf/datafusion.git

commit 7acbe03efb853274aac1bdd31c1dfdc8ea647642
Author: Ratul Dawar <ra...@gmail.com>
AuthorDate: Tue Apr 21 01:24:00 2026 +0530

    test: add tests for spill file sizes to verify View GC (#21750)
    
    ## Summary
    This PR adds unit tests to verify that `StringView` and `BinaryView`
    arrays are correctly compacted (garbage collected) before being spilled
    to disk.
    
    The tests address [Issue
    #21683](https://github.com/apache/datafusion/issues/21683) by:
    1. Creating "bloated" batches with large underlying buffers.
    2. Slicing them to a small number of rows (1%).
    3. Spilling them to disk.
    4. Asserting that the resulting file size is small (proving that GC
    removed the unused "ghost" data).
    
    ## Test plan
    - Run `cargo test -p datafusion-physical-plan --lib
    spill::mod::tests::test_spill_file_size_gc_verification_string_view`
    - Run `cargo test -p datafusion-physical-plan --lib
    spill::mod::tests::test_spill_file_size_gc_verification_binary_view`
    
    Made with [Cursor](https://cursor.com)
---
 datafusion/physical-plan/src/spill/mod.rs | 88 +++++++++++++++++++++++++++++++
 1 file changed, 88 insertions(+)

diff --git a/datafusion/physical-plan/src/spill/mod.rs b/datafusion/physical-plan/src/spill/mod.rs
index 1c900b7579..e8c9e9167b 100644
--- a/datafusion/physical-plan/src/spill/mod.rs
+++ b/datafusion/physical-plan/src/spill/mod.rs
@@ -1421,4 +1421,92 @@ mod tests {
 
         Ok(())
     }
+
+    #[tokio::test]
+    async fn test_spill_file_size_gc_verification_string_view() -> Result<()> {
+        use arrow::array::StringViewArray;
+        use std::fs;
+
+        // 1. Setup bloated data (large buffers)
+        let num_rows = 1000;
+        let string_array: StringViewArray = (0..num_rows)
+            .map(|i| Some(format!("this_is_a_long_string_to_ensure_it_is_not_inlined_and_causes_waste_{i}")))
+            .collect();
+        let schema = Arc::new(Schema::new(vec![Field::new(
+            "s",
+            DataType::Utf8View,
+            false,
+        )]));
+        let batch = RecordBatch::try_new(
+            Arc::clone(&schema),
+            vec![Arc::new(string_array.clone()) as ArrayRef],
+        )?;
+
+        // 2. Slice it heavily (1% of the data)
+        let sliced_batch = batch.slice(0, 10);
+
+        // 3. Spill to disk using SpillManager
+        let env = Arc::new(RuntimeEnv::default());
+        let metrics = SpillMetrics::new(&ExecutionPlanMetricsSet::new(), 0);
+        let spill_manager = SpillManager::new(env, metrics, schema);
+        let spill_file = spill_manager
+            .spill_record_batch_and_finish(&[sliced_batch], "TestGC")?
+            .unwrap();
+
+        // 4. Check file size on disk
+        let file_size = fs::metadata(spill_file.path())?.len();
+
+        // The original buffer size is around 70KB.
+        // Without GC, the spill file would be > 70KB.
+        // With GC, it should be much smaller (only 10 rows of ~70 bytes each + metadata).
+        assert!(
+            file_size < 10 * 1024,
+            "Spill file is too large ({file_size} bytes)! GC might not be working."
+        );
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_spill_file_size_gc_verification_binary_view() -> Result<()> {
+        use arrow::array::BinaryViewArray;
+        use std::fs;
+
+        // 1. Setup bloated data (large buffers)
+        let num_rows = 1000;
+        let binary_array: BinaryViewArray =
+            (0..num_rows).map(|i| Some(vec![i as u8; 100])).collect();
+        let schema = Arc::new(Schema::new(vec![Field::new(
+            "b",
+            DataType::BinaryView,
+            false,
+        )]));
+        let batch = RecordBatch::try_new(
+            Arc::clone(&schema),
+            vec![Arc::new(binary_array.clone()) as ArrayRef],
+        )?;
+
+        // 2. Slice it heavily (1% of the data)
+        let sliced_batch = batch.slice(0, 10);
+
+        // 3. Spill to disk using SpillManager
+        let env = Arc::new(RuntimeEnv::default());
+        let metrics = SpillMetrics::new(&ExecutionPlanMetricsSet::new(), 0);
+        let spill_manager = SpillManager::new(env, metrics, schema);
+        let spill_file = spill_manager
+            .spill_record_batch_and_finish(&[sliced_batch], "TestGCBinary")?
+            .unwrap();
+
+        // 4. Check file size on disk
+        let file_size = fs::metadata(spill_file.path())?.len();
+
+        // Original buffer is 100KB.
+        // With GC, it should be much smaller.
+        assert!(
+            file_size < 10 * 1024,
+            "Spill file is too large ({file_size} bytes)! GC might not be working."
+        );
+
+        Ok(())
+    }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org