You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/09/29 23:18:38 UTC

(datafusion-comet) branch gh-readonly-queue/main/pr-6342-098f58db88a250ab79c48575983428ce0c704c9f created (now c7dd26a9c1)

This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a change to branch gh-readonly-queue/main/pr-6342-098f58db88a250ab79c48575983428ce0c704c9f
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git


      at c7dd26a9c1 perf: reuse Variant reconstruction buffers (#6342)

This branch includes the following new commits:

     new c7dd26a9c1 perf: reuse Variant reconstruction buffers (#6342)

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org


(datafusion-comet) 01/01: perf: reuse Variant reconstruction buffers (#6342)

Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-6342-098f58db88a250ab79c48575983428ce0c704c9f
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git

commit c7dd26a9c1c05041c4d60465ce31ef147d25b48c
Author: Peter Lee <pe...@gmail.com>
AuthorDate: Tue Sep 29 22:01:24 2026 +0000

    perf: reuse Variant reconstruction buffers (#6342)
    
    * perf: reuse Variant reconstruction buffers
    
    * ci: retry after Maven distribution download failure
---
 native/core/src/parquet/cast_column/variant.rs     | 95 ++++++++++++++--------
 .../core/src/parquet/cast_column/variant/tests.rs  | 58 +++++++++----
 2 files changed, 103 insertions(+), 50 deletions(-)

diff --git a/native/core/src/parquet/cast_column/variant.rs b/native/core/src/parquet/cast_column/variant.rs
index 5e18f603b0..4e50fc602b 100644
--- a/native/core/src/parquet/cast_column/variant.rs
+++ b/native/core/src/parquet/cast_column/variant.rs
@@ -20,6 +20,7 @@ use arrow::{
         make_array, Array, ArrayRef, AsArray, BinaryArray, BinaryBuilder, ListLikeArray,
         StructArray,
     },
+    buffer::OffsetBuffer,
     compute::{cast, cast_with_options},
     datatypes::{DataType, FieldRef, TimeUnit, DECIMAL128_MAX_PRECISION},
     error::ArrowError,
@@ -73,9 +74,11 @@ pub(super) fn normalize_variant_array(
             error => error.into(),
         })?;
         let (value, metadata) = if variant.typed_value_column().is_some() {
-            let value = cast(unshredded.value_column().as_ref(), &DataType::Binary)?;
-            let metadata = cast(unshredded.metadata_column().as_ref(), &DataType::Binary)?;
-            rebuild_spark_variant(&variant, &value, &metadata)?
+            rebuild_spark_variant(
+                &variant,
+                unshredded.value_column(),
+                unshredded.metadata_column(),
+            )?
         } else {
             // Spark passes unshredded bytes through, including dictionary order, unused keys,
             // and wide scalar encodings. Preparation above still validates legacy input.
@@ -768,7 +771,9 @@ impl MetadataBuilder for SparkOutputMetadata {
     }
 
     fn finish(&mut self) -> usize {
-        self.dictionary.finish()
+        let offset = self.dictionary.finish();
+        self.sort_keys.clear();
+        offset
     }
 }
 
@@ -984,40 +989,60 @@ fn rebuild_spark_variant(
     value: &ArrayRef,
     metadata: &ArrayRef,
 ) -> DataFusionResult<(ArrayRef, ArrayRef)> {
-    let mut values = BinaryBuilder::new();
-    let mut dictionaries = BinaryBuilder::new();
+    // Append directly to batch buffers. Finishing the metadata builder resets the dictionary
+    // for the next row while retaining its output buffer.
+    let mut values = ValueBuilder::new();
+    let mut dictionary = SparkOutputMetadata::default();
+    let mut value_offsets = Vec::with_capacity(source.len() + 1);
+    let mut metadata_offsets = Vec::with_capacity(source.len() + 1);
+    value_offsets.push(0);
+    metadata_offsets.push(0);
+    let checked_offset =
+        |offset: usize| i32::try_from(offset).map_err(|_| ArrowError::OffsetOverflowError(offset));
     for row in 0..source.len() {
-        if source.is_null(row) {
-            values.append_null();
-            dictionaries.append_null();
-            continue;
+        if source.is_valid(row) {
+            catch_unwind(AssertUnwindSafe(|| -> Result<(), ArrowError> {
+                let value =
+                    Variant::try_new(binary_value(metadata, row)?, binary_value(value, row)?)?;
+                // The preparation pass already validated/canonicalized legacy input metadata.
+                let original = VariantMetadata::new(binary_value(source.metadata_column(), row)?);
+                append_spark_variant(
+                    &mut SparkValueBuilder {
+                        value: &mut values,
+                        metadata: &mut dictionary,
+                    },
+                    value,
+                    Some((source.inner(), row)),
+                    &original,
+                )?;
+                dictionary.finish();
+                Ok(())
+            }))
+            .map_err(|_| SparkError::MalformedVariant)?
+            .map_err(|_| SparkError::MalformedVariant)?;
+        }
+        value_offsets.push(checked_offset(values.offset())?);
+        metadata_offsets.push(checked_offset(dictionary.dictionary.offset())?);
+    }
+    let mut metadata = dictionary.dictionary.into_inner();
+    for (row, &offset) in metadata_offsets[..source.len()].iter().enumerate() {
+        if source.is_valid(row) {
+            metadata[offset as usize] &= !0x10;
         }
-        let rebuilt = catch_unwind(AssertUnwindSafe(|| -> Result<_, ArrowError> {
-            let value = Variant::try_new(binary_value(metadata, row)?, binary_value(value, row)?)?;
-            // The preparation pass already validated/canonicalized legacy input metadata.
-            let original = VariantMetadata::new(binary_value(source.metadata_column(), row)?);
-            let mut output = ValueBuilder::new();
-            let mut dictionary = SparkOutputMetadata::default();
-            append_spark_variant(
-                &mut SparkValueBuilder {
-                    value: &mut output,
-                    metadata: &mut dictionary,
-                },
-                value,
-                Some((source.inner(), row)),
-                &original,
-            )?;
-            dictionary.finish();
-            let mut metadata = dictionary.dictionary.into_inner();
-            metadata[0] &= !0x10;
-            Ok((output.into_inner(), metadata))
-        }))
-        .map_err(|_| SparkError::MalformedVariant)?
-        .map_err(|_| SparkError::MalformedVariant)?;
-        values.append_value(rebuilt.0);
-        dictionaries.append_value(rebuilt.1);
     }
-    Ok((Arc::new(values.finish()), Arc::new(dictionaries.finish())))
+    let nulls = source.nulls();
+    Ok((
+        Arc::new(BinaryArray::try_new(
+            OffsetBuffer::new(value_offsets.into()),
+            values.into_inner().into(),
+            nulls.cloned(),
+        )?),
+        Arc::new(BinaryArray::try_new(
+            OffsetBuffer::new(metadata_offsets.into()),
+            metadata.into(),
+            nulls.cloned(),
+        )?),
+    ))
 }
 
 /// Supplies sort-only field names whose Rust ordering matches Java `String.compareTo` ordering.
diff --git a/native/core/src/parquet/cast_column/variant/tests.rs b/native/core/src/parquet/cast_column/variant/tests.rs
index 93d0f6a7bc..fc5c6a1b11 100644
--- a/native/core/src/parquet/cast_column/variant/tests.rs
+++ b/native/core/src/parquet/cast_column/variant/tests.rs
@@ -473,12 +473,17 @@ fn normalize_rejects_missing_required_shredding_states() {
 
 #[test]
 fn canonical_and_shredded_values_normalize_equally() {
-    let mut builder = VariantArrayBuilder::new(6);
+    let mut builder = VariantArrayBuilder::new(8);
+    // Cross an object-offset width boundary before writing the remaining rows.
+    let payload = "x".repeat(65536);
+    builder.append_null();
     builder.new_object().with_field("known", 1_i8).finish();
+    builder.append_null();
     builder
         .new_object()
         .with_field("known", 2_i8)
         .with_field("extra", 3_i64)
+        .with_field("payload", payload.as_str())
         .finish();
     builder
         .new_list()
@@ -515,12 +520,33 @@ fn canonical_and_shredded_values_normalize_equally() {
         VariantArray::try_new(output.as_ref()).unwrap()
     };
     let canonical = normalize(&canonical);
-    let shredded = normalize(&shredded);
+    let output = normalize(&shredded);
+    for (offset, len) in [(1, 6), (2, 0), (2, 1)] {
+        assert_eq!(
+            normalize(&shredded.slice(offset, len)).inner(),
+            output.slice(offset, len).inner(),
+        );
+    }
 
     for index in 0..canonical.len() {
-        assert_eq!(canonical.is_null(index), shredded.is_null(index));
+        assert_eq!(canonical.is_null(index), output.is_null(index));
         if canonical.is_valid(index) {
-            assert_eq!(canonical.value(index), shredded.value(index));
+            assert_eq!(canonical.value(index), output.value(index));
+            assert_eq!(
+                binary_value(canonical.value_column(), index).unwrap(),
+                binary_value(output.value_column(), index).unwrap(),
+            );
+            let mut metadata = binary_value(canonical.metadata_column(), index)
+                .unwrap()
+                .to_vec();
+            metadata[0] &= !0x10;
+            assert_eq!(
+                binary_value(output.metadata_column(), index).unwrap(),
+                metadata
+            );
+        } else {
+            assert!(output.value_column().is_null(index));
+            assert!(output.metadata_column().is_null(index));
         }
     }
 }
@@ -710,15 +736,13 @@ fn benchmark_variant_buffer_reuse() {
     }
     let objects = objects.build();
     let empty_keys = empty_keys.build();
-    let shredded = shred_variant(
-        &objects,
-        &DataType::Struct(Fields::from(vec![Field::new(
-            "known",
-            DataType::Int64,
-            true,
-        )])),
-    )
-    .unwrap();
+    let partial_type = DataType::Struct(Fields::from(vec![Field::new(
+        "known",
+        DataType::Int64,
+        true,
+    )]));
+    let shredded = shred_variant(&objects, &partial_type).unwrap();
+    let empty_keys = shred_variant(&empty_keys, &partial_type).unwrap();
     let full = shred_variant(
         &objects,
         &DataType::Struct(Fields::from(vec![
@@ -742,13 +766,17 @@ fn benchmark_variant_buffer_reuse() {
                 empty_keys.value_column().data_type().clone(),
                 false,
             ),
-            Field::new("typed_value", DataType::Int64, true),
+            Field::new(
+                "typed_value",
+                empty_keys.typed_value_column().unwrap().data_type().clone(),
+                true,
+            ),
         ]
         .into(),
         vec![
             empty_metadata,
             Arc::clone(empty_keys.value_column()),
-            Arc::new(Int64Array::from(vec![None; rows])),
+            Arc::clone(empty_keys.typed_value_column().unwrap()),
         ],
         None,
     ));


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org