You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by al...@apache.org on 2025/06/27 16:25:31 UTC
(arrow-rs) branch main updated: feat: add constructor to efficiently upgrade dict key type to remaining builders (#7689)
This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/main by this push:
new d910a1d22f feat: add constructor to efficiently upgrade dict key type to remaining builders (#7689)
d910a1d22f is described below
commit d910a1d22ff4180131f00dc647f60026d0adedeb
Author: albertlockett <a....@f5.com>
AuthorDate: Fri Jun 27 13:25:26 2025 -0300
feat: add constructor to efficiently upgrade dict key type to remaining builders (#7689)
# Which issue does this PR close?
Closes https://github.com/apache/arrow-rs/issues/7654
# Rationale for this change
In https://github.com/apache/arrow-rs/pull/7611 we added a constructor
to the GenericByteDictionaryBuilder to create a new builder instance
from an existing one, but with a modified type for the key. It would be
good if other dictionary builders had similar capability.
# What changes are included in this PR?
This followup PR adds this capability to remaining builder types
# Are there any user-facing changes?
No
---
.../fixed_size_binary_dictionary_builder.rs | 146 +++++++++++++++++-
.../src/builder/primitive_dictionary_builder.rs | 170 ++++++++++++++++++++-
2 files changed, 312 insertions(+), 4 deletions(-)
diff --git a/arrow-array/src/builder/fixed_size_binary_dictionary_builder.rs b/arrow-array/src/builder/fixed_size_binary_dictionary_builder.rs
index f3460353b1..21e842723b 100644
--- a/arrow-array/src/builder/fixed_size_binary_dictionary_builder.rs
+++ b/arrow-array/src/builder/fixed_size_binary_dictionary_builder.rs
@@ -17,11 +17,12 @@
use crate::builder::{ArrayBuilder, FixedSizeBinaryBuilder, PrimitiveBuilder};
use crate::types::ArrowDictionaryKeyType;
-use crate::{Array, ArrayRef, DictionaryArray};
+use crate::{Array, ArrayRef, DictionaryArray, PrimitiveArray};
use arrow_buffer::ArrowNativeType;
use arrow_schema::DataType::FixedSizeBinary;
use arrow_schema::{ArrowError, DataType};
use hashbrown::HashTable;
+use num::NumCast;
use std::any::Any;
use std::sync::Arc;
@@ -100,6 +101,71 @@ where
byte_width,
}
}
+
+ /// Creates a new `FixedSizeBinaryDictionaryBuilder` from the existing builder with the same
+ /// keys and values, but with a new data type for the keys.
+ ///
+ /// # Example
+ /// ```
+ /// # use arrow_array::builder::FixedSizeBinaryDictionaryBuilder;
+ /// # use arrow_array::types::{UInt8Type, UInt16Type, UInt64Type};
+ /// # use arrow_array::UInt16Array;
+ /// # use arrow_schema::ArrowError;
+ ///
+ /// let mut u8_keyed_builder = FixedSizeBinaryDictionaryBuilder::<UInt8Type>::new(2);
+ /// // appending too many values causes the dictionary to overflow
+ /// for i in 0..=255 {
+ /// u8_keyed_builder.append_value(vec![0, i]);
+ /// }
+ /// let result = u8_keyed_builder.append(vec![1, 0]);
+ /// assert!(matches!(result, Err(ArrowError::DictionaryKeyOverflowError{})));
+ ///
+ /// // we need to upgrade to a larger key type
+ /// let mut u16_keyed_builder = FixedSizeBinaryDictionaryBuilder::<UInt16Type>::try_new_from_builder(u8_keyed_builder).unwrap();
+ /// let dictionary_array = u16_keyed_builder.finish();
+ /// let keys = dictionary_array.keys();
+ ///
+ /// assert_eq!(keys, &UInt16Array::from_iter(0..256));
+ /// ```
+ pub fn try_new_from_builder<K2>(
+ mut source: FixedSizeBinaryDictionaryBuilder<K2>,
+ ) -> Result<Self, ArrowError>
+ where
+ K::Native: NumCast,
+ K2: ArrowDictionaryKeyType,
+ K2::Native: NumCast,
+ {
+ let state = source.state;
+ let dedup = source.dedup;
+ let values_builder = source.values_builder;
+ let byte_width = source.byte_width;
+
+ let source_keys = source.keys_builder.finish();
+ let new_keys: PrimitiveArray<K> = source_keys.try_unary(|value| {
+ num::cast::cast::<K2::Native, K::Native>(value).ok_or_else(|| {
+ ArrowError::CastError(format!(
+ "Can't cast dictionary keys from source type {:?} to type {:?}",
+ K2::DATA_TYPE,
+ K::DATA_TYPE
+ ))
+ })
+ })?;
+
+ // drop source key here because currently source_keys and new_keys are holding reference to
+ // the same underlying null_buffer. Below we want to call new_keys.into_builder() it must
+ // be the only reference holder.
+ drop(source_keys);
+
+ Ok(Self {
+ state,
+ dedup,
+ keys_builder: new_keys
+ .into_builder()
+ .expect("underlying buffer has no references"),
+ values_builder,
+ byte_width,
+ })
+ }
}
impl<K> ArrayBuilder for FixedSizeBinaryDictionaryBuilder<K>
@@ -258,8 +324,8 @@ fn get_bytes(values: &FixedSizeBinaryBuilder, byte_width: i32, idx: usize) -> &[
mod tests {
use super::*;
- use crate::types::Int8Type;
- use crate::{FixedSizeBinaryArray, Int8Array};
+ use crate::types::{Int16Type, Int32Type, Int8Type, UInt16Type, UInt8Type};
+ use crate::{ArrowPrimitiveType, FixedSizeBinaryArray, Int8Array};
#[test]
fn test_fixed_size_dictionary_builder() {
@@ -368,4 +434,78 @@ mod tests {
assert_eq!(ava2.value(1), values[1].as_bytes());
assert_eq!(ava2.value(2), values[2].as_bytes());
}
+
+ fn _test_try_new_from_builder_generic_for_key_types<K1, K2>(values: Vec<[u8; 3]>)
+ where
+ K1: ArrowDictionaryKeyType,
+ K1::Native: NumCast,
+ K2: ArrowDictionaryKeyType,
+ K2::Native: NumCast + From<u8>,
+ {
+ let mut source = FixedSizeBinaryDictionaryBuilder::<K1>::new(3);
+ source.append_value(values[0]);
+ source.append_null();
+ source.append_value(values[1]);
+ source.append_value(values[2]);
+
+ let mut result =
+ FixedSizeBinaryDictionaryBuilder::<K2>::try_new_from_builder(source).unwrap();
+ let array = result.finish();
+
+ let mut expected_keys_builder = PrimitiveBuilder::<K2>::new();
+ expected_keys_builder
+ .append_value(<<K2 as ArrowPrimitiveType>::Native as From<u8>>::from(0u8));
+ expected_keys_builder.append_null();
+ expected_keys_builder
+ .append_value(<<K2 as ArrowPrimitiveType>::Native as From<u8>>::from(1u8));
+ expected_keys_builder
+ .append_value(<<K2 as ArrowPrimitiveType>::Native as From<u8>>::from(2u8));
+ let expected_keys = expected_keys_builder.finish();
+ assert_eq!(array.keys(), &expected_keys);
+
+ let av = array.values();
+ let ava = av.as_any().downcast_ref::<FixedSizeBinaryArray>().unwrap();
+ assert_eq!(ava.value(0), values[0]);
+ assert_eq!(ava.value(1), values[1]);
+ assert_eq!(ava.value(2), values[2]);
+ }
+
+ #[test]
+ fn test_try_new_from_builder() {
+ let values = vec![[1, 2, 3], [5, 6, 7], [6, 7, 8]];
+ // test cast to bigger size unsigned
+ _test_try_new_from_builder_generic_for_key_types::<UInt8Type, UInt16Type>(values.clone());
+ // test cast going to smaller size unsigned
+ _test_try_new_from_builder_generic_for_key_types::<UInt16Type, UInt8Type>(values.clone());
+ // test cast going to bigger size signed
+ _test_try_new_from_builder_generic_for_key_types::<Int8Type, Int16Type>(values.clone());
+ // test cast going to smaller size signed
+ _test_try_new_from_builder_generic_for_key_types::<Int32Type, Int16Type>(values.clone());
+ // test going from signed to signed for different size changes
+ _test_try_new_from_builder_generic_for_key_types::<UInt8Type, Int16Type>(values.clone());
+ _test_try_new_from_builder_generic_for_key_types::<Int8Type, UInt8Type>(values.clone());
+ _test_try_new_from_builder_generic_for_key_types::<Int8Type, UInt16Type>(values.clone());
+ _test_try_new_from_builder_generic_for_key_types::<Int32Type, Int16Type>(values.clone());
+ }
+
+ #[test]
+ fn test_try_new_from_builder_cast_fails() {
+ let mut source_builder = FixedSizeBinaryDictionaryBuilder::<UInt16Type>::new(2);
+ for i in 0u16..257u16 {
+ source_builder.append_value(vec![(i >> 8) as u8, i as u8]);
+ }
+
+ // there should be too many values that we can't downcast to the underlying type
+ // we have keys that wouldn't fit into UInt8Type
+ let result =
+ FixedSizeBinaryDictionaryBuilder::<UInt8Type>::try_new_from_builder(source_builder);
+ assert!(result.is_err());
+ if let Err(e) = result {
+ assert!(matches!(e, ArrowError::CastError(_)));
+ assert_eq!(
+ e.to_string(),
+ "Cast error: Can't cast dictionary keys from source type UInt16 to type UInt8"
+ );
+ }
+ }
}
diff --git a/arrow-array/src/builder/primitive_dictionary_builder.rs b/arrow-array/src/builder/primitive_dictionary_builder.rs
index f4a6662462..1d921c6df0 100644
--- a/arrow-array/src/builder/primitive_dictionary_builder.rs
+++ b/arrow-array/src/builder/primitive_dictionary_builder.rs
@@ -22,6 +22,7 @@ use crate::{
};
use arrow_buffer::{ArrowNativeType, ToByteSlice};
use arrow_schema::{ArrowError, DataType};
+use num::NumCast;
use std::any::Any;
use std::collections::HashMap;
use std::sync::Arc;
@@ -169,6 +170,68 @@ where
map: HashMap::with_capacity(values_capacity),
}
}
+
+ /// Creates a new `PrimitiveDictionaryBuilder` from the existing builder with the same
+ /// keys and values, but with a new data type for the keys.
+ ///
+ /// # Example
+ /// ```
+ /// #
+ /// # use arrow_array::builder::PrimitiveDictionaryBuilder;
+ /// # use arrow_array::types::{UInt8Type, UInt16Type, UInt64Type};
+ /// # use arrow_array::UInt16Array;
+ /// # use arrow_schema::ArrowError;
+ ///
+ /// let mut u8_keyed_builder = PrimitiveDictionaryBuilder::<UInt8Type, UInt64Type>::new();
+ ///
+ /// // appending too many values causes the dictionary to overflow
+ /// for i in 0..256 {
+ /// u8_keyed_builder.append_value(i);
+ /// }
+ /// let result = u8_keyed_builder.append(256);
+ /// assert!(matches!(result, Err(ArrowError::DictionaryKeyOverflowError{})));
+ ///
+ /// // we need to upgrade to a larger key type
+ /// let mut u16_keyed_builder = PrimitiveDictionaryBuilder::<UInt16Type, UInt64Type>::try_new_from_builder(u8_keyed_builder).unwrap();
+ /// let dictionary_array = u16_keyed_builder.finish();
+ /// let keys = dictionary_array.keys();
+ ///
+ /// assert_eq!(keys, &UInt16Array::from_iter(0..256));
+ pub fn try_new_from_builder<K2>(
+ mut source: PrimitiveDictionaryBuilder<K2, V>,
+ ) -> Result<Self, ArrowError>
+ where
+ K::Native: NumCast,
+ K2: ArrowDictionaryKeyType,
+ K2::Native: NumCast,
+ {
+ let map = source.map;
+ let values_builder = source.values_builder;
+
+ let source_keys = source.keys_builder.finish();
+ let new_keys: PrimitiveArray<K> = source_keys.try_unary(|value| {
+ num::cast::cast::<K2::Native, K::Native>(value).ok_or_else(|| {
+ ArrowError::CastError(format!(
+ "Can't cast dictionary keys from source type {:?} to type {:?}",
+ K2::DATA_TYPE,
+ K::DATA_TYPE
+ ))
+ })
+ })?;
+
+ // drop source key here because currently source_keys and new_keys are holding reference to
+ // the same underlying null_buffer. Below we want to call new_keys.into_builder() it must
+ // be the only reference holder.
+ drop(source_keys);
+
+ Ok(Self {
+ map,
+ keys_builder: new_keys
+ .into_builder()
+ .expect("underlying buffer has no references"),
+ values_builder,
+ })
+ }
}
impl<K, V> ArrayBuilder for PrimitiveDictionaryBuilder<K, V>
@@ -431,7 +494,11 @@ mod tests {
use crate::array::{Int32Array, UInt32Array, UInt8Array};
use crate::builder::Decimal128Builder;
use crate::cast::AsArray;
- use crate::types::{Decimal128Type, Int32Type, UInt32Type, UInt8Type};
+ use crate::types::{
+ Date32Type, Decimal128Type, DurationNanosecondType, Float32Type, Float64Type, Int16Type,
+ Int32Type, Int64Type, Int8Type, TimestampNanosecondType, UInt16Type, UInt32Type,
+ UInt64Type, UInt8Type,
+ };
#[test]
fn test_primitive_dictionary_builder() {
@@ -649,4 +716,105 @@ mod tests {
builder.values_builder.capacity()
)
}
+
+ fn _test_try_new_from_builder_generic_for_key_types<K1, K2, V>(values: Vec<V::Native>)
+ where
+ K1: ArrowDictionaryKeyType,
+ K1::Native: NumCast,
+ K2: ArrowDictionaryKeyType,
+ K2::Native: NumCast + From<u8>,
+ V: ArrowPrimitiveType,
+ {
+ let mut source = PrimitiveDictionaryBuilder::<K1, V>::new();
+ source.append(values[0]).unwrap();
+ source.append_null();
+ source.append(values[1]).unwrap();
+ source.append(values[2]).unwrap();
+
+ let mut result = PrimitiveDictionaryBuilder::<K2, V>::try_new_from_builder(source).unwrap();
+ let array = result.finish();
+
+ let mut expected_keys_builder = PrimitiveBuilder::<K2>::new();
+ expected_keys_builder
+ .append_value(<<K2 as ArrowPrimitiveType>::Native as From<u8>>::from(0u8));
+ expected_keys_builder.append_null();
+ expected_keys_builder
+ .append_value(<<K2 as ArrowPrimitiveType>::Native as From<u8>>::from(1u8));
+ expected_keys_builder
+ .append_value(<<K2 as ArrowPrimitiveType>::Native as From<u8>>::from(2u8));
+ let expected_keys = expected_keys_builder.finish();
+ assert_eq!(array.keys(), &expected_keys);
+
+ let av = array.values();
+ let ava = av.as_any().downcast_ref::<PrimitiveArray<V>>().unwrap();
+ assert_eq!(ava.value(0), values[0]);
+ assert_eq!(ava.value(1), values[1]);
+ assert_eq!(ava.value(2), values[2]);
+ }
+
+ fn _test_try_new_from_builder_generic_for_value<T>(values: Vec<T::Native>)
+ where
+ T: ArrowPrimitiveType,
+ {
+ // test cast to bigger size unsigned
+ _test_try_new_from_builder_generic_for_key_types::<UInt8Type, UInt16Type, T>(
+ values.clone(),
+ );
+ // test cast going to smaller size unsigned
+ _test_try_new_from_builder_generic_for_key_types::<UInt16Type, UInt8Type, T>(
+ values.clone(),
+ );
+ // test cast going to bigger size signed
+ _test_try_new_from_builder_generic_for_key_types::<Int8Type, Int16Type, T>(values.clone());
+ // test cast going to smaller size signed
+ _test_try_new_from_builder_generic_for_key_types::<Int32Type, Int16Type, T>(values.clone());
+ // test going from signed to signed for different size changes
+ _test_try_new_from_builder_generic_for_key_types::<UInt8Type, Int16Type, T>(values.clone());
+ _test_try_new_from_builder_generic_for_key_types::<Int8Type, UInt8Type, T>(values.clone());
+ _test_try_new_from_builder_generic_for_key_types::<Int8Type, UInt16Type, T>(values.clone());
+ _test_try_new_from_builder_generic_for_key_types::<Int32Type, Int16Type, T>(values.clone());
+ }
+
+ #[test]
+ fn test_try_new_from_builder() {
+ // test unsigned types
+ _test_try_new_from_builder_generic_for_value::<UInt8Type>(vec![1, 2, 3]);
+ _test_try_new_from_builder_generic_for_value::<UInt16Type>(vec![1, 2, 3]);
+ _test_try_new_from_builder_generic_for_value::<UInt32Type>(vec![1, 2, 3]);
+ _test_try_new_from_builder_generic_for_value::<UInt64Type>(vec![1, 2, 3]);
+ // test signed types
+ _test_try_new_from_builder_generic_for_value::<Int8Type>(vec![-1, 0, 1]);
+ _test_try_new_from_builder_generic_for_value::<Int16Type>(vec![-1, 0, 1]);
+ _test_try_new_from_builder_generic_for_value::<Int32Type>(vec![-1, 0, 1]);
+ _test_try_new_from_builder_generic_for_value::<Int64Type>(vec![-1, 0, 1]);
+ // test some date types
+ _test_try_new_from_builder_generic_for_value::<Date32Type>(vec![5, 6, 7]);
+ _test_try_new_from_builder_generic_for_value::<DurationNanosecondType>(vec![1, 2, 3]);
+ _test_try_new_from_builder_generic_for_value::<TimestampNanosecondType>(vec![1, 2, 3]);
+ // test some floating point types
+ _test_try_new_from_builder_generic_for_value::<Float32Type>(vec![0.1, 0.2, 0.3]);
+ _test_try_new_from_builder_generic_for_value::<Float64Type>(vec![-0.1, 0.2, 0.3]);
+ }
+
+ #[test]
+ fn test_try_new_from_builder_cast_fails() {
+ let mut source_builder = PrimitiveDictionaryBuilder::<UInt16Type, UInt64Type>::new();
+ for i in 0..257 {
+ source_builder.append_value(i);
+ }
+
+ // there should be too many values that we can't downcast to the underlying type
+ // we have keys that wouldn't fit into UInt8Type
+ let result = PrimitiveDictionaryBuilder::<UInt8Type, UInt64Type>::try_new_from_builder(
+ source_builder,
+ );
+ assert!(result.is_err());
+ if let Err(e) = result {
+ assert!(matches!(e, ArrowError::CastError(_)));
+ assert_eq!(
+ e.to_string(),
+ "Cast error: Can't cast dictionary keys from source type UInt16 to type UInt8"
+ );
+ }
+ }
}