You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by dh...@apache.org on 2023/07/01 07:47:20 UTC
[arrow-datafusion] branch create_hashes_primitive created (now b6ccfa450b)
This is an automated email from the ASF dual-hosted git repository.
dheres pushed a change to branch create_hashes_primitive
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
at b6ccfa450b Only rehash col >=1
This branch includes the following new commits:
new b6ccfa450b Only rehash col >=1
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[arrow-datafusion] 01/01: Only rehash col >=1
Posted by dh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dheres pushed a commit to branch create_hashes_primitive
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
commit b6ccfa450b86ed31814aaaa1df9a2c52a7513fcb
Author: Daniël Heres <da...@coralogix.com>
AuthorDate: Sat Jul 1 09:47:08 2023 +0200
Only rehash col >=1
---
datafusion/physical-expr/src/hash_utils.rs | 68 ++++++++++++++++--------------
1 file changed, 36 insertions(+), 32 deletions(-)
diff --git a/datafusion/physical-expr/src/hash_utils.rs b/datafusion/physical-expr/src/hash_utils.rs
index de3526992f..e3cb891902 100644
--- a/datafusion/physical-expr/src/hash_utils.rs
+++ b/datafusion/physical-expr/src/hash_utils.rs
@@ -88,31 +88,33 @@ fn hash_array_primitve<T>(
array: &PrimitiveArray<T>,
random_state: &RandomState,
hashes_buffer: &mut [u64],
- multi_col: bool,
+ rehash: bool,
) where
T: ArrowPrimitiveType,
<T as arrow_array::ArrowPrimitiveType>::Native: HashValue,
{
if array.null_count() == 0 {
- if multi_col {
- for (hash, &val) in hashes_buffer.iter_mut().zip(array.values().iter()) {
- *hash = combine_hashes(val.hash_one(&random_state), *hash);
+ if rehash {
+ for (hash, &value) in hashes_buffer.iter_mut().zip(array.values().iter()) {
+ *hash = combine_hashes(value.hash_one(&random_state), *hash);
}
} else {
- for (hash, &val) in hashes_buffer.iter_mut().zip(array.values().iter()) {
- *hash = val.hash_one(&random_state);
+ for (hash, &value) in hashes_buffer.iter_mut().zip(array.values().iter()) {
+ *hash = value.hash_one(&random_state);
}
}
- } else if multi_col {
+ } else if rehash {
for (i, hash) in hashes_buffer.iter_mut().enumerate() {
if !array.is_null(i) {
- *hash = combine_hashes(array.value(i).hash_one(random_state), *hash);
+ let value = unsafe {array.value_unchecked(i)};
+ *hash = combine_hashes(value.hash_one(random_state), *hash);
}
}
} else {
for (i, hash) in hashes_buffer.iter_mut().enumerate() {
if !array.is_null(i) {
- *hash = array.value(i).hash_one(random_state);
+ let value = unsafe {array.value_unchecked(i)};
+ *hash = value.hash_one(random_state);
}
}
}
@@ -122,31 +124,35 @@ fn hash_array<T>(
array: T,
random_state: &RandomState,
hashes_buffer: &mut [u64],
- multi_col: bool,
+ rehash: bool,
) where
T: ArrayAccessor,
T::Item: HashValue,
{
if array.null_count() == 0 {
- if multi_col {
+ if rehash {
for (i, hash) in hashes_buffer.iter_mut().enumerate() {
- *hash = combine_hashes(array.value(i).hash_one(random_state), *hash);
+ let value = unsafe {array.value_unchecked(i)};
+ *hash = combine_hashes(value.hash_one(random_state), *hash);
}
} else {
for (i, hash) in hashes_buffer.iter_mut().enumerate() {
- *hash = array.value(i).hash_one(random_state);
+ let value= unsafe {array.value_unchecked(i)};
+ *hash = value.hash_one(random_state);
}
}
- } else if multi_col {
+ } else if rehash {
for (i, hash) in hashes_buffer.iter_mut().enumerate() {
if !array.is_null(i) {
- *hash = combine_hashes(array.value(i).hash_one(random_state), *hash);
+ let value= unsafe {array.value_unchecked(i)};
+ *hash = combine_hashes(value.hash_one(random_state), *hash);
}
}
} else {
for (i, hash) in hashes_buffer.iter_mut().enumerate() {
if !array.is_null(i) {
- *hash = array.value(i).hash_one(random_state);
+ let value= unsafe {array.value_unchecked(i)};
+ *hash = value.hash_one(random_state);
}
}
}
@@ -242,34 +248,32 @@ pub fn create_hashes<'a>(
random_state: &RandomState,
hashes_buffer: &'a mut Vec<u64>,
) -> Result<&'a mut Vec<u64>> {
- // combine hashes with `combine_hashes` if we have more than 1 column
-
- let multi_col = arrays.len() > 1;
-
- for col in arrays {
+ for (i, col) in arrays.iter().enumerate() {
let array = col.as_ref();
+ // combine hashes with `combine_hashes` for all columns besides the first
+ let rehash = i >= 1;
downcast_primitive_array! {
- array => hash_array_primitve(array, random_state, hashes_buffer, multi_col),
- DataType::Null => hash_null(random_state, hashes_buffer, multi_col),
- DataType::Boolean => hash_array(as_boolean_array(array)?, random_state, hashes_buffer, multi_col),
- DataType::Utf8 => hash_array(as_string_array(array)?, random_state, hashes_buffer, multi_col),
- DataType::LargeUtf8 => hash_array(as_largestring_array(array), random_state, hashes_buffer, multi_col),
- DataType::Binary => hash_array(as_generic_binary_array::<i32>(array)?, random_state, hashes_buffer, multi_col),
- DataType::LargeBinary => hash_array(as_generic_binary_array::<i64>(array)?, random_state, hashes_buffer, multi_col),
+ array => hash_array_primitve(array, random_state, hashes_buffer, rehash),
+ DataType::Null => hash_null(random_state, hashes_buffer, rehash),
+ DataType::Boolean => hash_array(as_boolean_array(array)?, random_state, hashes_buffer, rehash),
+ DataType::Utf8 => hash_array(as_string_array(array)?, random_state, hashes_buffer, rehash),
+ DataType::LargeUtf8 => hash_array(as_largestring_array(array), random_state, hashes_buffer, rehash),
+ DataType::Binary => hash_array(as_generic_binary_array::<i32>(array)?, random_state, hashes_buffer, rehash),
+ DataType::LargeBinary => hash_array(as_generic_binary_array::<i64>(array)?, random_state, hashes_buffer, rehash),
DataType::FixedSizeBinary(_) => {
let array: &FixedSizeBinaryArray = array.as_any().downcast_ref().unwrap();
- hash_array(array, random_state, hashes_buffer, multi_col)
+ hash_array(array, random_state, hashes_buffer, rehash)
}
DataType::Decimal128(_, _) => {
let array = as_primitive_array::<Decimal128Type>(array)?;
- hash_array_primitve(array, random_state, hashes_buffer, multi_col)
+ hash_array_primitve(array, random_state, hashes_buffer, rehash)
}
DataType::Decimal256(_, _) => {
let array = as_primitive_array::<Decimal256Type>(array)?;
- hash_array_primitve(array, random_state, hashes_buffer, multi_col)
+ hash_array_primitve(array, random_state, hashes_buffer, rehash)
}
DataType::Dictionary(_, _) => downcast_dictionary_array! {
- array => hash_dictionary(array, random_state, hashes_buffer, multi_col)?,
+ array => hash_dictionary(array, random_state, hashes_buffer, rehash)?,
_ => unreachable!()
}
_ => {