You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by ka...@apache.org on 2026/07/10 04:23:18 UTC
(datafusion) branch improve-compare-in-view-map-v2 updated (65f2cac95f -> 7d5fa64f05)
This is an automated email from the ASF dual-hosted git repository.
Rachelint pushed a change to branch improve-compare-in-view-map-v2
in repository https://gitbox.apache.org/repos/asf/datafusion.git
discard 65f2cac95f test.
discard 74c9b4f03b add fast path.
new 7d5fa64f05 try improve hash.
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (65f2cac95f)
\
N -- N -- N refs/heads/improve-compare-in-view-map-v2 (7d5fa64f05)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../physical-expr-common/src/binary_view_map.rs | 179 +++++++++++++++------
1 file changed, 128 insertions(+), 51 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org
(datafusion) 01/01: try improve hash.
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
Rachelint pushed a commit to branch improve-compare-in-view-map-v2
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 7d5fa64f0542f0271f62d71fb88638fb4bdfc506
Author: kamille <ka...@apache.org>
AuthorDate: Fri Jul 10 12:23:03 2026 +0800
try improve hash.
---
.../physical-expr-common/src/binary_view_map.rs | 60 +++++++++++++++++-----
1 file changed, 46 insertions(+), 14 deletions(-)
diff --git a/datafusion/physical-expr-common/src/binary_view_map.rs b/datafusion/physical-expr-common/src/binary_view_map.rs
index 53b84e7dea..7a54fdd907 100644
--- a/datafusion/physical-expr-common/src/binary_view_map.rs
+++ b/datafusion/physical-expr-common/src/binary_view_map.rs
@@ -24,7 +24,6 @@ use arrow::array::{Array, ArrayRef, BinaryViewArray, ByteView, make_view};
use arrow::buffer::{Buffer, ScalarBuffer};
use arrow::datatypes::{BinaryViewType, ByteViewType, DataType, StringViewType};
use datafusion_common::hash_utils::RandomState;
-use datafusion_common::hash_utils::create_hashes;
use datafusion_common::utils::proxy::{HashTableAllocExt, VecAllocExt};
use std::fmt::Debug;
use std::mem::size_of;
@@ -273,22 +272,22 @@ where
OP: FnMut(V),
B: ByteViewType,
{
- // step 1: compute hashes
- let batch_hashes = &mut self.hashes_buffer;
- batch_hashes.clear();
- batch_hashes.resize(values.len(), 0);
- create_hashes([values], &self.random_state, batch_hashes)
- // hash is supported for all types and create_hashes only
- // returns errors for unsupported types
- .unwrap();
-
- // step 2: insert each value into the set, if not already present
let values = values.as_byte_view::<B>();
-
- // Get raw views buffer for direct comparison
let input_views = values.views();
- // Ensure lengths are equivalent
+ let batch_hashes = &mut self.hashes_buffer;
+ batch_hashes.clear();
+ batch_hashes.reserve(values.len());
+ for i in 0..values.len() {
+ let view_u128 = input_views[i];
+ let hash = if values.is_null(i) {
+ 0
+ } else {
+ Self::cheap_hash::<B>(values, i, view_u128)
+ };
+ batch_hashes.push(hash);
+ }
+
assert_eq!(values.len(), self.hashes_buffer.len());
for i in 0..values.len() {
@@ -394,6 +393,39 @@ where
}
}
+ #[inline(always)]
+ fn cheap_hash<B: ByteViewType>(
+ values: &arrow::array::GenericByteViewArray<B>,
+ row: usize,
+ view_u128: u128,
+ ) -> u64 {
+ let len = view_u128 as usize;
+ let (first8, last8) = if len <= 12 {
+ let view_bytes = view_u128.to_le_bytes();
+ let value = &view_bytes[4..4 + len];
+ let first8 = Self::sample_u64(value, 0);
+ let last8_offset = value.len().saturating_sub(8);
+ let last8 = Self::sample_u64(value, last8_offset);
+ (first8, last8)
+ } else {
+ let value: &[u8] = values.value(row).as_ref();
+ let first8 = Self::sample_u64(value, 0);
+ let last8_offset = value.len().saturating_sub(8);
+ let last8 = Self::sample_u64(value, last8_offset);
+ (first8, last8)
+ };
+
+ (len as u64).wrapping_add(first8).wrapping_add(last8)
+ }
+
+ #[inline(always)]
+ fn sample_u64(value: &[u8], offset: usize) -> u64 {
+ let mut sample = [0_u8; 8];
+ let sample_len = value.len().saturating_sub(offset).min(8);
+ sample[..sample_len].copy_from_slice(&value[offset..offset + sample_len]);
+ u64::from_le_bytes(sample)
+ }
+
/// Converts this set into a `StringViewArray`, or `BinaryViewArray`,
/// containing each distinct value
/// that was inserted. This is done without copying the values.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org