You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by tu...@apache.org on 2023/03/25 12:16:23 UTC
[arrow-rs] branch master updated: Add Zero-Copy Conversion between Vec and MutableBuffer (#3920)
This is an automated email from the ASF dual-hosted git repository.
tustvold pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/master by this push:
new bc8ba3c13 Add Zero-Copy Conversion between Vec and MutableBuffer (#3920)
bc8ba3c13 is described below
commit bc8ba3c136d5afebae18884040e5f91b89a410d5
Author: Raphael Taylor-Davies <17...@users.noreply.github.com>
AuthorDate: Sat Mar 25 12:16:17 2023 +0000
Add Zero-Copy Conversion between Vec and MutableBuffer (#3920)
* Add Zero-Copy Conversion between Vec and MutableBuffer
* Update test
* Fix docs
* Clippy
---
arrow-array/src/array/list_array.rs | 8 +-
arrow-buffer/src/alloc/mod.rs | 104 +---------------------
arrow-buffer/src/buffer/immutable.rs | 17 +---
arrow-buffer/src/buffer/mutable.rs | 166 ++++++++++++++++++++---------------
arrow-buffer/src/bytes.rs | 4 +-
5 files changed, 102 insertions(+), 197 deletions(-)
diff --git a/arrow-array/src/array/list_array.rs b/arrow-array/src/array/list_array.rs
index c7e2a817b..895f15007 100644
--- a/arrow-array/src/array/list_array.rs
+++ b/arrow-array/src/array/list_array.rs
@@ -842,10 +842,8 @@ mod tests {
#[test]
#[should_panic(expected = "memory is not aligned")]
- #[allow(deprecated)]
fn test_primitive_array_alignment() {
- let ptr = arrow_buffer::alloc::allocate_aligned(8);
- let buf = unsafe { Buffer::from_raw_parts(ptr, 8, 8) };
+ let buf = Buffer::from_slice_ref([0_u64]);
let buf2 = buf.slice(1);
let array_data = ArrayData::builder(DataType::Int32)
.add_buffer(buf2)
@@ -859,10 +857,8 @@ mod tests {
// Different error messages, so skip for now
// https://github.com/apache/arrow-rs/issues/1545
#[cfg(not(feature = "force_validate"))]
- #[allow(deprecated)]
fn test_list_array_alignment() {
- let ptr = arrow_buffer::alloc::allocate_aligned(8);
- let buf = unsafe { Buffer::from_raw_parts(ptr, 8, 8) };
+ let buf = Buffer::from_slice_ref([0_u64]);
let buf2 = buf.slice(1);
let values: [i32; 8] = [0; 8];
diff --git a/arrow-buffer/src/alloc/mod.rs b/arrow-buffer/src/alloc/mod.rs
index 7600a28d8..d1236eeaa 100644
--- a/arrow-buffer/src/alloc/mod.rs
+++ b/arrow-buffer/src/alloc/mod.rs
@@ -18,117 +18,15 @@
//! Defines memory-related functions, such as allocate/deallocate/reallocate memory
//! regions, cache and allocation alignments.
-use std::alloc::{handle_alloc_error, Layout};
+use std::alloc::Layout;
use std::fmt::{Debug, Formatter};
use std::panic::RefUnwindSafe;
-use std::ptr::NonNull;
use std::sync::Arc;
mod alignment;
pub use alignment::ALIGNMENT;
-/// Returns an aligned non null pointer similar to [`NonNull::dangling`]
-///
-/// Note that the pointer value may potentially represent a valid pointer, which means
-/// this must not be used as a "not yet initialized" sentinel value.
-///
-/// Types that lazily allocate must track initialization by some other means.
-#[inline]
-fn dangling_ptr() -> NonNull<u8> {
- // SAFETY: ALIGNMENT is a non-zero usize which is then casted
- // to a *mut T. Therefore, `ptr` is not null and the conditions for
- // calling new_unchecked() are respected.
- unsafe { NonNull::new_unchecked(ALIGNMENT as *mut u8) }
-}
-
-/// Allocates a cache-aligned memory region of `size` bytes with uninitialized values.
-/// This is more performant than using [allocate_aligned_zeroed] when all bytes will have
-/// an unknown or non-zero value and is semantically similar to `malloc`.
-#[deprecated(note = "Use Vec")]
-pub fn allocate_aligned(size: usize) -> NonNull<u8> {
- unsafe {
- if size == 0 {
- dangling_ptr()
- } else {
- let layout = Layout::from_size_align_unchecked(size, ALIGNMENT);
- let raw_ptr = std::alloc::alloc(layout);
- NonNull::new(raw_ptr).unwrap_or_else(|| handle_alloc_error(layout))
- }
- }
-}
-
-/// Allocates a cache-aligned memory region of `size` bytes with `0` on all of them.
-/// This is more performant than using [allocate_aligned] and setting all bytes to zero
-/// and is semantically similar to `calloc`.
-#[deprecated(note = "Use Vec")]
-pub fn allocate_aligned_zeroed(size: usize) -> NonNull<u8> {
- unsafe {
- if size == 0 {
- dangling_ptr()
- } else {
- let layout = Layout::from_size_align_unchecked(size, ALIGNMENT);
- let raw_ptr = std::alloc::alloc_zeroed(layout);
- NonNull::new(raw_ptr).unwrap_or_else(|| handle_alloc_error(layout))
- }
- }
-}
-
-/// # Safety
-///
-/// This function is unsafe because undefined behavior can result if the caller does not ensure all
-/// of the following:
-///
-/// * ptr must denote a block of memory currently allocated via this allocator,
-///
-/// * size must be the same size that was used to allocate that block of memory,
-#[deprecated(note = "Use Vec")]
-pub unsafe fn free_aligned(ptr: NonNull<u8>, size: usize) {
- if size != 0 {
- std::alloc::dealloc(
- ptr.as_ptr() as *mut u8,
- Layout::from_size_align_unchecked(size, ALIGNMENT),
- );
- }
-}
-
-/// # Safety
-///
-/// This function is unsafe because undefined behavior can result if the caller does not ensure all
-/// of the following:
-///
-/// * ptr must be currently allocated via this allocator,
-///
-/// * new_size must be greater than zero.
-///
-/// * new_size, when rounded up to the nearest multiple of [ALIGNMENT], must not overflow (i.e.,
-/// the rounded value must be less than usize::MAX).
-#[deprecated(note = "Use Vec")]
-#[allow(deprecated)]
-pub unsafe fn reallocate(
- ptr: NonNull<u8>,
- old_size: usize,
- new_size: usize,
-) -> NonNull<u8> {
- if old_size == 0 {
- return allocate_aligned(new_size);
- }
-
- if new_size == 0 {
- free_aligned(ptr, old_size);
- return dangling_ptr();
- }
-
- let raw_ptr = std::alloc::realloc(
- ptr.as_ptr() as *mut u8,
- Layout::from_size_align_unchecked(old_size, ALIGNMENT),
- new_size,
- );
- NonNull::new(raw_ptr).unwrap_or_else(|| {
- handle_alloc_error(Layout::from_size_align_unchecked(new_size, ALIGNMENT))
- })
-}
-
/// The owner of an allocation.
/// The trait implementation is responsible for dropping the allocations once no more references exist.
pub trait Allocation: RefUnwindSafe + Send + Sync {}
diff --git a/arrow-buffer/src/buffer/immutable.rs b/arrow-buffer/src/buffer/immutable.rs
index 5f42035c9..15d9ff783 100644
--- a/arrow-buffer/src/buffer/immutable.rs
+++ b/arrow-buffer/src/buffer/immutable.rs
@@ -71,20 +71,10 @@ impl Buffer {
}
}
- /// Create a [`Buffer`] from the provided `Vec` without copying
+ /// Create a [`Buffer`] from the provided [`Vec`] without copying
#[inline]
pub fn from_vec<T: ArrowNativeType>(vec: Vec<T>) -> Self {
- // Safety
- // Vec::as_ptr guaranteed to not be null and ArrowNativeType are trivially transmutable
- let ptr = unsafe { NonNull::new_unchecked(vec.as_ptr() as _) };
- let len = vec.len() * std::mem::size_of::<T>();
- // Safety
- // Vec guaranteed to have a valid layout matching that of `Layout::array`
- // This is based on `RawVec::current_memory`
- let layout = unsafe { Layout::array::<T>(vec.capacity()).unwrap_unchecked() };
- std::mem::forget(vec);
- let b = unsafe { Bytes::new(ptr, len, Deallocation::Standard(layout)) };
- Self::from_bytes(b)
+ MutableBuffer::from_vec(vec).into()
}
/// Initializes a [Buffer] from a slice of items.
@@ -810,7 +800,8 @@ mod tests {
b.into_mutable().unwrap();
let b = Buffer::from_vec(vec![1_u32, 3, 5]);
- let b = b.into_mutable().unwrap_err(); // Invalid layout
+ let b = b.into_mutable().unwrap();
+ let b = Buffer::from(b);
let b = b.into_vec::<u32>().unwrap();
assert_eq!(b, &[1, 3, 5]);
}
diff --git a/arrow-buffer/src/buffer/mutable.rs b/arrow-buffer/src/buffer/mutable.rs
index 250ac9f31..9a905a322 100644
--- a/arrow-buffer/src/buffer/mutable.rs
+++ b/arrow-buffer/src/buffer/mutable.rs
@@ -15,17 +15,18 @@
// specific language governing permissions and limitations
// under the License.
-use super::Buffer;
+use std::alloc::{handle_alloc_error, Layout};
+use std::mem;
+use std::ptr::NonNull;
+
use crate::alloc::{Deallocation, ALIGNMENT};
use crate::{
- alloc,
bytes::Bytes,
native::{ArrowNativeType, ToByteSlice},
util::bit_util,
};
-use std::alloc::Layout;
-use std::mem;
-use std::ptr::NonNull;
+
+use super::Buffer;
/// A [`MutableBuffer`] is Arrow's interface to build a [`Buffer`] out of items or slices of items.
///
@@ -55,7 +56,7 @@ pub struct MutableBuffer {
data: NonNull<u8>,
// invariant: len <= capacity
len: usize,
- capacity: usize,
+ layout: Layout,
}
impl MutableBuffer {
@@ -67,14 +68,21 @@ impl MutableBuffer {
/// Allocate a new [MutableBuffer] with initial capacity to be at least `capacity`.
#[inline]
- #[allow(deprecated)]
pub fn with_capacity(capacity: usize) -> Self {
let capacity = bit_util::round_upto_multiple_of_64(capacity);
- let ptr = alloc::allocate_aligned(capacity);
+ let layout = Layout::from_size_align(capacity, ALIGNMENT).unwrap();
+ let data = match layout.size() {
+ 0 => dangling_ptr(),
+ _ => {
+ // Safety: Verified size != 0
+ let raw_ptr = unsafe { std::alloc::alloc(layout) };
+ NonNull::new(raw_ptr).unwrap_or_else(|| handle_alloc_error(layout))
+ }
+ };
Self {
- data: ptr,
+ data,
len: 0,
- capacity,
+ layout,
}
}
@@ -89,35 +97,46 @@ impl MutableBuffer {
/// let data = buffer.as_slice_mut();
/// assert_eq!(data[126], 0u8);
/// ```
- #[allow(deprecated)]
pub fn from_len_zeroed(len: usize) -> Self {
- let new_capacity = bit_util::round_upto_multiple_of_64(len);
- let ptr = alloc::allocate_aligned_zeroed(new_capacity);
- Self {
- data: ptr,
- len,
- capacity: new_capacity,
- }
+ let layout = Layout::from_size_align(len, ALIGNMENT).unwrap();
+ let data = match layout.size() {
+ 0 => dangling_ptr(),
+ _ => {
+ // Safety: Verified size != 0
+ let raw_ptr = unsafe { std::alloc::alloc_zeroed(layout) };
+ NonNull::new(raw_ptr).unwrap_or_else(|| handle_alloc_error(layout))
+ }
+ };
+ Self { data, len, layout }
+ }
+
+ /// Create a [`MutableBuffer`] from the provided [`Vec`] without copying
+ #[inline]
+ pub fn from_vec<T: ArrowNativeType>(vec: Vec<T>) -> Self {
+ // Safety
+ // Vec::as_ptr guaranteed to not be null and ArrowNativeType are trivially transmutable
+ let data = unsafe { NonNull::new_unchecked(vec.as_ptr() as _) };
+ let len = vec.len() * mem::size_of::<T>();
+ // Safety
+ // Vec guaranteed to have a valid layout matching that of `Layout::array`
+ // This is based on `RawVec::current_memory`
+ let layout = unsafe { Layout::array::<T>(vec.capacity()).unwrap_unchecked() };
+ mem::forget(vec);
+ Self { data, len, layout }
}
/// Allocates a new [MutableBuffer] from given `Bytes`.
pub(crate) fn from_bytes(bytes: Bytes) -> Result<Self, Bytes> {
- let capacity = match bytes.deallocation() {
- Deallocation::Standard(layout) if layout.align() == ALIGNMENT => {
- layout.size()
- }
+ let layout = match bytes.deallocation() {
+ Deallocation::Standard(layout) => *layout,
_ => return Err(bytes),
};
let len = bytes.len();
- let ptr = bytes.ptr();
+ let data = bytes.ptr();
mem::forget(bytes);
- Ok(Self {
- data: ptr,
- len,
- capacity,
- })
+ Ok(Self { data, len, layout })
}
/// creates a new [MutableBuffer] with capacity and length capable of holding `len` bits.
@@ -134,7 +153,7 @@ impl MutableBuffer {
/// the buffer directly (e.g., modifying the buffer by holding a mutable reference
/// from `data_mut()`).
pub fn with_bitset(mut self, end: usize, val: bool) -> Self {
- assert!(end <= self.capacity);
+ assert!(end <= self.layout.size());
let v = if val { 255 } else { 0 };
unsafe {
std::ptr::write_bytes(self.data.as_ptr(), v, end);
@@ -149,7 +168,7 @@ impl MutableBuffer {
/// `len` of the buffer and so can be used to initialize the memory region from
/// `len` to `capacity`.
pub fn set_null_bits(&mut self, start: usize, count: usize) {
- assert!(start + count <= self.capacity);
+ assert!(start + count <= self.layout.size());
unsafe {
std::ptr::write_bytes(self.data.as_ptr().add(start), 0, count);
}
@@ -171,17 +190,33 @@ impl MutableBuffer {
#[inline(always)]
pub fn reserve(&mut self, additional: usize) {
let required_cap = self.len + additional;
- if required_cap > self.capacity {
- // JUSTIFICATION
- // Benefit
- // necessity
- // Soundness
- // `self.data` is valid for `self.capacity`.
- let (ptr, new_capacity) =
- unsafe { reallocate(self.data, self.capacity, required_cap) };
- self.data = ptr;
- self.capacity = new_capacity;
+ if required_cap > self.layout.size() {
+ let new_capacity = bit_util::round_upto_multiple_of_64(required_cap);
+ let new_capacity = std::cmp::max(new_capacity, self.layout.size() * 2);
+ self.reallocate(new_capacity)
+ }
+ }
+
+ #[cold]
+ fn reallocate(&mut self, capacity: usize) {
+ let new_layout = Layout::from_size_align(capacity, self.layout.align()).unwrap();
+ if new_layout.size() == 0 {
+ if self.layout.size() != 0 {
+ // Safety: data was allocated with layout
+ unsafe { std::alloc::dealloc(self.as_mut_ptr(), self.layout) };
+ self.layout = new_layout
+ }
+ return;
}
+
+ let data = match self.layout.size() {
+ // Safety: new_layout is not empty
+ 0 => unsafe { std::alloc::alloc(new_layout) },
+ // Safety: verified new layout is valid and not empty
+ _ => unsafe { std::alloc::realloc(self.as_mut_ptr(), self.layout, capacity) },
+ };
+ self.data = NonNull::new(data).unwrap_or_else(|| handle_alloc_error(new_layout));
+ self.layout = new_layout;
}
/// Truncates this buffer to `len` bytes
@@ -233,20 +268,10 @@ impl MutableBuffer {
/// buffer.shrink_to_fit();
/// assert!(buffer.capacity() >= 64 && buffer.capacity() < 128);
/// ```
- #[allow(deprecated)]
pub fn shrink_to_fit(&mut self) {
let new_capacity = bit_util::round_upto_multiple_of_64(self.len);
- if new_capacity < self.capacity {
- // JUSTIFICATION
- // Benefit
- // necessity
- // Soundness
- // `self.data` is valid for `self.capacity`.
- let ptr =
- unsafe { alloc::reallocate(self.data, self.capacity, new_capacity) };
-
- self.data = ptr;
- self.capacity = new_capacity;
+ if new_capacity < self.layout.size() {
+ self.reallocate(new_capacity)
}
}
@@ -267,7 +292,7 @@ impl MutableBuffer {
/// The invariant `buffer.len() <= buffer.capacity()` is always upheld.
#[inline]
pub const fn capacity(&self) -> usize {
- self.capacity
+ self.layout.size()
}
/// Clear all existing data from this buffer.
@@ -310,9 +335,9 @@ impl MutableBuffer {
#[inline]
pub(super) fn into_buffer(self) -> Buffer {
- let layout = Layout::from_size_align(self.capacity, ALIGNMENT).unwrap();
- let bytes =
- unsafe { Bytes::new(self.data, self.len, Deallocation::Standard(layout)) };
+ let bytes = unsafe {
+ Bytes::new(self.data, self.len, Deallocation::Standard(self.layout))
+ };
std::mem::forget(self);
Buffer::from_bytes(bytes)
}
@@ -455,19 +480,12 @@ impl MutableBuffer {
}
}
-/// # Safety
-/// `ptr` must be allocated for `old_capacity`.
-#[cold]
-#[allow(deprecated)]
-unsafe fn reallocate(
- ptr: NonNull<u8>,
- old_capacity: usize,
- new_capacity: usize,
-) -> (NonNull<u8>, usize) {
- let new_capacity = bit_util::round_upto_multiple_of_64(new_capacity);
- let new_capacity = std::cmp::max(new_capacity, old_capacity * 2);
- let ptr = alloc::reallocate(ptr, old_capacity, new_capacity);
- (ptr, new_capacity)
+#[inline]
+fn dangling_ptr() -> NonNull<u8> {
+ // SAFETY: ALIGNMENT is a non-zero usize which is then casted
+ // to a *mut T. Therefore, `ptr` is not null and the conditions for
+ // calling new_unchecked() are respected.
+ unsafe { NonNull::new_unchecked(ALIGNMENT as *mut u8) }
}
impl<A: ArrowNativeType> Extend<A> for MutableBuffer {
@@ -492,7 +510,7 @@ impl MutableBuffer {
// this is necessary because of https://github.com/rust-lang/rust/issues/32155
let mut len = SetLenOnDrop::new(&mut self.len);
let mut dst = unsafe { self.data.as_ptr().add(len.local_len) };
- let capacity = self.capacity;
+ let capacity = self.layout.size();
while len.local_len + item_size <= capacity {
if let Some(item) = iterator.next() {
@@ -641,9 +659,11 @@ impl std::ops::DerefMut for MutableBuffer {
}
impl Drop for MutableBuffer {
- #[allow(deprecated)]
fn drop(&mut self) {
- unsafe { alloc::free_aligned(self.data, self.capacity) };
+ if self.layout.size() != 0 {
+ // Safety: data was allocated with standard allocator with given layout
+ unsafe { std::alloc::dealloc(self.data.as_ptr() as _, self.layout) };
+ }
}
}
@@ -652,7 +672,7 @@ impl PartialEq for MutableBuffer {
if self.len != other.len {
return false;
}
- if self.capacity != other.capacity {
+ if self.layout != other.layout {
return false;
}
self.as_slice() == other.as_slice()
diff --git a/arrow-buffer/src/bytes.rs b/arrow-buffer/src/bytes.rs
index 2820fda78..b3105ed5a 100644
--- a/arrow-buffer/src/bytes.rs
+++ b/arrow-buffer/src/bytes.rs
@@ -30,8 +30,8 @@ use crate::alloc::Deallocation;
/// This structs' API is inspired by the `bytes::Bytes`, but it is not limited to using rust's
/// global allocator nor u8 alignment.
///
-/// In the most common case, this buffer is allocated using [`allocate_aligned`](crate::alloc::allocate_aligned)
-/// and deallocated accordingly [`free_aligned`](crate::alloc::free_aligned).
+/// In the most common case, this buffer is allocated using [`alloc`](std::alloc::alloc)
+/// with an alignment of [`ALIGNMENT`](crate::alloc::ALIGNMENT)
///
/// When the region is allocated by a different allocator, [Deallocation::Custom], this calls the
/// custom deallocator to deallocate the region when it is no longer needed.