You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by tu...@apache.org on 2023/06/02 00:23:24 UTC
[arrow-rs] branch master updated: Move BooleanBufferBuilder and NullBufferBuilder to arrow_buffer (#4338)
This is an automated email from the ASF dual-hosted git repository.
tustvold pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/master by this push:
new a121e0969 Move BooleanBufferBuilder and NullBufferBuilder to arrow_buffer (#4338)
a121e0969 is described below
commit a121e0969ee83d3396c59603717333864acc52fa
Author: Raphael Taylor-Davies <17...@users.noreply.github.com>
AuthorDate: Fri Jun 2 01:23:18 2023 +0100
Move BooleanBufferBuilder and NullBufferBuilder to arrow_buffer (#4338)
* Move BooleanBufferBuilder and NullBufferBuilder to arrow_buffer
* Clippy
---
arrow-array/src/builder/boolean_builder.rs | 11 +-
.../src/builder/fixed_size_binary_builder.rs | 10 +-
arrow-array/src/builder/fixed_size_list_builder.rs | 14 +-
arrow-array/src/builder/generic_bytes_builder.rs | 10 +-
arrow-array/src/builder/generic_list_builder.rs | 11 +-
arrow-array/src/builder/map_builder.rs | 14 +-
arrow-array/src/builder/mod.rs | 4 +-
arrow-array/src/builder/primitive_builder.rs | 13 +-
arrow-array/src/builder/struct_builder.rs | 14 +-
arrow-array/src/builder/union_builder.rs | 4 +-
.../src/builder/boolean.rs | 8 +-
arrow-buffer/src/{util => builder}/mod.rs | 9 +-
.../src/builder/null.rs | 22 +--
arrow-buffer/src/lib.rs | 3 +
.../src => arrow-buffer/src/util}/bit_mask.rs | 151 +--------------------
arrow-buffer/src/util/mod.rs | 1 +
arrow-data/src/lib.rs | 3 +-
17 files changed, 69 insertions(+), 233 deletions(-)
diff --git a/arrow-array/src/builder/boolean_builder.rs b/arrow-array/src/builder/boolean_builder.rs
index a35e6f6b9..0def0ec48 100644
--- a/arrow-array/src/builder/boolean_builder.rs
+++ b/arrow-array/src/builder/boolean_builder.rs
@@ -15,10 +15,10 @@
// specific language governing permissions and limitations
// under the License.
-use crate::builder::null_buffer_builder::NullBufferBuilder;
use crate::builder::{ArrayBuilder, BooleanBufferBuilder};
use crate::{ArrayRef, BooleanArray};
use arrow_buffer::Buffer;
+use arrow_buffer::NullBufferBuilder;
use arrow_data::ArrayData;
use arrow_schema::{ArrowError, DataType};
use std::any::Any;
@@ -150,7 +150,7 @@ impl BooleanBuilder {
let builder = ArrayData::builder(DataType::Boolean)
.len(len)
.add_buffer(self.values_builder.finish().into_inner())
- .null_bit_buffer(null_bit_buffer);
+ .nulls(null_bit_buffer);
let array_data = unsafe { builder.build_unchecked() };
BooleanArray::from(array_data)
@@ -159,15 +159,12 @@ impl BooleanBuilder {
/// Builds the [BooleanArray] without resetting the builder.
pub fn finish_cloned(&self) -> BooleanArray {
let len = self.len();
- let null_bit_buffer = self
- .null_buffer_builder
- .as_slice()
- .map(Buffer::from_slice_ref);
+ let nulls = self.null_buffer_builder.finish_cloned();
let value_buffer = Buffer::from_slice_ref(self.values_builder.as_slice());
let builder = ArrayData::builder(DataType::Boolean)
.len(len)
.add_buffer(value_buffer)
- .null_bit_buffer(null_bit_buffer);
+ .nulls(nulls);
let array_data = unsafe { builder.build_unchecked() };
BooleanArray::from(array_data)
diff --git a/arrow-array/src/builder/fixed_size_binary_builder.rs b/arrow-array/src/builder/fixed_size_binary_builder.rs
index a354a1db2..a213b3bbf 100644
--- a/arrow-array/src/builder/fixed_size_binary_builder.rs
+++ b/arrow-array/src/builder/fixed_size_binary_builder.rs
@@ -15,10 +15,10 @@
// specific language governing permissions and limitations
// under the License.
-use crate::builder::null_buffer_builder::NullBufferBuilder;
use crate::builder::{ArrayBuilder, UInt8BufferBuilder};
use crate::{ArrayRef, FixedSizeBinaryArray};
use arrow_buffer::Buffer;
+use arrow_buffer::NullBufferBuilder;
use arrow_data::ArrayData;
use arrow_schema::{ArrowError, DataType};
use std::any::Any;
@@ -98,7 +98,7 @@ impl FixedSizeBinaryBuilder {
let array_data_builder =
ArrayData::builder(DataType::FixedSizeBinary(self.value_length))
.add_buffer(self.values_builder.finish())
- .null_bit_buffer(self.null_buffer_builder.finish())
+ .nulls(self.null_buffer_builder.finish())
.len(array_length);
let array_data = unsafe { array_data_builder.build_unchecked() };
FixedSizeBinaryArray::from(array_data)
@@ -111,11 +111,7 @@ impl FixedSizeBinaryBuilder {
let array_data_builder =
ArrayData::builder(DataType::FixedSizeBinary(self.value_length))
.add_buffer(values_buffer)
- .null_bit_buffer(
- self.null_buffer_builder
- .as_slice()
- .map(Buffer::from_slice_ref),
- )
+ .nulls(self.null_buffer_builder.finish_cloned())
.len(array_length);
let array_data = unsafe { array_data_builder.build_unchecked() };
FixedSizeBinaryArray::from(array_data)
diff --git a/arrow-array/src/builder/fixed_size_list_builder.rs b/arrow-array/src/builder/fixed_size_list_builder.rs
index ab9fbf5fa..0dd580443 100644
--- a/arrow-array/src/builder/fixed_size_list_builder.rs
+++ b/arrow-array/src/builder/fixed_size_list_builder.rs
@@ -15,10 +15,9 @@
// specific language governing permissions and limitations
// under the License.
-use crate::builder::null_buffer_builder::NullBufferBuilder;
use crate::builder::ArrayBuilder;
use crate::{ArrayRef, FixedSizeListArray};
-use arrow_buffer::Buffer;
+use arrow_buffer::NullBufferBuilder;
use arrow_data::ArrayData;
use arrow_schema::{DataType, Field};
use std::any::Any;
@@ -167,14 +166,14 @@ where
len,
);
- let null_bit_buffer = self.null_buffer_builder.finish();
+ let nulls = self.null_buffer_builder.finish();
let array_data = ArrayData::builder(DataType::FixedSizeList(
Arc::new(Field::new("item", values_data.data_type().clone(), true)),
self.list_len,
))
.len(len)
.add_child_data(values_data)
- .null_bit_buffer(null_bit_buffer);
+ .nulls(nulls);
let array_data = unsafe { array_data.build_unchecked() };
@@ -195,17 +194,14 @@ where
len,
);
- let null_bit_buffer = self
- .null_buffer_builder
- .as_slice()
- .map(Buffer::from_slice_ref);
+ let nulls = self.null_buffer_builder.finish_cloned();
let array_data = ArrayData::builder(DataType::FixedSizeList(
Arc::new(Field::new("item", values_data.data_type().clone(), true)),
self.list_len,
))
.len(len)
.add_child_data(values_data)
- .null_bit_buffer(null_bit_buffer);
+ .nulls(nulls);
let array_data = unsafe { array_data.build_unchecked() };
diff --git a/arrow-array/src/builder/generic_bytes_builder.rs b/arrow-array/src/builder/generic_bytes_builder.rs
index 1887ab36c..f77940055 100644
--- a/arrow-array/src/builder/generic_bytes_builder.rs
+++ b/arrow-array/src/builder/generic_bytes_builder.rs
@@ -15,10 +15,10 @@
// specific language governing permissions and limitations
// under the License.
-use crate::builder::null_buffer_builder::NullBufferBuilder;
use crate::builder::{ArrayBuilder, BufferBuilder, UInt8BufferBuilder};
use crate::types::{ByteArrayType, GenericBinaryType, GenericStringType};
use crate::{ArrayRef, GenericByteArray, OffsetSizeTrait};
+use arrow_buffer::NullBufferBuilder;
use arrow_buffer::{ArrowNativeType, Buffer, MutableBuffer};
use arrow_data::ArrayDataBuilder;
use std::any::Any;
@@ -123,7 +123,7 @@ impl<T: ByteArrayType> GenericByteBuilder<T> {
.len(self.len())
.add_buffer(self.offsets_builder.finish())
.add_buffer(self.value_builder.finish())
- .null_bit_buffer(self.null_buffer_builder.finish());
+ .nulls(self.null_buffer_builder.finish());
self.offsets_builder.append(self.next_offset());
let array_data = unsafe { array_builder.build_unchecked() };
@@ -139,11 +139,7 @@ impl<T: ByteArrayType> GenericByteBuilder<T> {
.len(self.len())
.add_buffer(offset_buffer)
.add_buffer(value_buffer)
- .null_bit_buffer(
- self.null_buffer_builder
- .as_slice()
- .map(Buffer::from_slice_ref),
- );
+ .nulls(self.null_buffer_builder.finish_cloned());
let array_data = unsafe { array_builder.build_unchecked() };
GenericByteArray::from(array_data)
diff --git a/arrow-array/src/builder/generic_list_builder.rs b/arrow-array/src/builder/generic_list_builder.rs
index 054c87187..99e15d10f 100644
--- a/arrow-array/src/builder/generic_list_builder.rs
+++ b/arrow-array/src/builder/generic_list_builder.rs
@@ -15,10 +15,10 @@
// specific language governing permissions and limitations
// under the License.
-use crate::builder::null_buffer_builder::NullBufferBuilder;
use crate::builder::{ArrayBuilder, BufferBuilder};
use crate::{Array, ArrayRef, GenericListArray, OffsetSizeTrait};
use arrow_buffer::Buffer;
+use arrow_buffer::NullBufferBuilder;
use arrow_data::ArrayData;
use arrow_schema::Field;
use std::any::Any;
@@ -243,7 +243,7 @@ where
.len(len)
.add_buffer(offset_buffer)
.add_child_data(values_data)
- .null_bit_buffer(null_bit_buffer);
+ .nulls(null_bit_buffer);
let array_data = unsafe { array_data_builder.build_unchecked() };
@@ -257,10 +257,7 @@ where
let values_data = values_arr.to_data();
let offset_buffer = Buffer::from_slice_ref(self.offsets_builder.as_slice());
- let null_bit_buffer = self
- .null_buffer_builder
- .as_slice()
- .map(Buffer::from_slice_ref);
+ let nulls = self.null_buffer_builder.finish_cloned();
let field = Arc::new(Field::new(
"item",
values_data.data_type().clone(),
@@ -271,7 +268,7 @@ where
.len(len)
.add_buffer(offset_buffer)
.add_child_data(values_data)
- .null_bit_buffer(null_bit_buffer);
+ .nulls(nulls);
let array_data = unsafe { array_data_builder.build_unchecked() };
diff --git a/arrow-array/src/builder/map_builder.rs b/arrow-array/src/builder/map_builder.rs
index b73e65b11..56b5619ce 100644
--- a/arrow-array/src/builder/map_builder.rs
+++ b/arrow-array/src/builder/map_builder.rs
@@ -15,10 +15,10 @@
// specific language governing permissions and limitations
// under the License.
-use crate::builder::null_buffer_builder::NullBufferBuilder;
use crate::builder::{ArrayBuilder, BufferBuilder};
use crate::{Array, ArrayRef, MapArray, StructArray};
use arrow_buffer::Buffer;
+use arrow_buffer::{NullBuffer, NullBufferBuilder};
use arrow_data::ArrayData;
use arrow_schema::{ArrowError, DataType, Field};
use std::any::Any;
@@ -160,12 +160,8 @@ impl<K: ArrayBuilder, V: ArrayBuilder> MapBuilder<K, V> {
let keys_arr = self.key_builder.finish_cloned();
let values_arr = self.value_builder.finish_cloned();
let offset_buffer = Buffer::from_slice_ref(self.offsets_builder.as_slice());
- let null_bit_buffer = self
- .null_buffer_builder
- .as_slice()
- .map(Buffer::from_slice_ref);
-
- self.finish_helper(keys_arr, values_arr, offset_buffer, null_bit_buffer, len)
+ let nulls = self.null_buffer_builder.finish_cloned();
+ self.finish_helper(keys_arr, values_arr, offset_buffer, nulls, len)
}
fn finish_helper(
@@ -173,7 +169,7 @@ impl<K: ArrayBuilder, V: ArrayBuilder> MapBuilder<K, V> {
keys_arr: Arc<dyn Array>,
values_arr: Arc<dyn Array>,
offset_buffer: Buffer,
- null_bit_buffer: Option<Buffer>,
+ nulls: Option<NullBuffer>,
len: usize,
) -> MapArray {
assert!(
@@ -205,7 +201,7 @@ impl<K: ArrayBuilder, V: ArrayBuilder> MapBuilder<K, V> {
.len(len)
.add_buffer(offset_buffer)
.add_child_data(struct_array.into_data())
- .null_bit_buffer(null_bit_buffer);
+ .nulls(nulls);
let array_data = unsafe { array_data.build_unchecked() };
diff --git a/arrow-array/src/builder/mod.rs b/arrow-array/src/builder/mod.rs
index c4f581fbf..91df8c27c 100644
--- a/arrow-array/src/builder/mod.rs
+++ b/arrow-array/src/builder/mod.rs
@@ -148,8 +148,7 @@
//! }
//! ```
-mod boolean_buffer_builder;
-pub use boolean_buffer_builder::*;
+pub use arrow_buffer::BooleanBufferBuilder;
mod boolean_builder;
pub use boolean_builder::*;
@@ -165,7 +164,6 @@ mod generic_list_builder;
pub use generic_list_builder::*;
mod map_builder;
pub use map_builder::*;
-mod null_buffer_builder;
mod primitive_builder;
pub use primitive_builder::*;
mod primitive_dictionary_builder;
diff --git a/arrow-array/src/builder/primitive_builder.rs b/arrow-array/src/builder/primitive_builder.rs
index 440fb8a4b..f064519e4 100644
--- a/arrow-array/src/builder/primitive_builder.rs
+++ b/arrow-array/src/builder/primitive_builder.rs
@@ -15,10 +15,10 @@
// specific language governing permissions and limitations
// under the License.
-use crate::builder::null_buffer_builder::NullBufferBuilder;
use crate::builder::{ArrayBuilder, BufferBuilder};
use crate::types::*;
use crate::{ArrayRef, ArrowPrimitiveType, PrimitiveArray};
+use arrow_buffer::NullBufferBuilder;
use arrow_buffer::{Buffer, MutableBuffer};
use arrow_data::ArrayData;
use arrow_schema::DataType;
@@ -278,11 +278,11 @@ impl<T: ArrowPrimitiveType> PrimitiveBuilder<T> {
/// Builds the [`PrimitiveArray`] and reset this builder.
pub fn finish(&mut self) -> PrimitiveArray<T> {
let len = self.len();
- let null_bit_buffer = self.null_buffer_builder.finish();
+ let nulls = self.null_buffer_builder.finish();
let builder = ArrayData::builder(self.data_type.clone())
.len(len)
.add_buffer(self.values_builder.finish())
- .null_bit_buffer(null_bit_buffer);
+ .nulls(nulls);
let array_data = unsafe { builder.build_unchecked() };
PrimitiveArray::<T>::from(array_data)
@@ -291,15 +291,12 @@ impl<T: ArrowPrimitiveType> PrimitiveBuilder<T> {
/// Builds the [`PrimitiveArray`] without resetting the builder.
pub fn finish_cloned(&self) -> PrimitiveArray<T> {
let len = self.len();
- let null_bit_buffer = self
- .null_buffer_builder
- .as_slice()
- .map(Buffer::from_slice_ref);
+ let nulls = self.null_buffer_builder.finish_cloned();
let values_buffer = Buffer::from_slice_ref(self.values_builder.as_slice());
let builder = ArrayData::builder(self.data_type.clone())
.len(len)
.add_buffer(values_buffer)
- .null_bit_buffer(null_bit_buffer);
+ .nulls(nulls);
let array_data = unsafe { builder.build_unchecked() };
PrimitiveArray::<T>::from(array_data)
diff --git a/arrow-array/src/builder/struct_builder.rs b/arrow-array/src/builder/struct_builder.rs
index 41ede9c7a..f5e3f2806 100644
--- a/arrow-array/src/builder/struct_builder.rs
+++ b/arrow-array/src/builder/struct_builder.rs
@@ -15,10 +15,9 @@
// specific language governing permissions and limitations
// under the License.
-use crate::builder::null_buffer_builder::NullBufferBuilder;
use crate::builder::*;
use crate::{Array, ArrayRef, StructArray};
-use arrow_buffer::Buffer;
+use arrow_buffer::NullBufferBuilder;
use arrow_data::ArrayData;
use arrow_schema::{DataType, Fields, IntervalUnit, TimeUnit};
use std::any::Any;
@@ -247,12 +246,12 @@ impl StructBuilder {
child_data.push(arr.to_data());
}
let length = self.len();
- let null_bit_buffer = self.null_buffer_builder.finish();
+ let nulls = self.null_buffer_builder.finish();
let builder = ArrayData::builder(DataType::Struct(self.fields.clone()))
.len(length)
.child_data(child_data)
- .null_bit_buffer(null_bit_buffer);
+ .nulls(nulls);
let array_data = unsafe { builder.build_unchecked() };
StructArray::from(array_data)
@@ -268,15 +267,12 @@ impl StructBuilder {
child_data.push(arr.to_data());
}
let length = self.len();
- let null_bit_buffer = self
- .null_buffer_builder
- .as_slice()
- .map(Buffer::from_slice_ref);
+ let nulls = self.null_buffer_builder.finish_cloned();
let builder = ArrayData::builder(DataType::Struct(self.fields.clone()))
.len(length)
.child_data(child_data)
- .null_bit_buffer(null_bit_buffer);
+ .nulls(nulls);
let array_data = unsafe { builder.build_unchecked() };
StructArray::from(array_data)
diff --git a/arrow-array/src/builder/union_builder.rs b/arrow-array/src/builder/union_builder.rs
index 6461a56aa..f74afb2aa 100644
--- a/arrow-array/src/builder/union_builder.rs
+++ b/arrow-array/src/builder/union_builder.rs
@@ -16,9 +16,9 @@
// under the License.
use crate::builder::buffer_builder::{Int32BufferBuilder, Int8BufferBuilder};
-use crate::builder::null_buffer_builder::NullBufferBuilder;
use crate::builder::BufferBuilder;
use crate::{make_array, ArrowPrimitiveType, UnionArray};
+use arrow_buffer::NullBufferBuilder;
use arrow_buffer::{ArrowNativeType, Buffer};
use arrow_data::ArrayDataBuilder;
use arrow_schema::{ArrowError, DataType, Field};
@@ -292,7 +292,7 @@ impl UnionBuilder {
let arr_data_builder = ArrayDataBuilder::new(data_type.clone())
.add_buffer(buffer)
.len(slots)
- .null_bit_buffer(bitmap_builder.finish());
+ .nulls(bitmap_builder.finish());
let arr_data_ref = unsafe { arr_data_builder.build_unchecked() };
let array_ref = make_array(arr_data_ref);
diff --git a/arrow-array/src/builder/boolean_buffer_builder.rs b/arrow-buffer/src/builder/boolean.rs
similarity index 98%
rename from arrow-array/src/builder/boolean_buffer_builder.rs
rename to arrow-buffer/src/builder/boolean.rs
index 1a3473e19..f84cfa79c 100644
--- a/arrow-array/src/builder/boolean_buffer_builder.rs
+++ b/arrow-buffer/src/builder/boolean.rs
@@ -15,8 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-use arrow_buffer::{bit_util, BooleanBuffer, Buffer, MutableBuffer};
-use arrow_data::bit_mask;
+use crate::{bit_mask, bit_util, BooleanBuffer, Buffer, MutableBuffer};
use std::ops::Range;
/// Builder for [`BooleanBuffer`]
@@ -221,6 +220,11 @@ impl BooleanBufferBuilder {
let len = std::mem::replace(&mut self.len, 0);
BooleanBuffer::new(buf.into(), 0, len)
}
+
+ /// Builds the [BooleanBuffer] without resetting the builder.
+ pub fn finish_cloned(&self) -> BooleanBuffer {
+ BooleanBuffer::new(Buffer::from_slice_ref(self.as_slice()), 0, self.len)
+ }
}
impl From<BooleanBufferBuilder> for Buffer {
diff --git a/arrow-buffer/src/util/mod.rs b/arrow-buffer/src/builder/mod.rs
similarity index 90%
copy from arrow-buffer/src/util/mod.rs
copy to arrow-buffer/src/builder/mod.rs
index 0f1825eae..f9d2d0935 100644
--- a/arrow-buffer/src/util/mod.rs
+++ b/arrow-buffer/src/builder/mod.rs
@@ -15,6 +15,9 @@
// specific language governing permissions and limitations
// under the License.
-pub mod bit_chunk_iterator;
-pub mod bit_iterator;
-pub mod bit_util;
+//! Buffer builders
+
+mod boolean;
+pub use boolean::*;
+mod null;
+pub use null::*;
diff --git a/arrow-array/src/builder/null_buffer_builder.rs b/arrow-buffer/src/builder/null.rs
similarity index 91%
rename from arrow-array/src/builder/null_buffer_builder.rs
rename to arrow-buffer/src/builder/null.rs
index f37ce3a74..d805b79f0 100644
--- a/arrow-array/src/builder/null_buffer_builder.rs
+++ b/arrow-buffer/src/builder/null.rs
@@ -15,8 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-use crate::builder::BooleanBufferBuilder;
-use arrow_buffer::{Buffer, MutableBuffer};
+use crate::{BooleanBufferBuilder, MutableBuffer, NullBuffer};
/// Builder for creating the null bit buffer.
/// This builder only materializes the buffer when we append `false`.
@@ -24,7 +23,7 @@ use arrow_buffer::{Buffer, MutableBuffer};
/// `None` when calling [`finish`](#method.finish).
/// This optimization is **very** important for the performance.
#[derive(Debug)]
-pub(super) struct NullBufferBuilder {
+pub struct NullBufferBuilder {
bitmap_builder: Option<BooleanBufferBuilder>,
/// Store the length of the buffer before materializing.
len: usize,
@@ -128,10 +127,15 @@ impl NullBufferBuilder {
/// Builds the null buffer and resets the builder.
/// Returns `None` if the builder only contains `true`s.
- pub fn finish(&mut self) -> Option<Buffer> {
- let buf = self.bitmap_builder.take().map(Into::into);
+ pub fn finish(&mut self) -> Option<NullBuffer> {
self.len = 0;
- buf
+ Some(NullBuffer::new(self.bitmap_builder.take()?.finish()))
+ }
+
+ /// Builds the [NullBuffer] without resetting the builder.
+ pub fn finish_cloned(&self) -> Option<NullBuffer> {
+ let buffer = self.bitmap_builder.as_ref()?.finish_cloned();
+ Some(NullBuffer::new(buffer))
}
/// Returns the inner bitmap builder as slice
@@ -187,7 +191,7 @@ mod tests {
assert_eq!(6, builder.len());
let buf = builder.finish().unwrap();
- assert_eq!(Buffer::from(&[0b110010_u8]), buf);
+ assert_eq!(&[0b110010_u8], buf.validity());
}
#[test]
@@ -199,7 +203,7 @@ mod tests {
assert_eq!(6, builder.len());
let buf = builder.finish().unwrap();
- assert_eq!(Buffer::from(&[0b0_u8]), buf);
+ assert_eq!(&[0b0_u8], buf.validity());
}
#[test]
@@ -228,6 +232,6 @@ mod tests {
builder.append_slice(&[true, true, false, true]);
let buf = builder.finish().unwrap();
- assert_eq!(Buffer::from(&[0b1011_u8]), buf);
+ assert_eq!(&[0b1011_u8], buf.validity());
}
}
diff --git a/arrow-buffer/src/lib.rs b/arrow-buffer/src/lib.rs
index 364e92db2..90b801c4a 100644
--- a/arrow-buffer/src/lib.rs
+++ b/arrow-buffer/src/lib.rs
@@ -21,6 +21,9 @@ pub mod alloc;
pub mod buffer;
pub use buffer::*;
+pub mod builder;
+pub use builder::*;
+
mod bigint;
mod bytes;
mod native;
diff --git a/arrow-data/src/bit_mask.rs b/arrow-buffer/src/util/bit_mask.rs
similarity index 54%
rename from arrow-data/src/bit_mask.rs
rename to arrow-buffer/src/util/bit_mask.rs
index d978f2b74..2af24b782 100644
--- a/arrow-data/src/bit_mask.rs
+++ b/arrow-buffer/src/util/bit_mask.rs
@@ -17,11 +17,8 @@
//! Utils for working with packed bit masks
-use crate::ArrayData;
-use arrow_buffer::bit_chunk_iterator::BitChunks;
-use arrow_buffer::bit_util::{ceil, get_bit, set_bit};
-use arrow_buffer::buffer::buffer_bin_and;
-use arrow_buffer::Buffer;
+use crate::bit_chunk_iterator::BitChunks;
+use crate::bit_util::{ceil, get_bit, set_bit};
/// Sets all bits on `write_data` in the range `[offset_write..offset_write+len]` to be equal to the
/// bits in `data` in the range `[offset_read..offset_read+len]`
@@ -65,45 +62,9 @@ pub fn set_bits(
null_count as usize
}
-/// Combines the null bitmaps of multiple arrays using a bitwise `and` operation.
-///
-/// This function is useful when implementing operations on higher level arrays.
-#[deprecated(note = "Use NullBuffer::union")]
-pub fn combine_option_bitmap(
- arrays: &[&ArrayData],
- len_in_bits: usize,
-) -> Option<Buffer> {
- let (buffer, offset) = arrays
- .iter()
- .map(|array| match array.nulls() {
- Some(n) => (Some(n.buffer().clone()), n.offset()),
- None => (None, 0),
- })
- .reduce(|acc, buffer_and_offset| match (acc, buffer_and_offset) {
- ((None, _), (None, _)) => (None, 0),
- ((Some(buffer), offset), (None, _)) | ((None, _), (Some(buffer), offset)) => {
- (Some(buffer), offset)
- }
- ((Some(buffer_left), offset_left), (Some(buffer_right), offset_right)) => (
- Some(buffer_bin_and(
- &buffer_left,
- offset_left,
- &buffer_right,
- offset_right,
- len_in_bits,
- )),
- 0,
- ),
- })?;
-
- Some(buffer?.bit_slice(offset, len_in_bits))
-}
-
#[cfg(test)]
mod tests {
use super::*;
- use arrow_schema::DataType;
- use std::sync::Arc;
#[test]
fn test_set_bits_aligned() {
@@ -226,112 +187,4 @@ mod tests {
assert_eq!(destination, expected_data);
assert_eq!(result, expected_null_count);
}
-
- fn make_data_with_null_bit_buffer(
- len: usize,
- offset: usize,
- null_bit_buffer: Option<Buffer>,
- ) -> Arc<ArrayData> {
- let buffer = Buffer::from(&vec![11; len + offset]);
-
- Arc::new(
- ArrayData::try_new(
- DataType::UInt8,
- len,
- null_bit_buffer,
- offset,
- vec![buffer],
- vec![],
- )
- .unwrap(),
- )
- }
-
- #[test]
- #[allow(deprecated)]
- fn test_combine_option_bitmap() {
- let none_bitmap = make_data_with_null_bit_buffer(8, 0, None);
- let some_bitmap =
- make_data_with_null_bit_buffer(8, 0, Some(Buffer::from([0b01001010])));
- let inverse_bitmap =
- make_data_with_null_bit_buffer(8, 0, Some(Buffer::from([0b10110101])));
- let some_other_bitmap =
- make_data_with_null_bit_buffer(8, 0, Some(Buffer::from([0b11010111])));
- assert_eq!(None, combine_option_bitmap(&[], 8));
- assert_eq!(
- Some(Buffer::from([0b01001010])),
- combine_option_bitmap(&[&some_bitmap], 8)
- );
- assert_eq!(
- None,
- combine_option_bitmap(&[&none_bitmap, &none_bitmap], 8)
- );
- assert_eq!(
- Some(Buffer::from([0b01001010])),
- combine_option_bitmap(&[&some_bitmap, &none_bitmap], 8)
- );
- assert_eq!(
- Some(Buffer::from([0b11010111])),
- combine_option_bitmap(&[&none_bitmap, &some_other_bitmap], 8)
- );
- assert_eq!(
- Some(Buffer::from([0b01001010])),
- combine_option_bitmap(&[&some_bitmap, &some_bitmap], 8,)
- );
- assert_eq!(
- Some(Buffer::from([0b0])),
- combine_option_bitmap(&[&some_bitmap, &inverse_bitmap], 8,)
- );
- assert_eq!(
- Some(Buffer::from([0b01000010])),
- combine_option_bitmap(&[&some_bitmap, &some_other_bitmap, &none_bitmap], 8,)
- );
- assert_eq!(
- Some(Buffer::from([0b00001001])),
- combine_option_bitmap(
- &[
- &some_bitmap.slice(3, 5),
- &inverse_bitmap.slice(2, 5),
- &some_other_bitmap.slice(1, 5)
- ],
- 5,
- )
- );
- }
-
- #[test]
- #[allow(deprecated)]
- fn test_combine_option_bitmap_with_offsets() {
- let none_bitmap = make_data_with_null_bit_buffer(8, 0, None);
- let bitmap0 =
- make_data_with_null_bit_buffer(8, 0, Some(Buffer::from([0b10101010])));
- let bitmap1 =
- make_data_with_null_bit_buffer(8, 1, Some(Buffer::from([0b01010100, 0b1])));
- let bitmap2 =
- make_data_with_null_bit_buffer(8, 2, Some(Buffer::from([0b10101000, 0b10])));
- assert_eq!(
- Some(Buffer::from([0b10101010])),
- combine_option_bitmap(&[&bitmap1], 8)
- );
- assert_eq!(
- Some(Buffer::from([0b10101010])),
- combine_option_bitmap(&[&bitmap2], 8)
- );
- assert_eq!(
- Some(Buffer::from([0b10101010])),
- combine_option_bitmap(&[&bitmap1, &none_bitmap], 8)
- );
- assert_eq!(
- Some(Buffer::from([0b10101010])),
- combine_option_bitmap(&[&none_bitmap, &bitmap2], 8)
- );
- assert_eq!(
- Some(Buffer::from([0b10101010])),
- combine_option_bitmap(&[&bitmap0, &bitmap1], 8)
- );
- assert_eq!(
- Some(Buffer::from([0b10101010])),
- combine_option_bitmap(&[&bitmap1, &bitmap2], 8)
- );
- }
}
diff --git a/arrow-buffer/src/util/mod.rs b/arrow-buffer/src/util/mod.rs
index 0f1825eae..9023fe4a0 100644
--- a/arrow-buffer/src/util/mod.rs
+++ b/arrow-buffer/src/util/mod.rs
@@ -17,4 +17,5 @@
pub mod bit_chunk_iterator;
pub mod bit_iterator;
+pub mod bit_mask;
pub mod bit_util;
diff --git a/arrow-data/src/lib.rs b/arrow-data/src/lib.rs
index 15f6acd2c..b864b7860 100644
--- a/arrow-data/src/lib.rs
+++ b/arrow-data/src/lib.rs
@@ -23,8 +23,7 @@ pub use data::*;
mod equal;
pub mod transform;
-pub use arrow_buffer::bit_iterator;
-pub mod bit_mask;
+pub use arrow_buffer::{bit_iterator, bit_mask};
pub mod decimal;
#[cfg(feature = "ffi")]