You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by pa...@apache.org on 2022/08/19 13:26:43 UTC
[arrow-nanoarrow] branch main updated: Pass the ArrowBufferAllocator by value and implement a custom deallocator (#24)
This is an automated email from the ASF dual-hosted git repository.
paleolimbot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-nanoarrow.git
The following commit(s) were added to refs/heads/main by this push:
new a78c809 Pass the ArrowBufferAllocator by value and implement a custom deallocator (#24)
a78c809 is described below
commit a78c809e48606d00c2de036691e6016da09c614d
Author: Dewey Dunnington <de...@fishandwhistle.net>
AuthorDate: Fri Aug 19 10:26:40 2022 -0300
Pass the ArrowBufferAllocator by value and implement a custom deallocator (#24)
* pass allocator by value
* implement a deallocating allocator
* document the 'why' of the deallocator
* remove the 'allocate' method from the buffer allocator (it wasn't used)
---
src/nanoarrow/allocator.c | 29 +++++++++++++++++++---
src/nanoarrow/allocator_test.cc | 55 ++++++++++++++++++++++++++---------------
src/nanoarrow/buffer_inline.h | 10 ++++----
src/nanoarrow/buffer_test.cc | 17 +++++--------
src/nanoarrow/nanoarrow.h | 18 +++++++++++---
src/nanoarrow/typedefs_inline.h | 5 +---
6 files changed, 86 insertions(+), 48 deletions(-)
diff --git a/src/nanoarrow/allocator.c b/src/nanoarrow/allocator.c
index 8495037..a016bce 100644
--- a/src/nanoarrow/allocator.c
+++ b/src/nanoarrow/allocator.c
@@ -43,9 +43,30 @@ static void ArrowBufferAllocatorMallocFree(struct ArrowBufferAllocator* allocato
}
static struct ArrowBufferAllocator ArrowBufferAllocatorMalloc = {
- &ArrowBufferAllocatorMallocAllocate, &ArrowBufferAllocatorMallocReallocate,
- &ArrowBufferAllocatorMallocFree, NULL};
+ &ArrowBufferAllocatorMallocReallocate, &ArrowBufferAllocatorMallocFree, NULL};
-struct ArrowBufferAllocator* ArrowBufferAllocatorDefault() {
- return &ArrowBufferAllocatorMalloc;
+struct ArrowBufferAllocator ArrowBufferAllocatorDefault() {
+ return ArrowBufferAllocatorMalloc;
+}
+
+static uint8_t* ArrowBufferAllocatorNeverAllocate(struct ArrowBufferAllocator* allocator,
+ int64_t size) {
+ return NULL;
+}
+
+static uint8_t* ArrowBufferAllocatorNeverReallocate(
+ struct ArrowBufferAllocator* allocator, uint8_t* ptr, int64_t old_size,
+ int64_t new_size) {
+ return NULL;
+}
+
+struct ArrowBufferAllocator ArrowBufferDeallocator(
+ void (*custom_free)(struct ArrowBufferAllocator* allocator, uint8_t* ptr,
+ int64_t size),
+ void* private_data) {
+ struct ArrowBufferAllocator allocator;
+ allocator.reallocate = &ArrowBufferAllocatorNeverReallocate;
+ allocator.free = custom_free;
+ allocator.private_data = private_data;
+ return allocator;
}
diff --git a/src/nanoarrow/allocator_test.cc b/src/nanoarrow/allocator_test.cc
index 70aaf56..6ea2048 100644
--- a/src/nanoarrow/allocator_test.cc
+++ b/src/nanoarrow/allocator_test.cc
@@ -25,16 +25,6 @@
using namespace arrow;
-static uint8_t* MemoryPoolAllocate(struct ArrowBufferAllocator* allocator, int64_t size) {
- MemoryPool* pool = reinterpret_cast<MemoryPool*>(allocator->private_data);
- uint8_t* out;
- if (pool->Allocate(size, &out).ok()) {
- return out;
- } else {
- return nullptr;
- }
-}
-
static uint8_t* MemoryPoolReallocate(struct ArrowBufferAllocator* allocator, uint8_t* ptr,
int64_t old_size, int64_t new_size) {
MemoryPool* pool = reinterpret_cast<MemoryPool*>(allocator->private_data);
@@ -54,39 +44,64 @@ static void MemoryPoolFree(struct ArrowBufferAllocator* allocator, uint8_t* ptr,
static void MemoryPoolAllocatorInit(MemoryPool* pool,
struct ArrowBufferAllocator* allocator) {
- allocator->allocate = &MemoryPoolAllocate;
allocator->reallocate = &MemoryPoolReallocate;
allocator->free = &MemoryPoolFree;
allocator->private_data = pool;
}
TEST(AllocatorTest, AllocatorTestDefault) {
- struct ArrowBufferAllocator* allocator = ArrowBufferAllocatorDefault();
+ struct ArrowBufferAllocator allocator = ArrowBufferAllocatorDefault();
- uint8_t* buffer = allocator->allocate(allocator, 10);
+ uint8_t* buffer = allocator.reallocate(&allocator, nullptr, 0, 10);
const char* test_str = "abcdefg";
memcpy(buffer, test_str, strlen(test_str) + 1);
- buffer = allocator->reallocate(allocator, buffer, 10, 100);
+ buffer = allocator.reallocate(&allocator, buffer, 10, 100);
EXPECT_STREQ(reinterpret_cast<const char*>(buffer), test_str);
- allocator->free(allocator, buffer, 100);
+ allocator.free(&allocator, buffer, 100);
- buffer = allocator->allocate(allocator, std::numeric_limits<int64_t>::max());
+ buffer =
+ allocator.reallocate(&allocator, nullptr, 0, std::numeric_limits<int64_t>::max());
EXPECT_EQ(buffer, nullptr);
buffer =
- allocator->reallocate(allocator, buffer, 0, std::numeric_limits<int64_t>::max());
+ allocator.reallocate(&allocator, buffer, 0, std::numeric_limits<int64_t>::max());
EXPECT_EQ(buffer, nullptr);
}
+// In a non-trivial test this struct could hold a reference to an object
+// that keeps the buffer from being garbage collected (e.g., an SEXP in R)
+struct CustomFreeData {
+ void* pointer_proxy;
+};
+
+static void CustomFree(struct ArrowBufferAllocator* allocator, uint8_t* ptr,
+ int64_t size) {
+ auto data = reinterpret_cast<struct CustomFreeData*>(allocator->private_data);
+ ArrowFree(data->pointer_proxy);
+ data->pointer_proxy = nullptr;
+}
+
+TEST(AllocatorTest, AllocatorTestDeallocator) {
+ struct CustomFreeData data;
+ data.pointer_proxy = reinterpret_cast<uint8_t*>(ArrowMalloc(12));
+
+ struct ArrowBufferAllocator deallocator = ArrowBufferDeallocator(&CustomFree, &data);
+
+ EXPECT_EQ(deallocator.reallocate(&deallocator, nullptr, 0, 12), nullptr);
+ EXPECT_EQ(deallocator.reallocate(&deallocator, nullptr, 0, 12), nullptr);
+ deallocator.free(&deallocator, nullptr, 12);
+ EXPECT_EQ(data.pointer_proxy, nullptr);
+}
+
TEST(AllocatorTest, AllocatorTestMemoryPool) {
struct ArrowBufferAllocator arrow_allocator;
MemoryPoolAllocatorInit(system_memory_pool(), &arrow_allocator);
int64_t allocated0 = system_memory_pool()->bytes_allocated();
- uint8_t* buffer = arrow_allocator.allocate(&arrow_allocator, 10);
+ uint8_t* buffer = arrow_allocator.reallocate(&arrow_allocator, nullptr, 0, 10);
EXPECT_EQ(system_memory_pool()->bytes_allocated() - allocated0, 10);
memset(buffer, 0, 10);
@@ -100,8 +115,8 @@ TEST(AllocatorTest, AllocatorTestMemoryPool) {
arrow_allocator.free(&arrow_allocator, buffer, 100);
EXPECT_EQ(system_memory_pool()->bytes_allocated(), allocated0);
- buffer =
- arrow_allocator.allocate(&arrow_allocator, std::numeric_limits<int64_t>::max());
+ buffer = arrow_allocator.reallocate(&arrow_allocator, nullptr, 0,
+ std::numeric_limits<int64_t>::max());
EXPECT_EQ(buffer, nullptr);
buffer = arrow_allocator.reallocate(&arrow_allocator, buffer, 0,
diff --git a/src/nanoarrow/buffer_inline.h b/src/nanoarrow/buffer_inline.h
index f7ef2d8..1b6c43b 100644
--- a/src/nanoarrow/buffer_inline.h
+++ b/src/nanoarrow/buffer_inline.h
@@ -46,7 +46,7 @@ static inline void ArrowBufferInit(struct ArrowBuffer* buffer) {
}
static inline ArrowErrorCode ArrowBufferSetAllocator(
- struct ArrowBuffer* buffer, struct ArrowBufferAllocator* allocator) {
+ struct ArrowBuffer* buffer, struct ArrowBufferAllocator allocator) {
if (buffer->data == NULL) {
buffer->allocator = allocator;
return NANOARROW_OK;
@@ -57,8 +57,8 @@ static inline ArrowErrorCode ArrowBufferSetAllocator(
static inline void ArrowBufferReset(struct ArrowBuffer* buffer) {
if (buffer->data != NULL) {
- buffer->allocator->free(buffer->allocator, (uint8_t*)buffer->data,
- buffer->capacity_bytes);
+ buffer->allocator.free(&buffer->allocator, (uint8_t*)buffer->data,
+ buffer->capacity_bytes);
buffer->data = NULL;
}
@@ -81,8 +81,8 @@ static inline ArrowErrorCode ArrowBufferResize(struct ArrowBuffer* buffer,
}
if (new_capacity_bytes > buffer->capacity_bytes || shrink_to_fit) {
- buffer->data = buffer->allocator->reallocate(
- buffer->allocator, buffer->data, buffer->capacity_bytes, new_capacity_bytes);
+ buffer->data = buffer->allocator.reallocate(
+ &buffer->allocator, buffer->data, buffer->capacity_bytes, new_capacity_bytes);
if (buffer->data == NULL && new_capacity_bytes > 0) {
buffer->capacity_bytes = 0;
buffer->size_bytes = 0;
diff --git a/src/nanoarrow/buffer_test.cc b/src/nanoarrow/buffer_test.cc
index 7bb4496..8381d75 100644
--- a/src/nanoarrow/buffer_test.cc
+++ b/src/nanoarrow/buffer_test.cc
@@ -26,15 +26,10 @@
// This test allocator guarantees that allocator->reallocate will return
// a new pointer so that we can test when reallocations happen whilst
// building buffers.
-static uint8_t* TestAllocatorAllocate(struct ArrowBufferAllocator* allocator,
- int64_t size) {
- return reinterpret_cast<uint8_t*>(malloc(size));
-}
-
static uint8_t* TestAllocatorReallocate(struct ArrowBufferAllocator* allocator,
uint8_t* ptr, int64_t old_size,
int64_t new_size) {
- uint8_t* new_ptr = TestAllocatorAllocate(allocator, new_size);
+ uint8_t* new_ptr = reinterpret_cast<uint8_t*>(malloc(new_size));
int64_t copy_size = std::min<int64_t>(old_size, new_size);
if (new_ptr != nullptr && copy_size > 0) {
@@ -53,15 +48,15 @@ static void TestAllocatorFree(struct ArrowBufferAllocator* allocator, uint8_t* p
free(ptr);
}
-static struct ArrowBufferAllocator test_allocator = {
- &TestAllocatorAllocate, &TestAllocatorReallocate, &TestAllocatorFree, nullptr};
+static struct ArrowBufferAllocator test_allocator = {&TestAllocatorReallocate,
+ &TestAllocatorFree, nullptr};
TEST(BufferTest, BufferTestBasic) {
struct ArrowBuffer buffer;
// Init
ArrowBufferInit(&buffer);
- ASSERT_EQ(ArrowBufferSetAllocator(&buffer, &test_allocator), NANOARROW_OK);
+ ASSERT_EQ(ArrowBufferSetAllocator(&buffer, test_allocator), NANOARROW_OK);
EXPECT_EQ(buffer.data, nullptr);
EXPECT_EQ(buffer.capacity_bytes, 0);
EXPECT_EQ(buffer.size_bytes, 0);
@@ -109,7 +104,7 @@ TEST(BufferTest, BufferTestMove) {
struct ArrowBuffer buffer;
ArrowBufferInit(&buffer);
- ASSERT_EQ(ArrowBufferSetAllocator(&buffer, &test_allocator), NANOARROW_OK);
+ ASSERT_EQ(ArrowBufferSetAllocator(&buffer, test_allocator), NANOARROW_OK);
ASSERT_EQ(ArrowBufferAppend(&buffer, "1234567", 7), NANOARROW_OK);
EXPECT_EQ(buffer.size_bytes, 7);
EXPECT_EQ(buffer.capacity_bytes, 7);
@@ -152,7 +147,7 @@ TEST(BufferTest, BufferTestResize0) {
struct ArrowBuffer buffer;
ArrowBufferInit(&buffer);
- ASSERT_EQ(ArrowBufferSetAllocator(&buffer, &test_allocator), NANOARROW_OK);
+ ASSERT_EQ(ArrowBufferSetAllocator(&buffer, test_allocator), NANOARROW_OK);
ASSERT_EQ(ArrowBufferAppend(&buffer, "1234567", 7), NANOARROW_OK);
EXPECT_EQ(buffer.size_bytes, 7);
EXPECT_EQ(buffer.capacity_bytes, 7);
diff --git a/src/nanoarrow/nanoarrow.h b/src/nanoarrow/nanoarrow.h
index d3957d8..38b0efc 100644
--- a/src/nanoarrow/nanoarrow.h
+++ b/src/nanoarrow/nanoarrow.h
@@ -61,7 +61,18 @@ void ArrowFree(void* ptr);
///
/// The default allocator uses ArrowMalloc(), ArrowRealloc(), and
/// ArrowFree().
-struct ArrowBufferAllocator* ArrowBufferAllocatorDefault();
+struct ArrowBufferAllocator ArrowBufferAllocatorDefault();
+
+/// \brief Create a custom deallocator
+///
+/// Creates a buffer allocator with only a free method that can be used to
+/// attach a custom deallocator to an ArrowBuffer. This may be used to
+/// avoid copying an existing buffer that was not allocated using the
+/// infrastructure provided here (e.g., by an R or Python object).
+struct ArrowBufferAllocator ArrowBufferDeallocator(
+ void (*custom_free)(struct ArrowBufferAllocator* allocator, uint8_t* ptr,
+ int64_t size),
+ void* private_data);
/// }@
@@ -343,7 +354,7 @@ static inline void ArrowBufferInit(struct ArrowBuffer* buffer);
///
/// Returns EINVAL if the buffer has already been allocated.
static inline ArrowErrorCode ArrowBufferSetAllocator(
- struct ArrowBuffer* buffer, struct ArrowBufferAllocator* allocator);
+ struct ArrowBuffer* buffer, struct ArrowBufferAllocator allocator);
/// \brief Reset an ArrowBuffer
///
@@ -673,8 +684,7 @@ void ArrowArrayViewSetLength(struct ArrowArrayView* array_view, int64_t length);
/// \brief Set buffer sizes and data pointers from an ArrowArray
ArrowErrorCode ArrowArrayViewSetArray(struct ArrowArrayView* array_view,
- struct ArrowArray* array,
- struct ArrowError* error);
+ struct ArrowArray* array, struct ArrowError* error);
/// \brief Reset the contents of an ArrowArrayView and frees resources
void ArrowArrayViewReset(struct ArrowArrayView* array_view);
diff --git a/src/nanoarrow/typedefs_inline.h b/src/nanoarrow/typedefs_inline.h
index 0959be5..47e2892 100644
--- a/src/nanoarrow/typedefs_inline.h
+++ b/src/nanoarrow/typedefs_inline.h
@@ -233,9 +233,6 @@ struct ArrowBufferView {
/// to customize allocation and deallocation of buffers when constructing
/// an ArrowArray.
struct ArrowBufferAllocator {
- /// \brief Allocate a buffer or return NULL if it cannot be allocated
- uint8_t* (*allocate)(struct ArrowBufferAllocator* allocator, int64_t size);
-
/// \brief Reallocate a buffer or return NULL if it cannot be reallocated
uint8_t* (*reallocate)(struct ArrowBufferAllocator* allocator, uint8_t* ptr,
int64_t old_size, int64_t new_size);
@@ -261,7 +258,7 @@ struct ArrowBuffer {
int64_t capacity_bytes;
/// \brief The allocator that will be used to reallocate and/or free the buffer
- struct ArrowBufferAllocator* allocator;
+ struct ArrowBufferAllocator allocator;
};
/// \brief An owning mutable view of a bitmap