You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by pa...@apache.org on 2022/08/19 13:26:43 UTC

[arrow-nanoarrow] branch main updated: Pass the ArrowBufferAllocator by value and implement a custom deallocator (#24)

This is an automated email from the ASF dual-hosted git repository.

paleolimbot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-nanoarrow.git


The following commit(s) were added to refs/heads/main by this push:
     new a78c809  Pass the ArrowBufferAllocator by value and implement a custom deallocator (#24)
a78c809 is described below

commit a78c809e48606d00c2de036691e6016da09c614d
Author: Dewey Dunnington <de...@fishandwhistle.net>
AuthorDate: Fri Aug 19 10:26:40 2022 -0300

    Pass the ArrowBufferAllocator by value and implement a custom deallocator (#24)
    
    * pass allocator by value
    
    * implement a deallocating allocator
    
    * document the 'why' of the deallocator
    
    * remove the 'allocate' method from the buffer allocator (it wasn't used)
---
 src/nanoarrow/allocator.c       | 29 +++++++++++++++++++---
 src/nanoarrow/allocator_test.cc | 55 ++++++++++++++++++++++++++---------------
 src/nanoarrow/buffer_inline.h   | 10 ++++----
 src/nanoarrow/buffer_test.cc    | 17 +++++--------
 src/nanoarrow/nanoarrow.h       | 18 +++++++++++---
 src/nanoarrow/typedefs_inline.h |  5 +---
 6 files changed, 86 insertions(+), 48 deletions(-)

diff --git a/src/nanoarrow/allocator.c b/src/nanoarrow/allocator.c
index 8495037..a016bce 100644
--- a/src/nanoarrow/allocator.c
+++ b/src/nanoarrow/allocator.c
@@ -43,9 +43,30 @@ static void ArrowBufferAllocatorMallocFree(struct ArrowBufferAllocator* allocato
 }
 
 static struct ArrowBufferAllocator ArrowBufferAllocatorMalloc = {
-    &ArrowBufferAllocatorMallocAllocate, &ArrowBufferAllocatorMallocReallocate,
-    &ArrowBufferAllocatorMallocFree, NULL};
+    &ArrowBufferAllocatorMallocReallocate, &ArrowBufferAllocatorMallocFree, NULL};
 
-struct ArrowBufferAllocator* ArrowBufferAllocatorDefault() {
-  return &ArrowBufferAllocatorMalloc;
+struct ArrowBufferAllocator ArrowBufferAllocatorDefault() {
+  return ArrowBufferAllocatorMalloc;
+}
+
+static uint8_t* ArrowBufferAllocatorNeverAllocate(struct ArrowBufferAllocator* allocator,
+                                                  int64_t size) {
+  return NULL;
+}
+
+static uint8_t* ArrowBufferAllocatorNeverReallocate(
+    struct ArrowBufferAllocator* allocator, uint8_t* ptr, int64_t old_size,
+    int64_t new_size) {
+  return NULL;
+}
+
+struct ArrowBufferAllocator ArrowBufferDeallocator(
+    void (*custom_free)(struct ArrowBufferAllocator* allocator, uint8_t* ptr,
+                        int64_t size),
+    void* private_data) {
+  struct ArrowBufferAllocator allocator;
+  allocator.reallocate = &ArrowBufferAllocatorNeverReallocate;
+  allocator.free = custom_free;
+  allocator.private_data = private_data;
+  return allocator;
 }
diff --git a/src/nanoarrow/allocator_test.cc b/src/nanoarrow/allocator_test.cc
index 70aaf56..6ea2048 100644
--- a/src/nanoarrow/allocator_test.cc
+++ b/src/nanoarrow/allocator_test.cc
@@ -25,16 +25,6 @@
 
 using namespace arrow;
 
-static uint8_t* MemoryPoolAllocate(struct ArrowBufferAllocator* allocator, int64_t size) {
-  MemoryPool* pool = reinterpret_cast<MemoryPool*>(allocator->private_data);
-  uint8_t* out;
-  if (pool->Allocate(size, &out).ok()) {
-    return out;
-  } else {
-    return nullptr;
-  }
-}
-
 static uint8_t* MemoryPoolReallocate(struct ArrowBufferAllocator* allocator, uint8_t* ptr,
                                      int64_t old_size, int64_t new_size) {
   MemoryPool* pool = reinterpret_cast<MemoryPool*>(allocator->private_data);
@@ -54,39 +44,64 @@ static void MemoryPoolFree(struct ArrowBufferAllocator* allocator, uint8_t* ptr,
 
 static void MemoryPoolAllocatorInit(MemoryPool* pool,
                                     struct ArrowBufferAllocator* allocator) {
-  allocator->allocate = &MemoryPoolAllocate;
   allocator->reallocate = &MemoryPoolReallocate;
   allocator->free = &MemoryPoolFree;
   allocator->private_data = pool;
 }
 
 TEST(AllocatorTest, AllocatorTestDefault) {
-  struct ArrowBufferAllocator* allocator = ArrowBufferAllocatorDefault();
+  struct ArrowBufferAllocator allocator = ArrowBufferAllocatorDefault();
 
-  uint8_t* buffer = allocator->allocate(allocator, 10);
+  uint8_t* buffer = allocator.reallocate(&allocator, nullptr, 0, 10);
   const char* test_str = "abcdefg";
   memcpy(buffer, test_str, strlen(test_str) + 1);
 
-  buffer = allocator->reallocate(allocator, buffer, 10, 100);
+  buffer = allocator.reallocate(&allocator, buffer, 10, 100);
   EXPECT_STREQ(reinterpret_cast<const char*>(buffer), test_str);
 
-  allocator->free(allocator, buffer, 100);
+  allocator.free(&allocator, buffer, 100);
 
-  buffer = allocator->allocate(allocator, std::numeric_limits<int64_t>::max());
+  buffer =
+      allocator.reallocate(&allocator, nullptr, 0, std::numeric_limits<int64_t>::max());
   EXPECT_EQ(buffer, nullptr);
 
   buffer =
-      allocator->reallocate(allocator, buffer, 0, std::numeric_limits<int64_t>::max());
+      allocator.reallocate(&allocator, buffer, 0, std::numeric_limits<int64_t>::max());
   EXPECT_EQ(buffer, nullptr);
 }
 
+// In a non-trivial test this struct could hold a reference to an object
+// that keeps the buffer from being garbage collected (e.g., an SEXP in R)
+struct CustomFreeData {
+  void* pointer_proxy;
+};
+
+static void CustomFree(struct ArrowBufferAllocator* allocator, uint8_t* ptr,
+                       int64_t size) {
+  auto data = reinterpret_cast<struct CustomFreeData*>(allocator->private_data);
+  ArrowFree(data->pointer_proxy);
+  data->pointer_proxy = nullptr;
+}
+
+TEST(AllocatorTest, AllocatorTestDeallocator) {
+  struct CustomFreeData data;
+  data.pointer_proxy = reinterpret_cast<uint8_t*>(ArrowMalloc(12));
+
+  struct ArrowBufferAllocator deallocator = ArrowBufferDeallocator(&CustomFree, &data);
+
+  EXPECT_EQ(deallocator.reallocate(&deallocator, nullptr, 0, 12), nullptr);
+  EXPECT_EQ(deallocator.reallocate(&deallocator, nullptr, 0, 12), nullptr);
+  deallocator.free(&deallocator, nullptr, 12);
+  EXPECT_EQ(data.pointer_proxy, nullptr);
+}
+
 TEST(AllocatorTest, AllocatorTestMemoryPool) {
   struct ArrowBufferAllocator arrow_allocator;
   MemoryPoolAllocatorInit(system_memory_pool(), &arrow_allocator);
 
   int64_t allocated0 = system_memory_pool()->bytes_allocated();
 
-  uint8_t* buffer = arrow_allocator.allocate(&arrow_allocator, 10);
+  uint8_t* buffer = arrow_allocator.reallocate(&arrow_allocator, nullptr, 0, 10);
   EXPECT_EQ(system_memory_pool()->bytes_allocated() - allocated0, 10);
   memset(buffer, 0, 10);
 
@@ -100,8 +115,8 @@ TEST(AllocatorTest, AllocatorTestMemoryPool) {
   arrow_allocator.free(&arrow_allocator, buffer, 100);
   EXPECT_EQ(system_memory_pool()->bytes_allocated(), allocated0);
 
-  buffer =
-      arrow_allocator.allocate(&arrow_allocator, std::numeric_limits<int64_t>::max());
+  buffer = arrow_allocator.reallocate(&arrow_allocator, nullptr, 0,
+                                      std::numeric_limits<int64_t>::max());
   EXPECT_EQ(buffer, nullptr);
 
   buffer = arrow_allocator.reallocate(&arrow_allocator, buffer, 0,
diff --git a/src/nanoarrow/buffer_inline.h b/src/nanoarrow/buffer_inline.h
index f7ef2d8..1b6c43b 100644
--- a/src/nanoarrow/buffer_inline.h
+++ b/src/nanoarrow/buffer_inline.h
@@ -46,7 +46,7 @@ static inline void ArrowBufferInit(struct ArrowBuffer* buffer) {
 }
 
 static inline ArrowErrorCode ArrowBufferSetAllocator(
-    struct ArrowBuffer* buffer, struct ArrowBufferAllocator* allocator) {
+    struct ArrowBuffer* buffer, struct ArrowBufferAllocator allocator) {
   if (buffer->data == NULL) {
     buffer->allocator = allocator;
     return NANOARROW_OK;
@@ -57,8 +57,8 @@ static inline ArrowErrorCode ArrowBufferSetAllocator(
 
 static inline void ArrowBufferReset(struct ArrowBuffer* buffer) {
   if (buffer->data != NULL) {
-    buffer->allocator->free(buffer->allocator, (uint8_t*)buffer->data,
-                            buffer->capacity_bytes);
+    buffer->allocator.free(&buffer->allocator, (uint8_t*)buffer->data,
+                           buffer->capacity_bytes);
     buffer->data = NULL;
   }
 
@@ -81,8 +81,8 @@ static inline ArrowErrorCode ArrowBufferResize(struct ArrowBuffer* buffer,
   }
 
   if (new_capacity_bytes > buffer->capacity_bytes || shrink_to_fit) {
-    buffer->data = buffer->allocator->reallocate(
-        buffer->allocator, buffer->data, buffer->capacity_bytes, new_capacity_bytes);
+    buffer->data = buffer->allocator.reallocate(
+        &buffer->allocator, buffer->data, buffer->capacity_bytes, new_capacity_bytes);
     if (buffer->data == NULL && new_capacity_bytes > 0) {
       buffer->capacity_bytes = 0;
       buffer->size_bytes = 0;
diff --git a/src/nanoarrow/buffer_test.cc b/src/nanoarrow/buffer_test.cc
index 7bb4496..8381d75 100644
--- a/src/nanoarrow/buffer_test.cc
+++ b/src/nanoarrow/buffer_test.cc
@@ -26,15 +26,10 @@
 // This test allocator guarantees that allocator->reallocate will return
 // a new pointer so that we can test when reallocations happen whilst
 // building buffers.
-static uint8_t* TestAllocatorAllocate(struct ArrowBufferAllocator* allocator,
-                                      int64_t size) {
-  return reinterpret_cast<uint8_t*>(malloc(size));
-}
-
 static uint8_t* TestAllocatorReallocate(struct ArrowBufferAllocator* allocator,
                                         uint8_t* ptr, int64_t old_size,
                                         int64_t new_size) {
-  uint8_t* new_ptr = TestAllocatorAllocate(allocator, new_size);
+  uint8_t* new_ptr = reinterpret_cast<uint8_t*>(malloc(new_size));
 
   int64_t copy_size = std::min<int64_t>(old_size, new_size);
   if (new_ptr != nullptr && copy_size > 0) {
@@ -53,15 +48,15 @@ static void TestAllocatorFree(struct ArrowBufferAllocator* allocator, uint8_t* p
   free(ptr);
 }
 
-static struct ArrowBufferAllocator test_allocator = {
-    &TestAllocatorAllocate, &TestAllocatorReallocate, &TestAllocatorFree, nullptr};
+static struct ArrowBufferAllocator test_allocator = {&TestAllocatorReallocate,
+                                                     &TestAllocatorFree, nullptr};
 
 TEST(BufferTest, BufferTestBasic) {
   struct ArrowBuffer buffer;
 
   // Init
   ArrowBufferInit(&buffer);
-  ASSERT_EQ(ArrowBufferSetAllocator(&buffer, &test_allocator), NANOARROW_OK);
+  ASSERT_EQ(ArrowBufferSetAllocator(&buffer, test_allocator), NANOARROW_OK);
   EXPECT_EQ(buffer.data, nullptr);
   EXPECT_EQ(buffer.capacity_bytes, 0);
   EXPECT_EQ(buffer.size_bytes, 0);
@@ -109,7 +104,7 @@ TEST(BufferTest, BufferTestMove) {
   struct ArrowBuffer buffer;
 
   ArrowBufferInit(&buffer);
-  ASSERT_EQ(ArrowBufferSetAllocator(&buffer, &test_allocator), NANOARROW_OK);
+  ASSERT_EQ(ArrowBufferSetAllocator(&buffer, test_allocator), NANOARROW_OK);
   ASSERT_EQ(ArrowBufferAppend(&buffer, "1234567", 7), NANOARROW_OK);
   EXPECT_EQ(buffer.size_bytes, 7);
   EXPECT_EQ(buffer.capacity_bytes, 7);
@@ -152,7 +147,7 @@ TEST(BufferTest, BufferTestResize0) {
   struct ArrowBuffer buffer;
 
   ArrowBufferInit(&buffer);
-  ASSERT_EQ(ArrowBufferSetAllocator(&buffer, &test_allocator), NANOARROW_OK);
+  ASSERT_EQ(ArrowBufferSetAllocator(&buffer, test_allocator), NANOARROW_OK);
   ASSERT_EQ(ArrowBufferAppend(&buffer, "1234567", 7), NANOARROW_OK);
   EXPECT_EQ(buffer.size_bytes, 7);
   EXPECT_EQ(buffer.capacity_bytes, 7);
diff --git a/src/nanoarrow/nanoarrow.h b/src/nanoarrow/nanoarrow.h
index d3957d8..38b0efc 100644
--- a/src/nanoarrow/nanoarrow.h
+++ b/src/nanoarrow/nanoarrow.h
@@ -61,7 +61,18 @@ void ArrowFree(void* ptr);
 ///
 /// The default allocator uses ArrowMalloc(), ArrowRealloc(), and
 /// ArrowFree().
-struct ArrowBufferAllocator* ArrowBufferAllocatorDefault();
+struct ArrowBufferAllocator ArrowBufferAllocatorDefault();
+
+/// \brief Create a custom deallocator
+///
+/// Creates a buffer allocator with only a free method that can be used to
+/// attach a custom deallocator to an ArrowBuffer. This may be used to
+/// avoid copying an existing buffer that was not allocated using the
+/// infrastructure provided here (e.g., by an R or Python object).
+struct ArrowBufferAllocator ArrowBufferDeallocator(
+    void (*custom_free)(struct ArrowBufferAllocator* allocator, uint8_t* ptr,
+                        int64_t size),
+    void* private_data);
 
 /// }@
 
@@ -343,7 +354,7 @@ static inline void ArrowBufferInit(struct ArrowBuffer* buffer);
 ///
 /// Returns EINVAL if the buffer has already been allocated.
 static inline ArrowErrorCode ArrowBufferSetAllocator(
-    struct ArrowBuffer* buffer, struct ArrowBufferAllocator* allocator);
+    struct ArrowBuffer* buffer, struct ArrowBufferAllocator allocator);
 
 /// \brief Reset an ArrowBuffer
 ///
@@ -673,8 +684,7 @@ void ArrowArrayViewSetLength(struct ArrowArrayView* array_view, int64_t length);
 
 /// \brief Set buffer sizes and data pointers from an ArrowArray
 ArrowErrorCode ArrowArrayViewSetArray(struct ArrowArrayView* array_view,
-                                      struct ArrowArray* array,
-                                      struct ArrowError* error);
+                                      struct ArrowArray* array, struct ArrowError* error);
 
 /// \brief Reset the contents of an ArrowArrayView and frees resources
 void ArrowArrayViewReset(struct ArrowArrayView* array_view);
diff --git a/src/nanoarrow/typedefs_inline.h b/src/nanoarrow/typedefs_inline.h
index 0959be5..47e2892 100644
--- a/src/nanoarrow/typedefs_inline.h
+++ b/src/nanoarrow/typedefs_inline.h
@@ -233,9 +233,6 @@ struct ArrowBufferView {
 /// to customize allocation and deallocation of buffers when constructing
 /// an ArrowArray.
 struct ArrowBufferAllocator {
-  /// \brief Allocate a buffer or return NULL if it cannot be allocated
-  uint8_t* (*allocate)(struct ArrowBufferAllocator* allocator, int64_t size);
-
   /// \brief Reallocate a buffer or return NULL if it cannot be reallocated
   uint8_t* (*reallocate)(struct ArrowBufferAllocator* allocator, uint8_t* ptr,
                          int64_t old_size, int64_t new_size);
@@ -261,7 +258,7 @@ struct ArrowBuffer {
   int64_t capacity_bytes;
 
   /// \brief The allocator that will be used to reallocate and/or free the buffer
-  struct ArrowBufferAllocator* allocator;
+  struct ArrowBufferAllocator allocator;
 };
 
 /// \brief An owning mutable view of a bitmap