You are viewing a plain text version of this content. The canonical link for it is here.
Posted to github@arrow.apache.org by "paleolimbot (via GitHub)" <gi...@apache.org> on 2024/05/02 00:11:54 UTC
[PR] fix(python): Make shallow CArray copies less shallow to accomodate moving children [arrow-nanoarrow]
paleolimbot opened a new pull request, #451:
URL: https://github.com/apache/arrow-nanoarrow/pull/451
This PR updates the logic that creates a "shallow copy" of an `ArrowArray`. Before, it simply made a shallow copy of the outer array, which works in most cases. However, the spec allows for "moving" child arrays, which means that the guarantee of a valid *outer* array does not guarantee anything about the validity of the `ArrowArray*` child pointers. It's difficult, but possible, to trigger this in Python (below); however, I think this sort of code is common for importers (because it allows the lifecycle of columns in an array to be independent).
```python
import nanoarrow as na
import pyarrow as pa
# Given some array
array = na.c_array_from_buffers(
na.struct({"col1": na.int32()}),
3,
[None],
children=[na.c_array([1, 2, 3], na.int32())]
)
user_array = na.Array(array)
user_array
#> nanoarrow.Array<int32>[3]
#> {'col1': 1}
#> {'col1': 2}
#> {'col1': 3}
# totally valid shallow copy of this array
schema_capsule, array_capsule = array.__arrow_c_array__()
array2 = na.c_array(array_capsule, schema_capsule)
# A consumer is technically allowed to move a child array
x = pa.Array._import_from_c(array2.child(0)._addr(), pa.int32())
del array
del x
# With the previous shallow copy implementation, this could segfault or fail
list(user_array.iter_py())
# Instead of a segfault I tend to get:
#> NanoarrowException: ArrowBasicArrayStreamValidate() failed (22): Expected int32 array buffer 1 to have size >= 12 bytes but found buffer with 0 bytes
```
After this PR the original array remains valid:
```python
list(user_array.iter_py())
#> [{'col1': 1}, {'col1': 2}, {'col1': 3}]
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] fix(python): Make shallow CArray copies less shallow to accomodate moving children [arrow-nanoarrow]
Posted by "paleolimbot (via GitHub)" <gi...@apache.org>.
paleolimbot commented on code in PR #451:
URL: https://github.com/apache/arrow-nanoarrow/pull/451#discussion_r1588165906
##########
python/src/nanoarrow/_lib.pyx:
##########
@@ -160,68 +160,93 @@ cdef void pycapsule_array_view_deleter(object array_capsule) noexcept:
ArrowFree(array_view)
-cdef object alloc_c_array_view(ArrowArrayView** c_array_view) noexcept:
+cdef object alloc_c_array_view(ArrowArrayView** c_array_view):
c_array_view[0] = <ArrowArrayView*> ArrowMalloc(sizeof(ArrowArrayView))
ArrowArrayViewInitFromType(c_array_view[0], NANOARROW_TYPE_UNINITIALIZED)
return PyCapsule_New(c_array_view[0], 'nanoarrow_array_view', &pycapsule_array_view_deleter)
-cdef void arrow_array_release(ArrowArray* array) noexcept with gil:
- Py_DECREF(<object>array.private_data)
- array.private_data = NULL
- array.release = NULL
+# Provide a way to validate that we release all references we create
+cdef int64_t pyobject_buffer_count = 0
+def get_pyobject_buffer_count():
+ global pyobject_buffer_count
+ return pyobject_buffer_count
-cdef void c_array_shallow_copy(object base, const ArrowArray* c_array,
- ArrowArray* c_array_out) noexcept:
- # shallow copy
- memcpy(c_array_out, c_array, sizeof(ArrowArray))
- c_array_out.release = NULL
- c_array_out.private_data = NULL
- # track original base
- c_array_out.private_data = <void*>base
+cdef void c_deallocate_pyobject_buffer(ArrowBufferAllocator* allocator, uint8_t* ptr, int64_t size) noexcept with gil:
+ Py_DECREF(<object>allocator.private_data)
+
+ global pyobject_buffer_count
+ pyobject_buffer_count -= 1
+
+
+cdef void c_pyobject_buffer(object base, const void* buf, int64_t size_bytes, ArrowBuffer* out):
+ out.data = <uint8_t*>buf
+ out.size_bytes = size_bytes
+ out.allocator = ArrowBufferDeallocator(
+ <ArrowBufferDeallocatorCallback>c_deallocate_pyobject_buffer,
+ <void*>base
+ )
Py_INCREF(base)
- c_array_out.release = arrow_array_release
+ global pyobject_buffer_count
+ pyobject_buffer_count += 1
-cdef object alloc_c_array_shallow_copy(object base, const ArrowArray* c_array):
- """Make a shallow copy of an ArrowArray
- To more safely implement export of an ArrowArray whose address may be
- depended on by some other Python object, we implement a shallow copy
- whose constructor calls Py_INCREF() on a Python object responsible
- for the ArrowArray's lifecycle and whose deleter calls Py_DECREF() on
- the same object.
- """
- cdef ArrowArray* c_array_out
- array_capsule = alloc_c_array(&c_array_out)
- c_array_shallow_copy(base, c_array, c_array_out)
- return array_capsule
+cdef void c_array_shallow_copy(object base, const ArrowArray* src, ArrowArray* dst):
+ # Allocate an ArrowArray* that will definitely be cleaned up should an exception
+ # be raised in the process of shallow copying its contents
+ cdef ArrowArray* tmp
+ shelter = alloc_c_array(&tmp)
+ cdef int code
+ code = ArrowArrayInitFromType(tmp, NANOARROW_TYPE_UNINITIALIZED)
+ Error.raise_error_not_ok("ArrowArrayInitFromType()", code)
-cdef void c_device_array_shallow_copy(object base, const ArrowDeviceArray* c_array,
- ArrowDeviceArray* c_array_out) noexcept:
- # shallow copy
- memcpy(c_array_out, c_array, sizeof(ArrowDeviceArray))
- c_array_out.array.release = NULL
- c_array_out.array.private_data = NULL
+ # Copy data for this array, adding a reference for each buffer
+ # This allows us to use nanoarrow's ArrowArray implementation without
+ # writing our own release callbacks/private_data.
+ tmp.length = src.length
+ tmp.offset = src.offset
+ tmp.null_count = src.null_count
- # track original base
- c_array_out.array.private_data = <void*>base
- Py_INCREF(base)
- c_array_out.array.release = arrow_array_release
+ for i in range(src.n_buffers):
+ if src.buffers[i] != NULL:
+ c_pyobject_buffer(base, src.buffers[i], 0, ArrowArrayBuffer(tmp, i))
Review Comment:
I added a comment...it's because we don't know how long the buffer is (although it wouldn't necessarily matter since the only function that will ever see it is our own deallocator).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] fix(python): Make shallow CArray copies less shallow to accomodate moving children [arrow-nanoarrow]
Posted by "paleolimbot (via GitHub)" <gi...@apache.org>.
paleolimbot commented on code in PR #451:
URL: https://github.com/apache/arrow-nanoarrow/pull/451#discussion_r1587994366
##########
python/src/nanoarrow/_lib.pyx:
##########
@@ -160,68 +160,93 @@ cdef void pycapsule_array_view_deleter(object array_capsule) noexcept:
ArrowFree(array_view)
-cdef object alloc_c_array_view(ArrowArrayView** c_array_view) noexcept:
+cdef object alloc_c_array_view(ArrowArrayView** c_array_view):
c_array_view[0] = <ArrowArrayView*> ArrowMalloc(sizeof(ArrowArrayView))
ArrowArrayViewInitFromType(c_array_view[0], NANOARROW_TYPE_UNINITIALIZED)
return PyCapsule_New(c_array_view[0], 'nanoarrow_array_view', &pycapsule_array_view_deleter)
-cdef void arrow_array_release(ArrowArray* array) noexcept with gil:
- Py_DECREF(<object>array.private_data)
- array.private_data = NULL
- array.release = NULL
+# Provide a way to validate that we release all references we create
+cdef int64_t pyobject_buffer_count = 0
+def get_pyobject_buffer_count():
+ global pyobject_buffer_count
+ return pyobject_buffer_count
Review Comment:
I checked the generated C code and it looks like it is "just" incrementing and decrementing `static int64_t __pyx_v_9nanoarrow_4_lib_pyobject_buffer_count;` (which I don't believe is prohibitively expensive). I have a similar reference counting scheme in the R package and it has been mission critical in debugging!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] fix(python): Make shallow CArray copies less shallow to accomodate moving children [arrow-nanoarrow]
Posted by "jorisvandenbossche (via GitHub)" <gi...@apache.org>.
jorisvandenbossche commented on code in PR #451:
URL: https://github.com/apache/arrow-nanoarrow/pull/451#discussion_r1587525749
##########
python/src/nanoarrow/_lib.pyx:
##########
@@ -160,68 +160,93 @@ cdef void pycapsule_array_view_deleter(object array_capsule) noexcept:
ArrowFree(array_view)
-cdef object alloc_c_array_view(ArrowArrayView** c_array_view) noexcept:
+cdef object alloc_c_array_view(ArrowArrayView** c_array_view):
c_array_view[0] = <ArrowArrayView*> ArrowMalloc(sizeof(ArrowArrayView))
ArrowArrayViewInitFromType(c_array_view[0], NANOARROW_TYPE_UNINITIALIZED)
return PyCapsule_New(c_array_view[0], 'nanoarrow_array_view', &pycapsule_array_view_deleter)
-cdef void arrow_array_release(ArrowArray* array) noexcept with gil:
- Py_DECREF(<object>array.private_data)
- array.private_data = NULL
- array.release = NULL
+# Provide a way to validate that we release all references we create
+cdef int64_t pyobject_buffer_count = 0
+def get_pyobject_buffer_count():
+ global pyobject_buffer_count
+ return pyobject_buffer_count
Review Comment:
My first thought was whether we should only do this in some kind of debug build, but AFAIK this is not actually used in the code itself (se don't check the value), and only accessed in test code? (and so that should be safe and cheap)
##########
python/src/nanoarrow/_lib.pyx:
##########
@@ -160,68 +160,93 @@ cdef void pycapsule_array_view_deleter(object array_capsule) noexcept:
ArrowFree(array_view)
-cdef object alloc_c_array_view(ArrowArrayView** c_array_view) noexcept:
+cdef object alloc_c_array_view(ArrowArrayView** c_array_view):
c_array_view[0] = <ArrowArrayView*> ArrowMalloc(sizeof(ArrowArrayView))
ArrowArrayViewInitFromType(c_array_view[0], NANOARROW_TYPE_UNINITIALIZED)
return PyCapsule_New(c_array_view[0], 'nanoarrow_array_view', &pycapsule_array_view_deleter)
-cdef void arrow_array_release(ArrowArray* array) noexcept with gil:
- Py_DECREF(<object>array.private_data)
- array.private_data = NULL
- array.release = NULL
+# Provide a way to validate that we release all references we create
+cdef int64_t pyobject_buffer_count = 0
+def get_pyobject_buffer_count():
+ global pyobject_buffer_count
+ return pyobject_buffer_count
-cdef void c_array_shallow_copy(object base, const ArrowArray* c_array,
- ArrowArray* c_array_out) noexcept:
- # shallow copy
- memcpy(c_array_out, c_array, sizeof(ArrowArray))
- c_array_out.release = NULL
- c_array_out.private_data = NULL
- # track original base
- c_array_out.private_data = <void*>base
+cdef void c_deallocate_pyobject_buffer(ArrowBufferAllocator* allocator, uint8_t* ptr, int64_t size) noexcept with gil:
+ Py_DECREF(<object>allocator.private_data)
+
+ global pyobject_buffer_count
+ pyobject_buffer_count -= 1
+
+
+cdef void c_pyobject_buffer(object base, const void* buf, int64_t size_bytes, ArrowBuffer* out):
+ out.data = <uint8_t*>buf
+ out.size_bytes = size_bytes
+ out.allocator = ArrowBufferDeallocator(
+ <ArrowBufferDeallocatorCallback>c_deallocate_pyobject_buffer,
+ <void*>base
+ )
Py_INCREF(base)
- c_array_out.release = arrow_array_release
+ global pyobject_buffer_count
+ pyobject_buffer_count += 1
-cdef object alloc_c_array_shallow_copy(object base, const ArrowArray* c_array):
- """Make a shallow copy of an ArrowArray
- To more safely implement export of an ArrowArray whose address may be
- depended on by some other Python object, we implement a shallow copy
- whose constructor calls Py_INCREF() on a Python object responsible
- for the ArrowArray's lifecycle and whose deleter calls Py_DECREF() on
- the same object.
- """
- cdef ArrowArray* c_array_out
- array_capsule = alloc_c_array(&c_array_out)
- c_array_shallow_copy(base, c_array, c_array_out)
- return array_capsule
+cdef void c_array_shallow_copy(object base, const ArrowArray* src, ArrowArray* dst):
Review Comment:
Can you add a docstring that says explains something about "how shallow" this function?
##########
python/src/nanoarrow/_lib.pyx:
##########
@@ -160,68 +160,93 @@ cdef void pycapsule_array_view_deleter(object array_capsule) noexcept:
ArrowFree(array_view)
-cdef object alloc_c_array_view(ArrowArrayView** c_array_view) noexcept:
+cdef object alloc_c_array_view(ArrowArrayView** c_array_view):
c_array_view[0] = <ArrowArrayView*> ArrowMalloc(sizeof(ArrowArrayView))
ArrowArrayViewInitFromType(c_array_view[0], NANOARROW_TYPE_UNINITIALIZED)
return PyCapsule_New(c_array_view[0], 'nanoarrow_array_view', &pycapsule_array_view_deleter)
-cdef void arrow_array_release(ArrowArray* array) noexcept with gil:
- Py_DECREF(<object>array.private_data)
- array.private_data = NULL
- array.release = NULL
+# Provide a way to validate that we release all references we create
+cdef int64_t pyobject_buffer_count = 0
+def get_pyobject_buffer_count():
+ global pyobject_buffer_count
+ return pyobject_buffer_count
-cdef void c_array_shallow_copy(object base, const ArrowArray* c_array,
- ArrowArray* c_array_out) noexcept:
- # shallow copy
- memcpy(c_array_out, c_array, sizeof(ArrowArray))
- c_array_out.release = NULL
- c_array_out.private_data = NULL
- # track original base
- c_array_out.private_data = <void*>base
+cdef void c_deallocate_pyobject_buffer(ArrowBufferAllocator* allocator, uint8_t* ptr, int64_t size) noexcept with gil:
+ Py_DECREF(<object>allocator.private_data)
+
+ global pyobject_buffer_count
+ pyobject_buffer_count -= 1
+
+
+cdef void c_pyobject_buffer(object base, const void* buf, int64_t size_bytes, ArrowBuffer* out):
+ out.data = <uint8_t*>buf
+ out.size_bytes = size_bytes
+ out.allocator = ArrowBufferDeallocator(
+ <ArrowBufferDeallocatorCallback>c_deallocate_pyobject_buffer,
+ <void*>base
+ )
Py_INCREF(base)
- c_array_out.release = arrow_array_release
+ global pyobject_buffer_count
+ pyobject_buffer_count += 1
-cdef object alloc_c_array_shallow_copy(object base, const ArrowArray* c_array):
- """Make a shallow copy of an ArrowArray
- To more safely implement export of an ArrowArray whose address may be
- depended on by some other Python object, we implement a shallow copy
- whose constructor calls Py_INCREF() on a Python object responsible
- for the ArrowArray's lifecycle and whose deleter calls Py_DECREF() on
- the same object.
- """
- cdef ArrowArray* c_array_out
- array_capsule = alloc_c_array(&c_array_out)
- c_array_shallow_copy(base, c_array, c_array_out)
- return array_capsule
+cdef void c_array_shallow_copy(object base, const ArrowArray* src, ArrowArray* dst):
+ # Allocate an ArrowArray* that will definitely be cleaned up should an exception
+ # be raised in the process of shallow copying its contents
+ cdef ArrowArray* tmp
+ shelter = alloc_c_array(&tmp)
+ cdef int code
+ code = ArrowArrayInitFromType(tmp, NANOARROW_TYPE_UNINITIALIZED)
+ Error.raise_error_not_ok("ArrowArrayInitFromType()", code)
-cdef void c_device_array_shallow_copy(object base, const ArrowDeviceArray* c_array,
- ArrowDeviceArray* c_array_out) noexcept:
- # shallow copy
- memcpy(c_array_out, c_array, sizeof(ArrowDeviceArray))
- c_array_out.array.release = NULL
- c_array_out.array.private_data = NULL
+ # Copy data for this array, adding a reference for each buffer
+ # This allows us to use nanoarrow's ArrowArray implementation without
+ # writing our own release callbacks/private_data.
+ tmp.length = src.length
+ tmp.offset = src.offset
+ tmp.null_count = src.null_count
- # track original base
- c_array_out.array.private_data = <void*>base
- Py_INCREF(base)
- c_array_out.array.release = arrow_array_release
+ for i in range(src.n_buffers):
+ if src.buffers[i] != NULL:
+ c_pyobject_buffer(base, src.buffers[i], 0, ArrowArrayBuffer(tmp, i))
Review Comment:
Why the 0 size?
##########
python/src/nanoarrow/_lib.pyx:
##########
@@ -160,68 +160,93 @@ cdef void pycapsule_array_view_deleter(object array_capsule) noexcept:
ArrowFree(array_view)
-cdef object alloc_c_array_view(ArrowArrayView** c_array_view) noexcept:
+cdef object alloc_c_array_view(ArrowArrayView** c_array_view):
c_array_view[0] = <ArrowArrayView*> ArrowMalloc(sizeof(ArrowArrayView))
ArrowArrayViewInitFromType(c_array_view[0], NANOARROW_TYPE_UNINITIALIZED)
return PyCapsule_New(c_array_view[0], 'nanoarrow_array_view', &pycapsule_array_view_deleter)
-cdef void arrow_array_release(ArrowArray* array) noexcept with gil:
- Py_DECREF(<object>array.private_data)
- array.private_data = NULL
- array.release = NULL
+# Provide a way to validate that we release all references we create
+cdef int64_t pyobject_buffer_count = 0
+def get_pyobject_buffer_count():
+ global pyobject_buffer_count
+ return pyobject_buffer_count
-cdef void c_array_shallow_copy(object base, const ArrowArray* c_array,
- ArrowArray* c_array_out) noexcept:
- # shallow copy
- memcpy(c_array_out, c_array, sizeof(ArrowArray))
- c_array_out.release = NULL
- c_array_out.private_data = NULL
- # track original base
- c_array_out.private_data = <void*>base
+cdef void c_deallocate_pyobject_buffer(ArrowBufferAllocator* allocator, uint8_t* ptr, int64_t size) noexcept with gil:
+ Py_DECREF(<object>allocator.private_data)
+
+ global pyobject_buffer_count
+ pyobject_buffer_count -= 1
+
+
+cdef void c_pyobject_buffer(object base, const void* buf, int64_t size_bytes, ArrowBuffer* out):
+ out.data = <uint8_t*>buf
+ out.size_bytes = size_bytes
+ out.allocator = ArrowBufferDeallocator(
+ <ArrowBufferDeallocatorCallback>c_deallocate_pyobject_buffer,
+ <void*>base
+ )
Py_INCREF(base)
- c_array_out.release = arrow_array_release
+ global pyobject_buffer_count
+ pyobject_buffer_count += 1
-cdef object alloc_c_array_shallow_copy(object base, const ArrowArray* c_array):
- """Make a shallow copy of an ArrowArray
- To more safely implement export of an ArrowArray whose address may be
- depended on by some other Python object, we implement a shallow copy
- whose constructor calls Py_INCREF() on a Python object responsible
- for the ArrowArray's lifecycle and whose deleter calls Py_DECREF() on
- the same object.
- """
- cdef ArrowArray* c_array_out
- array_capsule = alloc_c_array(&c_array_out)
- c_array_shallow_copy(base, c_array, c_array_out)
- return array_capsule
+cdef void c_array_shallow_copy(object base, const ArrowArray* src, ArrowArray* dst):
+ # Allocate an ArrowArray* that will definitely be cleaned up should an exception
+ # be raised in the process of shallow copying its contents
+ cdef ArrowArray* tmp
+ shelter = alloc_c_array(&tmp)
+ cdef int code
+ code = ArrowArrayInitFromType(tmp, NANOARROW_TYPE_UNINITIALIZED)
+ Error.raise_error_not_ok("ArrowArrayInitFromType()", code)
-cdef void c_device_array_shallow_copy(object base, const ArrowDeviceArray* c_array,
- ArrowDeviceArray* c_array_out) noexcept:
- # shallow copy
- memcpy(c_array_out, c_array, sizeof(ArrowDeviceArray))
- c_array_out.array.release = NULL
- c_array_out.array.private_data = NULL
+ # Copy data for this array, adding a reference for each buffer
+ # This allows us to use nanoarrow's ArrowArray implementation without
+ # writing our own release callbacks/private_data.
Review Comment:
I don't understand this comment, because we _are_ in nanoarrow?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] fix(python): Make shallow CArray copies less shallow to accomodate moving children [arrow-nanoarrow]
Posted by "paleolimbot (via GitHub)" <gi...@apache.org>.
paleolimbot merged PR #451:
URL: https://github.com/apache/arrow-nanoarrow/pull/451
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] fix(python): Make shallow CArray copies less shallow to accomodate moving children [arrow-nanoarrow]
Posted by "paleolimbot (via GitHub)" <gi...@apache.org>.
paleolimbot commented on code in PR #451:
URL: https://github.com/apache/arrow-nanoarrow/pull/451#discussion_r1588225773
##########
python/src/nanoarrow/_lib.pyx:
##########
@@ -160,68 +160,93 @@ cdef void pycapsule_array_view_deleter(object array_capsule) noexcept:
ArrowFree(array_view)
-cdef object alloc_c_array_view(ArrowArrayView** c_array_view) noexcept:
+cdef object alloc_c_array_view(ArrowArrayView** c_array_view):
c_array_view[0] = <ArrowArrayView*> ArrowMalloc(sizeof(ArrowArrayView))
ArrowArrayViewInitFromType(c_array_view[0], NANOARROW_TYPE_UNINITIALIZED)
return PyCapsule_New(c_array_view[0], 'nanoarrow_array_view', &pycapsule_array_view_deleter)
-cdef void arrow_array_release(ArrowArray* array) noexcept with gil:
- Py_DECREF(<object>array.private_data)
- array.private_data = NULL
- array.release = NULL
+# Provide a way to validate that we release all references we create
+cdef int64_t pyobject_buffer_count = 0
+def get_pyobject_buffer_count():
+ global pyobject_buffer_count
+ return pyobject_buffer_count
-cdef void c_array_shallow_copy(object base, const ArrowArray* c_array,
- ArrowArray* c_array_out) noexcept:
- # shallow copy
- memcpy(c_array_out, c_array, sizeof(ArrowArray))
- c_array_out.release = NULL
- c_array_out.private_data = NULL
- # track original base
- c_array_out.private_data = <void*>base
+cdef void c_deallocate_pyobject_buffer(ArrowBufferAllocator* allocator, uint8_t* ptr, int64_t size) noexcept with gil:
+ Py_DECREF(<object>allocator.private_data)
+
+ global pyobject_buffer_count
+ pyobject_buffer_count -= 1
+
+
+cdef void c_pyobject_buffer(object base, const void* buf, int64_t size_bytes, ArrowBuffer* out):
+ out.data = <uint8_t*>buf
+ out.size_bytes = size_bytes
+ out.allocator = ArrowBufferDeallocator(
+ <ArrowBufferDeallocatorCallback>c_deallocate_pyobject_buffer,
+ <void*>base
+ )
Py_INCREF(base)
- c_array_out.release = arrow_array_release
+ global pyobject_buffer_count
+ pyobject_buffer_count += 1
-cdef object alloc_c_array_shallow_copy(object base, const ArrowArray* c_array):
- """Make a shallow copy of an ArrowArray
- To more safely implement export of an ArrowArray whose address may be
- depended on by some other Python object, we implement a shallow copy
- whose constructor calls Py_INCREF() on a Python object responsible
- for the ArrowArray's lifecycle and whose deleter calls Py_DECREF() on
- the same object.
- """
- cdef ArrowArray* c_array_out
- array_capsule = alloc_c_array(&c_array_out)
- c_array_shallow_copy(base, c_array, c_array_out)
- return array_capsule
+cdef void c_array_shallow_copy(object base, const ArrowArray* src, ArrowArray* dst):
+ # Allocate an ArrowArray* that will definitely be cleaned up should an exception
+ # be raised in the process of shallow copying its contents
+ cdef ArrowArray* tmp
+ shelter = alloc_c_array(&tmp)
+ cdef int code
+ code = ArrowArrayInitFromType(tmp, NANOARROW_TYPE_UNINITIALIZED)
+ Error.raise_error_not_ok("ArrowArrayInitFromType()", code)
-cdef void c_device_array_shallow_copy(object base, const ArrowDeviceArray* c_array,
- ArrowDeviceArray* c_array_out) noexcept:
- # shallow copy
- memcpy(c_array_out, c_array, sizeof(ArrowDeviceArray))
- c_array_out.array.release = NULL
- c_array_out.array.private_data = NULL
+ # Copy data for this array, adding a reference for each buffer
+ # This allows us to use nanoarrow's ArrowArray implementation without
+ # writing our own release callbacks/private_data.
Review Comment:
I added a comment here to specifically refer to the `ArrowArray` release callback provided by `ArrowArrayInitFromType()` (which walks over a private array of `ArrowBuffer` and calls `ArrowBufferReset()`).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] fix(python): Make shallow CArray copies less shallow to accomodate moving children [arrow-nanoarrow]
Posted by "paleolimbot (via GitHub)" <gi...@apache.org>.
paleolimbot commented on code in PR #451:
URL: https://github.com/apache/arrow-nanoarrow/pull/451#discussion_r1588166338
##########
python/src/nanoarrow/_lib.pyx:
##########
@@ -160,68 +160,93 @@ cdef void pycapsule_array_view_deleter(object array_capsule) noexcept:
ArrowFree(array_view)
-cdef object alloc_c_array_view(ArrowArrayView** c_array_view) noexcept:
+cdef object alloc_c_array_view(ArrowArrayView** c_array_view):
c_array_view[0] = <ArrowArrayView*> ArrowMalloc(sizeof(ArrowArrayView))
ArrowArrayViewInitFromType(c_array_view[0], NANOARROW_TYPE_UNINITIALIZED)
return PyCapsule_New(c_array_view[0], 'nanoarrow_array_view', &pycapsule_array_view_deleter)
-cdef void arrow_array_release(ArrowArray* array) noexcept with gil:
- Py_DECREF(<object>array.private_data)
- array.private_data = NULL
- array.release = NULL
+# Provide a way to validate that we release all references we create
+cdef int64_t pyobject_buffer_count = 0
+def get_pyobject_buffer_count():
+ global pyobject_buffer_count
+ return pyobject_buffer_count
-cdef void c_array_shallow_copy(object base, const ArrowArray* c_array,
- ArrowArray* c_array_out) noexcept:
- # shallow copy
- memcpy(c_array_out, c_array, sizeof(ArrowArray))
- c_array_out.release = NULL
- c_array_out.private_data = NULL
- # track original base
- c_array_out.private_data = <void*>base
+cdef void c_deallocate_pyobject_buffer(ArrowBufferAllocator* allocator, uint8_t* ptr, int64_t size) noexcept with gil:
+ Py_DECREF(<object>allocator.private_data)
+
+ global pyobject_buffer_count
+ pyobject_buffer_count -= 1
+
+
+cdef void c_pyobject_buffer(object base, const void* buf, int64_t size_bytes, ArrowBuffer* out):
+ out.data = <uint8_t*>buf
+ out.size_bytes = size_bytes
+ out.allocator = ArrowBufferDeallocator(
+ <ArrowBufferDeallocatorCallback>c_deallocate_pyobject_buffer,
+ <void*>base
+ )
Py_INCREF(base)
- c_array_out.release = arrow_array_release
+ global pyobject_buffer_count
+ pyobject_buffer_count += 1
-cdef object alloc_c_array_shallow_copy(object base, const ArrowArray* c_array):
- """Make a shallow copy of an ArrowArray
- To more safely implement export of an ArrowArray whose address may be
- depended on by some other Python object, we implement a shallow copy
- whose constructor calls Py_INCREF() on a Python object responsible
- for the ArrowArray's lifecycle and whose deleter calls Py_DECREF() on
- the same object.
- """
- cdef ArrowArray* c_array_out
- array_capsule = alloc_c_array(&c_array_out)
- c_array_shallow_copy(base, c_array, c_array_out)
- return array_capsule
+cdef void c_array_shallow_copy(object base, const ArrowArray* src, ArrowArray* dst):
Review Comment:
I added a nice long one!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] fix(python): Make shallow CArray copies less shallow to accomodate moving children [arrow-nanoarrow]
Posted by "jorisvandenbossche (via GitHub)" <gi...@apache.org>.
jorisvandenbossche commented on code in PR #451:
URL: https://github.com/apache/arrow-nanoarrow/pull/451#discussion_r1588814246
##########
python/src/nanoarrow/_lib.pyx:
##########
@@ -160,68 +160,93 @@ cdef void pycapsule_array_view_deleter(object array_capsule) noexcept:
ArrowFree(array_view)
-cdef object alloc_c_array_view(ArrowArrayView** c_array_view) noexcept:
+cdef object alloc_c_array_view(ArrowArrayView** c_array_view):
c_array_view[0] = <ArrowArrayView*> ArrowMalloc(sizeof(ArrowArrayView))
ArrowArrayViewInitFromType(c_array_view[0], NANOARROW_TYPE_UNINITIALIZED)
return PyCapsule_New(c_array_view[0], 'nanoarrow_array_view', &pycapsule_array_view_deleter)
-cdef void arrow_array_release(ArrowArray* array) noexcept with gil:
- Py_DECREF(<object>array.private_data)
- array.private_data = NULL
- array.release = NULL
+# Provide a way to validate that we release all references we create
+cdef int64_t pyobject_buffer_count = 0
+def get_pyobject_buffer_count():
+ global pyobject_buffer_count
+ return pyobject_buffer_count
-cdef void c_array_shallow_copy(object base, const ArrowArray* c_array,
- ArrowArray* c_array_out) noexcept:
- # shallow copy
- memcpy(c_array_out, c_array, sizeof(ArrowArray))
- c_array_out.release = NULL
- c_array_out.private_data = NULL
- # track original base
- c_array_out.private_data = <void*>base
+cdef void c_deallocate_pyobject_buffer(ArrowBufferAllocator* allocator, uint8_t* ptr, int64_t size) noexcept with gil:
+ Py_DECREF(<object>allocator.private_data)
+
+ global pyobject_buffer_count
+ pyobject_buffer_count -= 1
+
+
+cdef void c_pyobject_buffer(object base, const void* buf, int64_t size_bytes, ArrowBuffer* out):
+ out.data = <uint8_t*>buf
+ out.size_bytes = size_bytes
+ out.allocator = ArrowBufferDeallocator(
+ <ArrowBufferDeallocatorCallback>c_deallocate_pyobject_buffer,
+ <void*>base
+ )
Py_INCREF(base)
- c_array_out.release = arrow_array_release
+ global pyobject_buffer_count
+ pyobject_buffer_count += 1
-cdef object alloc_c_array_shallow_copy(object base, const ArrowArray* c_array):
- """Make a shallow copy of an ArrowArray
- To more safely implement export of an ArrowArray whose address may be
- depended on by some other Python object, we implement a shallow copy
- whose constructor calls Py_INCREF() on a Python object responsible
- for the ArrowArray's lifecycle and whose deleter calls Py_DECREF() on
- the same object.
- """
- cdef ArrowArray* c_array_out
- array_capsule = alloc_c_array(&c_array_out)
- c_array_shallow_copy(base, c_array, c_array_out)
- return array_capsule
+cdef void c_array_shallow_copy(object base, const ArrowArray* src, ArrowArray* dst):
Review Comment:
Thanks!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org