You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by li...@apache.org on 2023/01/31 20:03:17 UTC
[arrow] branch master updated: GH-33936: [Go] C Data Interface: export dummy buffer for nil buffers (#33951)
This is an automated email from the ASF dual-hosted git repository.
lidavidm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 5765aa29b5 GH-33936: [Go] C Data Interface: export dummy buffer for nil buffers (#33951)
5765aa29b5 is described below
commit 5765aa29b5ca93dae1a5fe8c7efce05dcd9f73c8
Author: David Li <li...@gmail.com>
AuthorDate: Tue Jan 31 15:03:10 2023 -0500
GH-33936: [Go] C Data Interface: export dummy buffer for nil buffers (#33951)
### Rationale for this change
While #14805 clarified that NULL buffers are valid in the C Data Interface, not all implementations handle this correctly (or were fixed, but earlier versions still exist).
### What changes are included in this PR?
Export a non-NULL dummy buffer for nil/empty buffers to ensure compatibility.
### Are these changes tested?
A regression test is included.
* Closes: #33936
Authored-by: David Li <li...@gmail.com>
Signed-off-by: David Li <li...@gmail.com>
---
go/arrow/cdata/cdata_exports.go | 14 +++++++++++--
go/arrow/cdata/cdata_test.go | 45 +++++++++++++++++++++++++++++++++++++++++
2 files changed, 57 insertions(+), 2 deletions(-)
diff --git a/go/arrow/cdata/cdata_exports.go b/go/arrow/cdata/cdata_exports.go
index d845ef8bc5..86a8080053 100644
--- a/go/arrow/cdata/cdata_exports.go
+++ b/go/arrow/cdata/cdata_exports.go
@@ -17,6 +17,7 @@
package cdata
// #include <errno.h>
+// #include <stdint.h>
// #include <stdlib.h>
// #include "arrow/c/abi.h"
// #include "arrow/c/helpers.h"
@@ -24,6 +25,8 @@ package cdata
// extern void releaseExportedSchema(struct ArrowSchema* schema);
// extern void releaseExportedArray(struct ArrowArray* array);
//
+// const uint8_t kGoCdataZeroRegion[8] = {0};
+//
// void goReleaseArray(struct ArrowArray* array) {
// releaseExportedArray(array);
// }
@@ -45,6 +48,7 @@ import (
"github.com/apache/arrow/go/v12/arrow"
"github.com/apache/arrow/go/v12/arrow/array"
"github.com/apache/arrow/go/v12/arrow/endian"
+ "github.com/apache/arrow/go/v12/arrow/internal"
"github.com/apache/arrow/go/v12/arrow/ipc"
)
@@ -376,7 +380,7 @@ func exportArray(arr arrow.Array, out *CArrowArray, outSchema *CArrowSchema) {
// unions don't have validity bitmaps, but we keep them shifted
// to make processing easier in other contexts. This means that
// we have to adjust for union arrays
- if arr.DataType().ID() == arrow.DENSE_UNION || arr.DataType().ID() == arrow.SPARSE_UNION {
+ if !internal.DefaultHasValidityBitmap(arr.DataType().ID()) {
out.n_buffers--
nbuffers--
bufs = bufs[1:]
@@ -385,7 +389,13 @@ func exportArray(arr arrow.Array, out *CArrowArray, outSchema *CArrowSchema) {
for i := range bufs {
buf := bufs[i]
if buf == nil || buf.Len() == 0 {
- buffers[i] = nil
+ if i > 0 || !internal.DefaultHasValidityBitmap(arr.DataType().ID()) {
+ // apache/arrow#33936: export a dummy buffer to be friendly to
+ // implementations that don't import NULL properly
+ buffers[i] = (*C.void)(unsafe.Pointer(&C.kGoCdataZeroRegion))
+ } else {
+ buffers[i] = nil
+ }
continue
}
diff --git a/go/arrow/cdata/cdata_test.go b/go/arrow/cdata/cdata_test.go
index a547075bd3..e6039cab59 100644
--- a/go/arrow/cdata/cdata_test.go
+++ b/go/arrow/cdata/cdata_test.go
@@ -795,6 +795,51 @@ func TestEmptyDictExport(t *testing.T) {
assert.Nil(t, out.dictionary.dictionary)
}
+func TestEmptyStringExport(t *testing.T) {
+ // apache/arrow#33936: regression test
+ bldr := array.NewBuilder(memory.DefaultAllocator, &arrow.StringType{})
+ defer bldr.Release()
+
+ arr := bldr.NewArray()
+ defer arr.Release()
+
+ var out CArrowArray
+ var sc CArrowSchema
+ ExportArrowArray(arr, &out, &sc)
+
+ assert.EqualValues(t, 'u', *sc.format)
+ assert.Zero(t, sc.n_children)
+ assert.Nil(t, sc.dictionary)
+
+ assert.EqualValues(t, 3, out.n_buffers)
+ buffers := (*[3]unsafe.Pointer)(unsafe.Pointer(out.buffers))
+ assert.EqualValues(t, unsafe.Pointer(nil), buffers[0])
+ assert.NotEqualValues(t, unsafe.Pointer(nil), buffers[1])
+ assert.NotEqualValues(t, unsafe.Pointer(nil), buffers[2])
+}
+
+func TestEmptyUnionExport(t *testing.T) {
+ // apache/arrow#33936: regression test
+ bldr := array.NewBuilder(memory.DefaultAllocator, arrow.SparseUnionOf([]arrow.Field{
+ {Name: "child", Type: &arrow.Int64Type{}},
+ }, []arrow.UnionTypeCode{0}))
+ defer bldr.Release()
+
+ arr := bldr.NewArray()
+ defer arr.Release()
+
+ var out CArrowArray
+ var sc CArrowSchema
+ ExportArrowArray(arr, &out, &sc)
+
+ assert.EqualValues(t, 1, sc.n_children)
+ assert.Nil(t, sc.dictionary)
+
+ assert.EqualValues(t, 1, out.n_buffers)
+ buffers := (*[1]unsafe.Pointer)(unsafe.Pointer(out.buffers))
+ assert.NotEqualValues(t, unsafe.Pointer(nil), buffers[0])
+}
+
func TestRecordReaderExport(t *testing.T) {
// Regression test for apache/arrow#33767
reclist := arrdata.Records["primitives"]