You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by sb...@apache.org on 2019/05/31 11:45:34 UTC
[arrow] branch master updated: ARROW-5462: [Go] support writing
zero-length List arrays
This is an automated email from the ASF dual-hosted git repository.
sbinet pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new e61bd90 ARROW-5462: [Go] support writing zero-length List arrays
e61bd90 is described below
commit e61bd90153f61a1b627e693a377f06636d2718bf
Author: Sebastien Binet <bi...@cern.ch>
AuthorDate: Fri May 31 13:45:16 2019 +0200
ARROW-5462: [Go] support writing zero-length List arrays
Author: Sebastien Binet <bi...@cern.ch>
Closes #4433 from sbinet/issue-5462 and squashes the following commits:
9637f35f <Sebastien Binet> ARROW-5462: support writing zero-length List arrays
---
go/arrow/array/record.go | 5 ++---
go/arrow/array/table.go | 2 +-
go/arrow/internal/arrdata/arrdata.go | 8 ++++++++
go/arrow/ipc/cmd/arrow-cat/main_test.go | 12 +++++++++---
go/arrow/ipc/cmd/arrow-ls/main_test.go | 6 +++---
go/arrow/ipc/file_test.go | 1 -
go/arrow/ipc/writer.go | 15 ++++++++++++---
7 files changed, 35 insertions(+), 14 deletions(-)
diff --git a/go/arrow/array/record.go b/go/arrow/array/record.go
index ac70b9c..7bbdab7 100644
--- a/go/arrow/array/record.go
+++ b/go/arrow/array/record.go
@@ -18,7 +18,6 @@ package array
import (
"fmt"
- "reflect"
"sync/atomic"
"github.com/apache/arrow/go/arrow"
@@ -187,10 +186,10 @@ func (rec *simpleRecord) validate() error {
arr.Len(), rec.rows,
)
}
- if !reflect.DeepEqual(f.Type, arr.DataType()) {
+ if !arrow.TypeEquals(f.Type, arr.DataType()) {
return fmt.Errorf("arrow/array: column %q type mismatch: got=%v, want=%v",
f.Name,
- arr.DataType().Name(), f.Type.Name(),
+ arr.DataType(), f.Type,
)
}
}
diff --git a/go/arrow/array/table.go b/go/arrow/array/table.go
index ff2c8ba..4d8bb45 100644
--- a/go/arrow/array/table.go
+++ b/go/arrow/array/table.go
@@ -55,7 +55,7 @@ func NewColumn(field arrow.Field, chunks *Chunked) *Column {
}
col.data.Retain()
- if col.data.DataType() != col.field.Type {
+ if !arrow.TypeEquals(col.data.DataType(), col.field.Type) {
col.data.Release()
panic("arrow/array: inconsistent data type")
}
diff --git a/go/arrow/internal/arrdata/arrdata.go b/go/arrow/internal/arrdata/arrdata.go
index c3ee8f1..0df3348 100644
--- a/go/arrow/internal/arrdata/arrdata.go
+++ b/go/arrow/internal/arrdata/arrdata.go
@@ -203,6 +203,14 @@ func makeListsRecords() []array.Record {
arrayOf(mem, []int32{-21, -22, -23, -24, -25}, mask),
}, []bool{true, false, true}),
},
+ []array.Interface{
+ func() array.Interface {
+ bldr := array.NewListBuilder(mem, arrow.PrimitiveTypes.Int32)
+ defer bldr.Release()
+
+ return bldr.NewListArray()
+ }(),
+ },
}
defer func() {
diff --git a/go/arrow/ipc/cmd/arrow-cat/main_test.go b/go/arrow/ipc/cmd/arrow-cat/main_test.go
index 0da306f..1fcd90a 100644
--- a/go/arrow/ipc/cmd/arrow-cat/main_test.go
+++ b/go/arrow/ipc/cmd/arrow-cat/main_test.go
@@ -91,6 +91,8 @@ record 2...
col[0] "list_nullable": [[-1 (null) (null) -4 -5] [-11 (null) (null) -14 -15] [-21 (null) (null) -24 -25]]
record 3...
col[0] "list_nullable": [[-1 (null) (null) -4 -5] (null) [-21 (null) (null) -24 -25]]
+record 4...
+ col[0] "list_nullable": []
`,
},
{
@@ -276,17 +278,21 @@ record 2...
col[0] "list_nullable": [[-1 (null) (null) -4 -5] [-11 (null) (null) -14 -15] [-21 (null) (null) -24 -25]]
record 3...
col[0] "list_nullable": [[-1 (null) (null) -4 -5] (null) [-21 (null) (null) -24 -25]]
+record 4...
+ col[0] "list_nullable": []
`,
},
{
name: "lists",
want: `version: V4
-record 1/3...
+record 1/4...
col[0] "list_nullable": [[1 (null) (null) 4 5] [11 (null) (null) 14 15] [21 (null) (null) 24 25]]
-record 2/3...
+record 2/4...
col[0] "list_nullable": [[-1 (null) (null) -4 -5] [-11 (null) (null) -14 -15] [-21 (null) (null) -24 -25]]
-record 3/3...
+record 3/4...
col[0] "list_nullable": [[-1 (null) (null) -4 -5] (null) [-21 (null) (null) -24 -25]]
+record 4/4...
+ col[0] "list_nullable": []
`,
},
{
diff --git a/go/arrow/ipc/cmd/arrow-ls/main_test.go b/go/arrow/ipc/cmd/arrow-ls/main_test.go
index 7f77304..7fcccd8 100644
--- a/go/arrow/ipc/cmd/arrow-ls/main_test.go
+++ b/go/arrow/ipc/cmd/arrow-ls/main_test.go
@@ -67,7 +67,7 @@ records: 2
want: `schema:
fields: 1
- list_nullable: type=list<item: int32>, nullable
-records: 3
+records: 4
`,
},
} {
@@ -195,7 +195,7 @@ records: 2
want: `schema:
fields: 1
- list_nullable: type=list<item: int32>, nullable
-records: 3
+records: 4
`,
},
{
@@ -204,7 +204,7 @@ records: 3
schema:
fields: 1
- list_nullable: type=list<item: int32>, nullable
-records: 3
+records: 4
`,
},
} {
diff --git a/go/arrow/ipc/file_test.go b/go/arrow/ipc/file_test.go
index 1f0d821..136b6a7 100644
--- a/go/arrow/ipc/file_test.go
+++ b/go/arrow/ipc/file_test.go
@@ -98,5 +98,4 @@ func TestFile(t *testing.T) {
}
})
}
-
}
diff --git a/go/arrow/ipc/writer.go b/go/arrow/ipc/writer.go
index def8f17..e252023 100644
--- a/go/arrow/ipc/writer.go
+++ b/go/arrow/ipc/writer.go
@@ -224,8 +224,15 @@ func (w *recordEncoder) visit(p *payload, arr array.Interface) error {
p.body = append(p.body, nil)
case *arrow.BooleanType:
- data := arr.Data()
- p.body = append(p.body, newTruncatedBitmap(w.mem, int64(data.Offset()), int64(data.Len()), data.Buffers()[1]))
+ var (
+ data = arr.Data()
+ bitm *memory.Buffer
+ )
+
+ if data.Len() != 0 {
+ bitm = newTruncatedBitmap(w.mem, int64(data.Offset()), int64(data.Len()), data.Buffers()[1])
+ }
+ p.body = append(p.body, bitm)
case arrow.FixedWidthDataType:
data := arr.Data()
@@ -237,7 +244,9 @@ func (w *recordEncoder) visit(p *payload, arr array.Interface) error {
case needTruncate(int64(data.Offset()), values, minLength):
panic("not implemented") // FIXME(sbinet) writer.cc:212
default:
- values.Retain()
+ if values != nil {
+ values.Retain()
+ }
}
p.body = append(p.body, values)