You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ks...@apache.org on 2019/10/05 09:47:57 UTC
[arrow] 18/45: ARROW-6752: [Go] make Null array implement Stringer, add tests for Nu…
This is an automated email from the ASF dual-hosted git repository.
kszucs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
commit e3bf93a1beb8b1f2671872cddf223570b8f7f52f
Author: Sebastien Binet <bi...@cern.ch>
AuthorDate: Wed Oct 2 10:32:42 2019 +0200
ARROW-6752: [Go] make Null array implement Stringer, add tests for Nu…
…ll arrays
Closes #5555 from sbinet/issue-6752 and squashes the following commits:
18b5387cc <Sebastien Binet> ARROW-6752: make Null array implement Stringer, add tests for Null arrays
Authored-by: Sebastien Binet <bi...@cern.ch>
Signed-off-by: Sebastien Binet <bi...@cern.ch>
---
go/arrow/array/compare_test.go | 14 +++++-----
go/arrow/array/null.go | 14 ++++++++++
go/arrow/internal/arrdata/arrdata.go | 52 ++++++++++++++++++++++++++++++++++++
go/arrow/internal/arrjson/arrjson.go | 13 +++++++++
4 files changed, 86 insertions(+), 7 deletions(-)
diff --git a/go/arrow/array/compare_test.go b/go/arrow/array/compare_test.go
index e9927f0..3ed326b 100644
--- a/go/arrow/array/compare_test.go
+++ b/go/arrow/array/compare_test.go
@@ -44,7 +44,7 @@ func TestArrayEqual(t *testing.T) {
sub2 := array.NewSlice(arr, 0, int64(arr.Len()-1))
defer sub2.Release()
- if array.ArrayEqual(sub1, sub2) {
+ if array.ArrayEqual(sub1, sub2) && name != "nulls" {
t.Fatalf("non-identical arrays should not compare equal:\nsub1=%v\nsub2=%v\narrf=%v\n", sub1, sub2, arr)
}
})
@@ -73,7 +73,7 @@ func TestArraySliceEqual(t *testing.T) {
sub2 := array.NewSlice(arr, 0, int64(arr.Len()-1))
defer sub2.Release()
- if array.ArraySliceEqual(sub1, 0, int64(sub1.Len()), sub2, 0, int64(sub2.Len())) {
+ if array.ArraySliceEqual(sub1, 0, int64(sub1.Len()), sub2, 0, int64(sub2.Len())) && name != "nulls" {
t.Fatalf("non-identical slices should not compare equal:\nsub1=%v\nsub2=%v\narrf=%v\n", sub1, sub2, arr)
}
})
@@ -99,7 +99,7 @@ func TestArrayApproxEqual(t *testing.T) {
sub2 := array.NewSlice(arr, 0, int64(arr.Len()-1))
defer sub2.Release()
- if array.ArrayApproxEqual(sub1, sub2) {
+ if array.ArrayApproxEqual(sub1, sub2) && name != "nulls" {
t.Fatalf("non-identical arrays should not compare equal:\nsub1=%v\nsub2=%v\narrf=%v\n", sub1, sub2, arr)
}
})
@@ -489,7 +489,7 @@ func TestRecordEqual(t *testing.T) {
t.Fatalf("identical records should compare equal:\nrecord:\n%v", rec0)
}
- if array.RecordEqual(rec0, rec1) {
+ if array.RecordEqual(rec0, rec1) && name != "nulls" {
t.Fatalf("non-identical records should not compare equal:\nrec0:\n%v\nrec1:\n%v", rec0, rec1)
}
@@ -498,7 +498,7 @@ func TestRecordEqual(t *testing.T) {
sub01 := rec0.NewSlice(1, recs[0].NumRows())
defer sub01.Release()
- if array.RecordEqual(sub00, sub01) {
+ if array.RecordEqual(sub00, sub01) && name != "nulls" {
t.Fatalf("non-identical records should not compare equal:\nsub0:\n%v\nsub1:\n%v", sub00, sub01)
}
})
@@ -514,7 +514,7 @@ func TestRecordApproxEqual(t *testing.T) {
t.Fatalf("identical records should compare equal:\nrecord:\n%v", rec0)
}
- if array.RecordApproxEqual(rec0, rec1) {
+ if array.RecordApproxEqual(rec0, rec1) && name != "nulls" {
t.Fatalf("non-identical records should not compare equal:\nrec0:\n%v\nrec1:\n%v", rec0, rec1)
}
@@ -523,7 +523,7 @@ func TestRecordApproxEqual(t *testing.T) {
sub01 := rec0.NewSlice(1, recs[0].NumRows())
defer sub01.Release()
- if array.RecordApproxEqual(sub00, sub01) {
+ if array.RecordApproxEqual(sub00, sub01) && name != "nulls" {
t.Fatalf("non-identical records should not compare equal:\nsub0:\n%v\nsub1:\n%v", sub00, sub01)
}
})
diff --git a/go/arrow/array/null.go b/go/arrow/array/null.go
index 346445d..56d7fa4 100644
--- a/go/arrow/array/null.go
+++ b/go/arrow/array/null.go
@@ -17,6 +17,7 @@
package array
import (
+ "strings"
"sync/atomic"
"github.com/apache/arrow/go/arrow"
@@ -53,6 +54,19 @@ func NewNullData(data *Data) *Null {
return a
}
+func (a *Null) String() string {
+ o := new(strings.Builder)
+ o.WriteString("[")
+ for i := 0; i < a.Len(); i++ {
+ if i > 0 {
+ o.WriteString(" ")
+ }
+ o.WriteString("(null)")
+ }
+ o.WriteString("]")
+ return o.String()
+}
+
func (a *Null) setData(data *Data) {
a.array.setData(data)
a.array.nullBitmapBytes = nil
diff --git a/go/arrow/internal/arrdata/arrdata.go b/go/arrow/internal/arrdata/arrdata.go
index c249236..40dee3b 100644
--- a/go/arrow/internal/arrdata/arrdata.go
+++ b/go/arrow/internal/arrdata/arrdata.go
@@ -34,6 +34,7 @@ var (
)
func init() {
+ Records["nulls"] = makeNullRecords()
Records["primitives"] = makePrimitiveRecords()
Records["structs"] = makeStructsRecords()
Records["lists"] = makeListsRecords()
@@ -51,6 +52,49 @@ func init() {
sort.Strings(RecordNames)
}
+func makeNullRecords() []array.Record {
+ mem := memory.NewGoAllocator()
+
+ meta := arrow.NewMetadata(
+ []string{"k1", "k2", "k3"},
+ []string{"v1", "v2", "v3"},
+ )
+
+ schema := arrow.NewSchema(
+ []arrow.Field{
+ arrow.Field{Name: "nulls", Type: arrow.Null, Nullable: true},
+ }, &meta,
+ )
+
+ mask := []bool{true, false, false, true, true}
+ chunks := [][]array.Interface{
+ []array.Interface{
+ arrayOf(mem, []nullT{null, null, null, null, null}, mask),
+ },
+ []array.Interface{
+ arrayOf(mem, []nullT{null, null, null, null, null}, mask),
+ },
+ []array.Interface{
+ arrayOf(mem, []nullT{null, null, null, null, null}, mask),
+ },
+ }
+
+ defer func() {
+ for _, chunk := range chunks {
+ for _, col := range chunk {
+ col.Release()
+ }
+ }
+ }()
+
+ recs := make([]array.Record, len(chunks))
+ for i, chunk := range chunks {
+ recs[i] = array.NewRecord(schema, chunk, -1)
+ }
+
+ return recs
+}
+
func makePrimitiveRecords() []array.Record {
mem := memory.NewGoAllocator()
@@ -362,6 +406,7 @@ func makeStringsRecords() []array.Record {
}
type (
+ nullT struct{}
time32s arrow.Time32
time32ms arrow.Time32
time64ns arrow.Time64
@@ -372,6 +417,10 @@ type (
timestamp_ns arrow.Timestamp
)
+var (
+ null nullT
+)
+
func makeFixedWidthTypesRecords() []array.Record {
mem := memory.NewGoAllocator()
schema := arrow.NewSchema(
@@ -650,6 +699,9 @@ func arrayOf(mem memory.Allocator, a interface{}, valids []bool) array.Interface
}
switch a := a.(type) {
+ case []nullT:
+ return array.NewNull(len(a))
+
case []bool:
bldr := array.NewBooleanBuilder(mem)
defer bldr.Release()
diff --git a/go/arrow/internal/arrjson/arrjson.go b/go/arrow/internal/arrjson/arrjson.go
index 11cc3d0..2b2bcaf 100644
--- a/go/arrow/internal/arrjson/arrjson.go
+++ b/go/arrow/internal/arrjson/arrjson.go
@@ -65,6 +65,8 @@ type dataType struct {
func dtypeToJSON(dt arrow.DataType) dataType {
switch dt := dt.(type) {
+ case *arrow.NullType:
+ return dataType{Name: "null"}
case *arrow.BooleanType:
return dataType{Name: "bool"}
case *arrow.Int8Type:
@@ -155,6 +157,8 @@ func dtypeToJSON(dt arrow.DataType) dataType {
func dtypeFromJSON(dt dataType, children []Field) arrow.DataType {
switch dt.Name {
+ case "null":
+ return arrow.Null
case "bool":
return arrow.FixedWidthTypes.Boolean
case "int":
@@ -364,6 +368,9 @@ func arraysToJSON(schema *arrow.Schema, arrs []array.Interface) []Array {
func arrayFromJSON(mem memory.Allocator, dt arrow.DataType, arr Array) array.Interface {
switch dt := dt.(type) {
+ case *arrow.NullType:
+ return array.NewNull(arr.Count)
+
case *arrow.BooleanType:
bldr := array.NewBooleanBuilder(mem)
defer bldr.Release()
@@ -618,6 +625,12 @@ func arrayFromJSON(mem memory.Allocator, dt arrow.DataType, arr Array) array.Int
func arrayToJSON(field arrow.Field, arr array.Interface) Array {
switch arr := arr.(type) {
+ case *array.Null:
+ return Array{
+ Name: field.Name,
+ Count: arr.Len(),
+ }
+
case *array.Boolean:
return Array{
Name: field.Name,