You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by sb...@apache.org on 2019/06/13 16:22:07 UTC
[arrow] branch master updated: ARROW-4972: [Go] implement
ArrayEquals
This is an automated email from the ASF dual-hosted git repository.
sbinet pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new b21999e ARROW-4972: [Go] implement ArrayEquals
b21999e is described below
commit b21999eaf414de60bfe389dd7d4d83043982f638
Author: Sebastien Binet <bi...@cern.ch>
AuthorDate: Thu Jun 13 18:21:42 2019 +0200
ARROW-4972: [Go] implement ArrayEquals
Author: Sebastien Binet <bi...@cern.ch>
Author: alexandreyc <al...@gmail.com>
Closes #4541 from sbinet/issue-4972 and squashes the following commits:
774d8b033 <Sebastien Binet> go/arrow/array: generate code for arrayEqualXXX
7f8a970f1 <Sebastien Binet> go/arrow/array: add 0-size array test
56de5313b <Sebastien Binet> go/arrow/array: add test for masked out array
4bb2cc781 <Sebastien Binet> go/arrow/array: add nullarray test
0d8b73022 <Sebastien Binet> go/arrow/array: rename xxxEquals into xxxEqual for consistency with Go stdlib
3cb6475a3 <Sebastien Binet> go/arrow/array: implement ArrayEquals for all array implementations
205855ba3 <Sebastien Binet> go/arrow/{ipc/cmd/arrow-cat,internal/arrdata}: extend struct array testdata
7792937c0 <alexandreyc> ARROW-4972: implement ArrayEqual
---
go/arrow/array/binary.go | 17 +++
go/arrow/array/boolean.go | 12 ++
go/arrow/array/compare.go | 143 +++++++++++++++++++++++
go/arrow/array/compare_test.go | 199 ++++++++++++++++++++++++++++++++
go/arrow/array/fixed_size_list.go | 31 ++++-
go/arrow/array/fixedsize_binary.go | 13 +++
go/arrow/array/float16.go | 12 ++
go/arrow/array/list.go | 31 ++++-
go/arrow/array/numeric.gen.go | 180 +++++++++++++++++++++++++++++
go/arrow/array/numeric.gen.go.tmpl | 13 +++
go/arrow/array/string.go | 12 ++
go/arrow/array/struct.go | 10 ++
go/arrow/internal/arrdata/arrdata.go | 60 ++++++++--
go/arrow/ipc/cmd/arrow-cat/main_test.go | 12 +-
14 files changed, 719 insertions(+), 26 deletions(-)
diff --git a/go/arrow/array/binary.go b/go/arrow/array/binary.go
index 474a35e..ed58910 100644
--- a/go/arrow/array/binary.go
+++ b/go/arrow/array/binary.go
@@ -17,6 +17,7 @@
package array
import (
+ "bytes"
"fmt"
"strings"
"unsafe"
@@ -115,3 +116,19 @@ func (a *Binary) setData(data *Data) {
a.valueOffsets = arrow.Int32Traits.CastFromBytes(valueOffsets.Bytes())
}
}
+
+func arrayEqualBinary(left, right *Binary) bool {
+ for i := 0; i < left.Len(); i++ {
+ if left.IsNull(i) {
+ continue
+ }
+ if bytes.Compare(left.Value(i), right.Value(i)) != 0 {
+ return false
+ }
+ }
+ return true
+}
+
+var (
+ _ Interface = (*Binary)(nil)
+)
diff --git a/go/arrow/array/boolean.go b/go/arrow/array/boolean.go
index b8bcf24..5095b17 100644
--- a/go/arrow/array/boolean.go
+++ b/go/arrow/array/boolean.go
@@ -78,6 +78,18 @@ func (a *Boolean) setData(data *Data) {
}
}
+func arrayEqualBoolean(left, right *Boolean) bool {
+ for i := 0; i < left.Len(); i++ {
+ if left.IsNull(i) {
+ continue
+ }
+ if left.Value(i) != right.Value(i) {
+ return false
+ }
+ }
+ return true
+}
+
var (
_ Interface = (*Boolean)(nil)
)
diff --git a/go/arrow/array/compare.go b/go/arrow/array/compare.go
new file mode 100644
index 0000000..570e9a4
--- /dev/null
+++ b/go/arrow/array/compare.go
@@ -0,0 +1,143 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array
+
+import (
+ "github.com/apache/arrow/go/arrow"
+ "github.com/pkg/errors"
+)
+
+// ArrayEqual reports whether the two provided arrays are equal.
+func ArrayEqual(left, right Interface) bool {
+ switch {
+ case !baseArrayEqual(left, right):
+ return false
+ case left.Len() == 0:
+ return true
+ case left.NullN() == left.Len():
+ return true
+ }
+
+ // at this point, we know both arrays have same type, same length, same number of nulls
+ // and nulls at the same place.
+ // compare the values.
+
+ switch l := left.(type) {
+ case *Null:
+ return true
+ case *Boolean:
+ r := right.(*Boolean)
+ return arrayEqualBoolean(l, r)
+ case *FixedSizeBinary:
+ r := right.(*FixedSizeBinary)
+ return arrayEqualFixedSizeBinary(l, r)
+ case *Binary:
+ r := right.(*Binary)
+ return arrayEqualBinary(l, r)
+ case *String:
+ r := right.(*String)
+ return arrayEqualString(l, r)
+ case *Int8:
+ r := right.(*Int8)
+ return arrayEqualInt8(l, r)
+ case *Int16:
+ r := right.(*Int16)
+ return arrayEqualInt16(l, r)
+ case *Int32:
+ r := right.(*Int32)
+ return arrayEqualInt32(l, r)
+ case *Int64:
+ r := right.(*Int64)
+ return arrayEqualInt64(l, r)
+ case *Uint8:
+ r := right.(*Uint8)
+ return arrayEqualUint8(l, r)
+ case *Uint16:
+ r := right.(*Uint16)
+ return arrayEqualUint16(l, r)
+ case *Uint32:
+ r := right.(*Uint32)
+ return arrayEqualUint32(l, r)
+ case *Uint64:
+ r := right.(*Uint64)
+ return arrayEqualUint64(l, r)
+ case *Float16:
+ r := right.(*Float16)
+ return arrayEqualFloat16(l, r)
+ case *Float32:
+ r := right.(*Float32)
+ return arrayEqualFloat32(l, r)
+ case *Float64:
+ r := right.(*Float64)
+ return arrayEqualFloat64(l, r)
+ case *Date32:
+ r := right.(*Date32)
+ return arrayEqualDate32(l, r)
+ case *Date64:
+ r := right.(*Date64)
+ return arrayEqualDate64(l, r)
+ case *Time32:
+ r := right.(*Time32)
+ return arrayEqualTime32(l, r)
+ case *Time64:
+ r := right.(*Time64)
+ return arrayEqualTime64(l, r)
+ case *Timestamp:
+ r := right.(*Timestamp)
+ return arrayEqualTimestamp(l, r)
+ case *List:
+ r := right.(*List)
+ return arrayEqualList(l, r)
+ case *FixedSizeList:
+ r := right.(*FixedSizeList)
+ return arrayEqualFixedSizeList(l, r)
+ case *Struct:
+ r := right.(*Struct)
+ return arrayEqualStruct(l, r)
+
+ default:
+ panic(errors.Errorf("arrow/array: unknown array type %T", l))
+ }
+}
+
+func baseArrayEqual(left, right Interface) bool {
+ switch {
+ case left.Len() != right.Len():
+ return false
+ case left.NullN() != right.NullN():
+ return false
+ case !arrow.TypeEquals(left.DataType(), right.DataType()): // We do not check for metadata as in the C++ implementation.
+ return false
+ case !validityBitmapEqual(left, right):
+ return false
+ }
+ return true
+}
+
+func validityBitmapEqual(left, right Interface) bool {
+ // TODO(alexandreyc): make it faster by comparing byte slices of the validity bitmap?
+ n := left.Len()
+ if n != right.Len() {
+ return false
+ }
+ for i := 0; i < n; i++ {
+ if left.IsNull(i) != right.IsNull(i) {
+ return false
+ }
+ }
+ return true
+}
diff --git a/go/arrow/array/compare_test.go b/go/arrow/array/compare_test.go
new file mode 100644
index 0000000..012611f
--- /dev/null
+++ b/go/arrow/array/compare_test.go
@@ -0,0 +1,199 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array_test
+
+import (
+ "testing"
+
+ "github.com/apache/arrow/go/arrow/array"
+ "github.com/apache/arrow/go/arrow/internal/arrdata"
+ "github.com/apache/arrow/go/arrow/memory"
+)
+
+func TestArrayEqual(t *testing.T) {
+ for name, recs := range arrdata.Records {
+ t.Run(name, func(t *testing.T) {
+ rec := recs[0]
+ schema := rec.Schema()
+ for i, col := range rec.Columns() {
+ t.Run(schema.Field(i).Name, func(t *testing.T) {
+ arr := col
+ if !array.ArrayEqual(arr, arr) {
+ t.Fatalf("identical arrays should compare equal:\narray=%v", arr)
+ }
+ sub1 := array.NewSlice(arr, 1, int64(arr.Len()))
+ defer sub1.Release()
+
+ sub2 := array.NewSlice(arr, 0, int64(arr.Len()-1))
+ defer sub2.Release()
+
+ if array.ArrayEqual(sub1, sub2) {
+ t.Fatalf("non-identical arrays should not compare equal:\nsub1=%v\nsub2=%v\narrf=%v\n", sub1, sub2, arr)
+ }
+ })
+ }
+ })
+ }
+}
+
+func TestArrayEqualBaseArray(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ b1 := array.NewBooleanBuilder(mem)
+ defer b1.Release()
+ b1.Append(true)
+ a1 := b1.NewBooleanArray()
+ defer a1.Release()
+
+ b2 := array.NewBooleanBuilder(mem)
+ defer b2.Release()
+ a2 := b2.NewBooleanArray()
+ defer a2.Release()
+
+ if array.ArrayEqual(a1, a2) {
+ t.Errorf("two arrays with different lengths must not be equal")
+ }
+
+ b3 := array.NewBooleanBuilder(mem)
+ defer b3.Release()
+ b3.AppendNull()
+ a3 := b3.NewBooleanArray()
+ defer a3.Release()
+
+ if array.ArrayEqual(a1, a3) {
+ t.Errorf("two arrays with different number of null values must not be equal")
+ }
+
+ b4 := array.NewInt32Builder(mem)
+ defer b4.Release()
+ b4.Append(0)
+ a4 := b4.NewInt32Array()
+ defer a4.Release()
+
+ if array.ArrayEqual(a1, a4) {
+ t.Errorf("two arrays with different types must not be equal")
+ }
+
+ b5 := array.NewBooleanBuilder(mem)
+ defer b5.Release()
+ b5.AppendNull()
+ b5.Append(true)
+ a5 := b5.NewBooleanArray()
+ defer a5.Release()
+ b1.AppendNull()
+
+ if array.ArrayEqual(a1, a5) {
+ t.Errorf("two arrays with different validity bitmaps must not be equal")
+ }
+}
+
+func TestArrayEqualNull(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ null := array.NewNull(0)
+ defer null.Release()
+
+ if !array.ArrayEqual(null, null) {
+ t.Fatalf("identical arrays should compare equal")
+ }
+
+ n0 := array.NewNull(10)
+ defer n0.Release()
+
+ n1 := array.NewNull(10)
+ defer n1.Release()
+
+ if !array.ArrayEqual(n0, n0) {
+ t.Fatalf("identical arrays should compare equal")
+ }
+ if !array.ArrayEqual(n1, n1) {
+ t.Fatalf("identical arrays should compare equal")
+ }
+ if !array.ArrayEqual(n0, n1) || !array.ArrayEqual(n1, n0) {
+ t.Fatalf("n0 and n1 should compare equal")
+ }
+
+ sub07 := array.NewSlice(n0, 0, 7)
+ defer sub07.Release()
+ sub08 := array.NewSlice(n0, 0, 8)
+ defer sub08.Release()
+ sub19 := array.NewSlice(n0, 1, 9)
+ defer sub19.Release()
+
+ if !array.ArrayEqual(sub08, sub19) {
+ t.Fatalf("sub08 and sub19 should compare equal")
+ }
+
+ if array.ArrayEqual(sub08, sub07) {
+ t.Fatalf("sub08 and sub07 should not compare equal")
+ }
+}
+
+func TestArrayEqualMaskedArray(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ ab := array.NewInt32Builder(mem)
+ defer ab.Release()
+
+ valids := []bool{false, false, false, false}
+ ab.AppendValues([]int32{1, 2, 0, 4}, valids)
+
+ a1 := ab.NewInt32Array()
+ defer a1.Release()
+
+ ab.AppendValues([]int32{1, 2, 3, 4}, valids)
+ a2 := ab.NewInt32Array()
+ defer a2.Release()
+
+ if !array.ArrayEqual(a1, a1) || !array.ArrayEqual(a2, a2) {
+ t.Errorf("an array must be equal to itself")
+ }
+
+ if !array.ArrayEqual(a1, a2) {
+ t.Errorf("%v must be equal to %v", a1, a2)
+ }
+}
+
+func TestArrayEqualDifferentMaskedValues(t *testing.T) {
+ // test 2 int32 arrays, with same nulls (but different masked values) compare equal.
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ ab := array.NewInt32Builder(mem)
+ defer ab.Release()
+
+ valids := []bool{true, true, false, true}
+ ab.AppendValues([]int32{1, 2, 0, 4}, valids)
+
+ a1 := ab.NewInt32Array()
+ defer a1.Release()
+
+ ab.AppendValues([]int32{1, 2, 3, 4}, valids)
+ a2 := ab.NewInt32Array()
+ defer a2.Release()
+
+ if !array.ArrayEqual(a1, a1) || !array.ArrayEqual(a2, a2) {
+ t.Errorf("an array must be equal to itself")
+ }
+
+ if !array.ArrayEqual(a1, a2) {
+ t.Errorf("%v must be equal to %v", a1, a2)
+ }
+}
diff --git a/go/arrow/array/fixed_size_list.go b/go/arrow/array/fixed_size_list.go
index 9816e65..1145e4e 100644
--- a/go/arrow/array/fixed_size_list.go
+++ b/go/arrow/array/fixed_size_list.go
@@ -56,10 +56,7 @@ func (a *FixedSizeList) String() string {
o.WriteString("(null)")
continue
}
- j := i + a.array.data.offset
- beg := int64(a.offsets[j])
- end := int64(a.offsets[j+1])
- sub := NewSlice(a.values, beg, end)
+ sub := a.newListValue(i)
fmt.Fprintf(o, "%v", sub)
sub.Release()
}
@@ -67,6 +64,13 @@ func (a *FixedSizeList) String() string {
return o.String()
}
+func (a *FixedSizeList) newListValue(i int) Interface {
+ j := i + a.array.data.offset
+ beg := int64(a.offsets[j])
+ end := int64(a.offsets[j+1])
+ return NewSlice(a.values, beg, end)
+}
+
func (a *FixedSizeList) setData(data *Data) {
a.array.setData(data)
vals := data.buffers[1]
@@ -76,6 +80,25 @@ func (a *FixedSizeList) setData(data *Data) {
a.values = MakeFromData(data.childData[0])
}
+func arrayEqualFixedSizeList(left, right *FixedSizeList) bool {
+ for i := 0; i < left.Len(); i++ {
+ if left.IsNull(i) {
+ continue
+ }
+ o := func() bool {
+ l := left.newListValue(i)
+ defer l.Release()
+ r := right.newListValue(i)
+ defer r.Release()
+ return ArrayEqual(l, r)
+ }()
+ if !o {
+ return false
+ }
+ }
+ return true
+}
+
// Len returns the number of elements in the array.
func (a *FixedSizeList) Len() int { return a.array.Len() }
diff --git a/go/arrow/array/fixedsize_binary.go b/go/arrow/array/fixedsize_binary.go
index 5134302..502fb99 100644
--- a/go/arrow/array/fixedsize_binary.go
+++ b/go/arrow/array/fixedsize_binary.go
@@ -17,6 +17,7 @@
package array
import (
+ "bytes"
"fmt"
"strings"
@@ -77,6 +78,18 @@ func (a *FixedSizeBinary) setData(data *Data) {
}
+func arrayEqualFixedSizeBinary(left, right *FixedSizeBinary) bool {
+ for i := 0; i < left.Len(); i++ {
+ if left.IsNull(i) {
+ continue
+ }
+ if bytes.Compare(left.Value(i), right.Value(i)) != 0 {
+ return false
+ }
+ }
+ return true
+}
+
var (
_ Interface = (*FixedSizeBinary)(nil)
)
diff --git a/go/arrow/array/float16.go b/go/arrow/array/float16.go
index 02f85de..931e2d9 100644
--- a/go/arrow/array/float16.go
+++ b/go/arrow/array/float16.go
@@ -70,6 +70,18 @@ func (a *Float16) setData(data *Data) {
}
}
+func arrayEqualFloat16(left, right *Float16) bool {
+ for i := 0; i < left.Len(); i++ {
+ if left.IsNull(i) {
+ continue
+ }
+ if left.Value(i) != right.Value(i) {
+ return false
+ }
+ }
+ return true
+}
+
var (
_ Interface = (*Float16)(nil)
)
diff --git a/go/arrow/array/list.go b/go/arrow/array/list.go
index 1b91176..2fccdea 100644
--- a/go/arrow/array/list.go
+++ b/go/arrow/array/list.go
@@ -55,10 +55,7 @@ func (a *List) String() string {
o.WriteString("(null)")
continue
}
- j := i + a.array.data.offset
- beg := int64(a.offsets[j])
- end := int64(a.offsets[j+1])
- sub := NewSlice(a.values, beg, end)
+ sub := a.newListValue(i)
fmt.Fprintf(o, "%v", sub)
sub.Release()
}
@@ -66,6 +63,13 @@ func (a *List) String() string {
return o.String()
}
+func (a *List) newListValue(i int) Interface {
+ j := i + a.array.data.offset
+ beg := int64(a.offsets[j])
+ end := int64(a.offsets[j+1])
+ return NewSlice(a.values, beg, end)
+}
+
func (a *List) setData(data *Data) {
a.array.setData(data)
vals := data.buffers[1]
@@ -75,6 +79,25 @@ func (a *List) setData(data *Data) {
a.values = MakeFromData(data.childData[0])
}
+func arrayEqualList(left, right *List) bool {
+ for i := 0; i < left.Len(); i++ {
+ if left.IsNull(i) {
+ continue
+ }
+ o := func() bool {
+ l := left.newListValue(i)
+ defer l.Release()
+ r := right.newListValue(i)
+ defer r.Release()
+ return ArrayEqual(l, r)
+ }()
+ if !o {
+ return false
+ }
+ }
+ return true
+}
+
// Len returns the number of elements in the array.
func (a *List) Len() int { return a.array.Len() }
diff --git a/go/arrow/array/numeric.gen.go b/go/arrow/array/numeric.gen.go
index 1fb8257..d72d7d0 100644
--- a/go/arrow/array/numeric.gen.go
+++ b/go/arrow/array/numeric.gen.go
@@ -70,6 +70,18 @@ func (a *Int64) setData(data *Data) {
}
}
+func arrayEqualInt64(left, right *Int64) bool {
+ for i := 0; i < left.Len(); i++ {
+ if left.IsNull(i) {
+ continue
+ }
+ if left.Value(i) != right.Value(i) {
+ return false
+ }
+ }
+ return true
+}
+
// A type which represents an immutable sequence of uint64 values.
type Uint64 struct {
array
@@ -115,6 +127,18 @@ func (a *Uint64) setData(data *Data) {
}
}
+func arrayEqualUint64(left, right *Uint64) bool {
+ for i := 0; i < left.Len(); i++ {
+ if left.IsNull(i) {
+ continue
+ }
+ if left.Value(i) != right.Value(i) {
+ return false
+ }
+ }
+ return true
+}
+
// A type which represents an immutable sequence of float64 values.
type Float64 struct {
array
@@ -160,6 +184,18 @@ func (a *Float64) setData(data *Data) {
}
}
+func arrayEqualFloat64(left, right *Float64) bool {
+ for i := 0; i < left.Len(); i++ {
+ if left.IsNull(i) {
+ continue
+ }
+ if left.Value(i) != right.Value(i) {
+ return false
+ }
+ }
+ return true
+}
+
// A type which represents an immutable sequence of int32 values.
type Int32 struct {
array
@@ -205,6 +241,18 @@ func (a *Int32) setData(data *Data) {
}
}
+func arrayEqualInt32(left, right *Int32) bool {
+ for i := 0; i < left.Len(); i++ {
+ if left.IsNull(i) {
+ continue
+ }
+ if left.Value(i) != right.Value(i) {
+ return false
+ }
+ }
+ return true
+}
+
// A type which represents an immutable sequence of uint32 values.
type Uint32 struct {
array
@@ -250,6 +298,18 @@ func (a *Uint32) setData(data *Data) {
}
}
+func arrayEqualUint32(left, right *Uint32) bool {
+ for i := 0; i < left.Len(); i++ {
+ if left.IsNull(i) {
+ continue
+ }
+ if left.Value(i) != right.Value(i) {
+ return false
+ }
+ }
+ return true
+}
+
// A type which represents an immutable sequence of float32 values.
type Float32 struct {
array
@@ -295,6 +355,18 @@ func (a *Float32) setData(data *Data) {
}
}
+func arrayEqualFloat32(left, right *Float32) bool {
+ for i := 0; i < left.Len(); i++ {
+ if left.IsNull(i) {
+ continue
+ }
+ if left.Value(i) != right.Value(i) {
+ return false
+ }
+ }
+ return true
+}
+
// A type which represents an immutable sequence of int16 values.
type Int16 struct {
array
@@ -340,6 +412,18 @@ func (a *Int16) setData(data *Data) {
}
}
+func arrayEqualInt16(left, right *Int16) bool {
+ for i := 0; i < left.Len(); i++ {
+ if left.IsNull(i) {
+ continue
+ }
+ if left.Value(i) != right.Value(i) {
+ return false
+ }
+ }
+ return true
+}
+
// A type which represents an immutable sequence of uint16 values.
type Uint16 struct {
array
@@ -385,6 +469,18 @@ func (a *Uint16) setData(data *Data) {
}
}
+func arrayEqualUint16(left, right *Uint16) bool {
+ for i := 0; i < left.Len(); i++ {
+ if left.IsNull(i) {
+ continue
+ }
+ if left.Value(i) != right.Value(i) {
+ return false
+ }
+ }
+ return true
+}
+
// A type which represents an immutable sequence of int8 values.
type Int8 struct {
array
@@ -430,6 +526,18 @@ func (a *Int8) setData(data *Data) {
}
}
+func arrayEqualInt8(left, right *Int8) bool {
+ for i := 0; i < left.Len(); i++ {
+ if left.IsNull(i) {
+ continue
+ }
+ if left.Value(i) != right.Value(i) {
+ return false
+ }
+ }
+ return true
+}
+
// A type which represents an immutable sequence of uint8 values.
type Uint8 struct {
array
@@ -475,6 +583,18 @@ func (a *Uint8) setData(data *Data) {
}
}
+func arrayEqualUint8(left, right *Uint8) bool {
+ for i := 0; i < left.Len(); i++ {
+ if left.IsNull(i) {
+ continue
+ }
+ if left.Value(i) != right.Value(i) {
+ return false
+ }
+ }
+ return true
+}
+
// A type which represents an immutable sequence of arrow.Timestamp values.
type Timestamp struct {
array
@@ -520,6 +640,18 @@ func (a *Timestamp) setData(data *Data) {
}
}
+func arrayEqualTimestamp(left, right *Timestamp) bool {
+ for i := 0; i < left.Len(); i++ {
+ if left.IsNull(i) {
+ continue
+ }
+ if left.Value(i) != right.Value(i) {
+ return false
+ }
+ }
+ return true
+}
+
// A type which represents an immutable sequence of arrow.Time32 values.
type Time32 struct {
array
@@ -565,6 +697,18 @@ func (a *Time32) setData(data *Data) {
}
}
+func arrayEqualTime32(left, right *Time32) bool {
+ for i := 0; i < left.Len(); i++ {
+ if left.IsNull(i) {
+ continue
+ }
+ if left.Value(i) != right.Value(i) {
+ return false
+ }
+ }
+ return true
+}
+
// A type which represents an immutable sequence of arrow.Time64 values.
type Time64 struct {
array
@@ -610,6 +754,18 @@ func (a *Time64) setData(data *Data) {
}
}
+func arrayEqualTime64(left, right *Time64) bool {
+ for i := 0; i < left.Len(); i++ {
+ if left.IsNull(i) {
+ continue
+ }
+ if left.Value(i) != right.Value(i) {
+ return false
+ }
+ }
+ return true
+}
+
// A type which represents an immutable sequence of arrow.Date32 values.
type Date32 struct {
array
@@ -655,6 +811,18 @@ func (a *Date32) setData(data *Data) {
}
}
+func arrayEqualDate32(left, right *Date32) bool {
+ for i := 0; i < left.Len(); i++ {
+ if left.IsNull(i) {
+ continue
+ }
+ if left.Value(i) != right.Value(i) {
+ return false
+ }
+ }
+ return true
+}
+
// A type which represents an immutable sequence of arrow.Date64 values.
type Date64 struct {
array
@@ -699,3 +867,15 @@ func (a *Date64) setData(data *Data) {
a.values = a.values[beg:end]
}
}
+
+func arrayEqualDate64(left, right *Date64) bool {
+ for i := 0; i < left.Len(); i++ {
+ if left.IsNull(i) {
+ continue
+ }
+ if left.Value(i) != right.Value(i) {
+ return false
+ }
+ }
+ return true
+}
diff --git a/go/arrow/array/numeric.gen.go.tmpl b/go/arrow/array/numeric.gen.go.tmpl
index 13ea3f4..1e4a2f2 100644
--- a/go/arrow/array/numeric.gen.go.tmpl
+++ b/go/arrow/array/numeric.gen.go.tmpl
@@ -69,4 +69,17 @@ func (a *{{.Name}}) setData(data *Data) {
a.values = a.values[beg:end]
}
}
+
+func arrayEqual{{.Name}}(left, right *{{.Name}}) bool {
+ for i := 0; i < left.Len(); i++ {
+ if left.IsNull(i) {
+ continue
+ }
+ if left.Value(i) != right.Value(i) {
+ return false
+ }
+ }
+ return true
+}
+
{{end}}
diff --git a/go/arrow/array/string.go b/go/arrow/array/string.go
index 8356d02..b7e607c 100644
--- a/go/arrow/array/string.go
+++ b/go/arrow/array/string.go
@@ -87,6 +87,18 @@ func (a *String) setData(data *Data) {
}
}
+func arrayEqualString(left, right *String) bool {
+ for i := 0; i < left.Len(); i++ {
+ if left.IsNull(i) {
+ continue
+ }
+ if left.Value(i) != right.Value(i) {
+ return false
+ }
+ }
+ return true
+}
+
// A StringBuilder is used to build a String array using the Append methods.
type StringBuilder struct {
builder *BinaryBuilder
diff --git a/go/arrow/array/struct.go b/go/arrow/array/struct.go
index 3e52009..b70d953 100644
--- a/go/arrow/array/struct.go
+++ b/go/arrow/array/struct.go
@@ -65,6 +65,16 @@ func (a *Struct) setData(data *Data) {
}
}
+func arrayEqualStruct(left, right *Struct) bool {
+ for i, lf := range left.fields {
+ rf := right.fields[i]
+ if !ArrayEqual(lf, rf) {
+ return false
+ }
+ }
+ return true
+}
+
func (a *Struct) Retain() {
a.array.Retain()
for _, f := range a.fields {
diff --git a/go/arrow/internal/arrdata/arrdata.go b/go/arrow/internal/arrdata/arrdata.go
index b7daf6f..e76d68a 100644
--- a/go/arrow/internal/arrdata/arrdata.go
+++ b/go/arrow/internal/arrdata/arrdata.go
@@ -146,16 +146,52 @@ func makeStructsRecords() []array.Record {
mask := []bool{true, false, false, true, true, true, false, true}
chunks := [][]array.Interface{
[]array.Interface{
- structOf(mem, dtype, []array.Interface{
- arrayOf(mem, []int32{-1, -2, -3, -4, -5}, mask[:5]),
- arrayOf(mem, []string{"111", "222", "333", "444", "555"}, mask[:5]),
- }, []bool{true}),
+ structOf(mem, dtype, [][]array.Interface{
+ []array.Interface{
+ arrayOf(mem, []int32{-1, -2, -3, -4, -5}, mask[:5]),
+ arrayOf(mem, []string{"111", "222", "333", "444", "555"}, mask[:5]),
+ },
+ []array.Interface{
+ arrayOf(mem, []int32{-11, -12, -13, -14, -15}, mask[:5]),
+ arrayOf(mem, []string{"1111", "1222", "1333", "1444", "1555"}, mask[:5]),
+ },
+ []array.Interface{
+ arrayOf(mem, []int32{-21, -22, -23, -24, -25}, mask[:5]),
+ arrayOf(mem, []string{"2111", "2222", "2333", "2444", "2555"}, mask[:5]),
+ },
+ []array.Interface{
+ arrayOf(mem, []int32{-31, -32, -33, -34, -35}, mask[:5]),
+ arrayOf(mem, []string{"3111", "3222", "3333", "3444", "3555"}, mask[:5]),
+ },
+ []array.Interface{
+ arrayOf(mem, []int32{-41, -42, -43, -44, -45}, mask[:5]),
+ arrayOf(mem, []string{"4111", "4222", "4333", "4444", "4555"}, mask[:5]),
+ },
+ }, []bool{true, false, true, true, true}),
},
[]array.Interface{
- structOf(mem, dtype, []array.Interface{
- arrayOf(mem, []int32{-11, -12, -13, -14, -15, -16, -17, -18}, mask),
- arrayOf(mem, []string{"1", "2", "3", "4", "5", "6", "7", "8"}, mask),
- }, []bool{true}),
+ structOf(mem, dtype, [][]array.Interface{
+ []array.Interface{
+ arrayOf(mem, []int32{1, 2, 3, 4, 5}, mask[:5]),
+ arrayOf(mem, []string{"-111", "-222", "-333", "-444", "-555"}, mask[:5]),
+ },
+ []array.Interface{
+ arrayOf(mem, []int32{11, 12, 13, 14, 15}, mask[:5]),
+ arrayOf(mem, []string{"-1111", "-1222", "-1333", "-1444", "-1555"}, mask[:5]),
+ },
+ []array.Interface{
+ arrayOf(mem, []int32{21, 22, 23, 24, 25}, mask[:5]),
+ arrayOf(mem, []string{"-2111", "-2222", "-2333", "-2444", "-2555"}, mask[:5]),
+ },
+ []array.Interface{
+ arrayOf(mem, []int32{31, 32, 33, 34, 35}, mask[:5]),
+ arrayOf(mem, []string{"-3111", "-3222", "-3333", "-3444", "-3555"}, mask[:5]),
+ },
+ []array.Interface{
+ arrayOf(mem, []int32{41, 42, 43, 44, 45}, mask[:5]),
+ arrayOf(mem, []string{"-4111", "-4222", "-4333", "-4444", "-4555"}, mask[:5]),
+ },
+ }, []bool{true, false, false, true, true}),
},
}
@@ -670,7 +706,7 @@ func fixedSizeListOf(mem memory.Allocator, n int32, values []array.Interface, va
return bldr.NewListArray()
}
-func structOf(mem memory.Allocator, dtype *arrow.StructType, fields []array.Interface, valids []bool) *array.Struct {
+func structOf(mem memory.Allocator, dtype *arrow.StructType, fields [][]array.Interface, valids []bool) *array.Struct {
if mem == nil {
mem = memory.NewGoAllocator()
}
@@ -679,17 +715,17 @@ func structOf(mem memory.Allocator, dtype *arrow.StructType, fields []array.Inte
defer bldr.Release()
if valids == nil {
- valids = make([]bool, fields[0].Len())
+ valids = make([]bool, fields[0][0].Len())
for i := range valids {
valids[i] = true
}
}
- for _, valid := range valids {
+ for i, valid := range valids {
bldr.Append(valid)
for j := range dtype.Fields() {
fbldr := bldr.FieldBuilder(j)
- buildArray(fbldr, fields[j])
+ buildArray(fbldr, fields[i][j])
}
}
diff --git a/go/arrow/ipc/cmd/arrow-cat/main_test.go b/go/arrow/ipc/cmd/arrow-cat/main_test.go
index 3f9c3e7..8f8e382 100644
--- a/go/arrow/ipc/cmd/arrow-cat/main_test.go
+++ b/go/arrow/ipc/cmd/arrow-cat/main_test.go
@@ -78,9 +78,9 @@ record 3...
{
name: "structs",
want: `record 1...
- col[0] "struct_nullable": {[-1 (null) (null) -4 -5] ["111" (null) (null) "444" "555"]}
+ col[0] "struct_nullable": {[-1 (null) (null) -4 -5 (null) -11 (null) (null) -14 -15 -21 (null) (null) -24 -25 -31 (null) (null) -34 -35 -41 (null) (null) -44 -45] ["111" (null) (null) "444" "555" (null) "1111" (null) (null) "1444" "1555" "2111" (null) (null) "2444" "2555" "3111" (null) (null) "3444" "3555" "4111" (null) (null) "4444" "4555"]}
record 2...
- col[0] "struct_nullable": {[-11 (null) (null) -14 -15 -16 (null) -18] ["1" (null) (null) "4" "5" "6" (null) "8"]}
+ col[0] "struct_nullable": {[1 (null) (null) 4 5 (null) 11 (null) (null) 14 15 (null) 21 (null) (null) 24 25 31 (null) (null) 34 35 41 (null) (null) 44 45] ["-111" (null) (null) "-444" "-555" (null) "-1111" (null) (null) "-1444" "-1555" (null) "-2111" (null) (null) "-2444" "-2555" "-3111" (null) (null) "-3444" "-3555" "-4111" (null) (null) "-4444" "-4555"]}
`,
},
{
@@ -306,18 +306,18 @@ record 3/3...
stream: true,
name: "structs",
want: `record 1...
- col[0] "struct_nullable": {[-1 (null) (null) -4 -5] ["111" (null) (null) "444" "555"]}
+ col[0] "struct_nullable": {[-1 (null) (null) -4 -5 (null) -11 (null) (null) -14 -15 -21 (null) (null) -24 -25 -31 (null) (null) -34 -35 -41 (null) (null) -44 -45] ["111" (null) (null) "444" "555" (null) "1111" (null) (null) "1444" "1555" "2111" (null) (null) "2444" "2555" "3111" (null) (null) "3444" "3555" "4111" (null) (null) "4444" "4555"]}
record 2...
- col[0] "struct_nullable": {[-11 (null) (null) -14 -15 -16 (null) -18] ["1" (null) (null) "4" "5" "6" (null) "8"]}
+ col[0] "struct_nullable": {[1 (null) (null) 4 5 (null) 11 (null) (null) 14 15 (null) 21 (null) (null) 24 25 31 (null) (null) 34 35 41 (null) (null) 44 45] ["-111" (null) (null) "-444" "-555" (null) "-1111" (null) (null) "-1444" "-1555" (null) "-2111" (null) (null) "-2444" "-2555" "-3111" (null) (null) "-3444" "-3555" "-4111" (null) (null) "-4444" "-4555"]}
`,
},
{
name: "structs",
want: `version: V4
record 1/2...
- col[0] "struct_nullable": {[-1 (null) (null) -4 -5] ["111" (null) (null) "444" "555"]}
+ col[0] "struct_nullable": {[-1 (null) (null) -4 -5 (null) -11 (null) (null) -14 -15 -21 (null) (null) -24 -25 -31 (null) (null) -34 -35 -41 (null) (null) -44 -45] ["111" (null) (null) "444" "555" (null) "1111" (null) (null) "1444" "1555" "2111" (null) (null) "2444" "2555" "3111" (null) (null) "3444" "3555" "4111" (null) (null) "4444" "4555"]}
record 2/2...
- col[0] "struct_nullable": {[-11 (null) (null) -14 -15 -16 (null) -18] ["1" (null) (null) "4" "5" "6" (null) "8"]}
+ col[0] "struct_nullable": {[1 (null) (null) 4 5 (null) 11 (null) (null) 14 15 (null) 21 (null) (null) 24 25 31 (null) (null) 34 35 41 (null) (null) 44 45] ["-111" (null) (null) "-444" "-555" (null) "-1111" (null) (null) "-1444" "-1555" (null) "-2111" (null) (null) "-2444" "-2555" "-3111" (null) (null) "-3444" "-3555" "-4111" (null) (null) "-4444" "-4555"]}
`,
},
{