You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2018/11/25 21:45:25 UTC
[arrow] branch master updated: ARROW-3677: [Go] Add fixed-length
binary builder and array
This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new d2256c5 ARROW-3677: [Go] Add fixed-length binary builder and array
d2256c5 is described below
commit d2256c5bdb25e46c53b3a70e01f0086898d9e021
Author: alexandreyc <al...@gmail.com>
AuthorDate: Sun Nov 25 16:45:17 2018 -0500
ARROW-3677: [Go] Add fixed-length binary builder and array
Author: alexandreyc <al...@gmail.com>
Closes #3012 from alexandreyc/master and squashes the following commits:
de15e7e8d <alexandreyc> add fixed size binary, time32 and time64 to newBuilder
0816da486 <alexandreyc> update field dtype of FixedSizeBinaryBuilder to be a pointer
3ada4f714 <alexandreyc> remove useless field byteWidth on FixedSizeBinary
ef8d98262 <alexandreyc> add time32, time64 and fixed size binary test for array api
51094761a <alexandreyc> add check that FixedSizeBinary implements Interface
14477e7ce <alexandreyc> update binary and fixed-size binary array test
b53921c14 <alexandreyc> remove old commented code
eaa7e0d7a <alexandreyc> add check that FixedSizeBinaryType implements FixedWidthDataType
4682cdb95 <alexandreyc> add license to new files
0a8654d30 <alexandreyc> remove log message from binary array test
25303ed18 <alexandreyc> add fixed-size binary builder and array
0e464373d <alexandreyc> add test for binary array
---
go/arrow/array/array.go | 2 +-
go/arrow/array/array_test.go | 3 +
go/arrow/array/binary_test.go | 64 +++++++++
go/arrow/array/builder.go | 6 +
go/arrow/array/fixedsize_binary.go | 63 +++++++++
go/arrow/array/fixedsize_binary_test.go | 65 +++++++++
go/arrow/array/fixedsize_binarybuilder.go | 174 +++++++++++++++++++++++++
go/arrow/array/fixedsize_binarybuilder_test.go | 70 ++++++++++
go/arrow/datatype_fixedwidth.go | 10 ++
9 files changed, 456 insertions(+), 1 deletion(-)
diff --git a/go/arrow/array/array.go b/go/arrow/array/array.go
index a225693..b188dcd 100644
--- a/go/arrow/array/array.go
+++ b/go/arrow/array/array.go
@@ -179,7 +179,7 @@ func init() {
arrow.FLOAT64: func(data *Data) Interface { return NewFloat64Data(data) },
arrow.STRING: func(data *Data) Interface { return NewStringData(data) },
arrow.BINARY: func(data *Data) Interface { return NewBinaryData(data) },
- arrow.FIXED_SIZE_BINARY: unsupportedArrayType,
+ arrow.FIXED_SIZE_BINARY: func(data *Data) Interface { return NewFixedSizeBinaryData(data) },
arrow.DATE32: unsupportedArrayType,
arrow.DATE64: unsupportedArrayType,
arrow.TIMESTAMP: func(data *Data) Interface { return NewTimestampData(data) },
diff --git a/go/arrow/array/array_test.go b/go/arrow/array/array_test.go
index 02f588a..eb97d9e 100644
--- a/go/arrow/array/array_test.go
+++ b/go/arrow/array/array_test.go
@@ -60,6 +60,9 @@ func TestMakeFromData(t *testing.T) {
{name: "float64", d: &testDataType{arrow.FLOAT64}},
{name: "binary", d: &testDataType{arrow.BINARY}, size: 3},
{name: "timestamp", d: &testDataType{arrow.TIMESTAMP}},
+ {name: "time32", d: &testDataType{arrow.TIME32}},
+ {name: "time64", d: &testDataType{arrow.TIME64}},
+ {name: "fixed_size_binary", d: &testDataType{arrow.FIXED_SIZE_BINARY}, size: 3},
{name: "list", d: &testDataType{arrow.LIST}, child: []*array.Data{
array.NewData(&testDataType{arrow.INT64}, 0, make([]*memory.Buffer, 4), nil, 0, 0),
diff --git a/go/arrow/array/binary_test.go b/go/arrow/array/binary_test.go
new file mode 100644
index 0000000..87d1b58
--- /dev/null
+++ b/go/arrow/array/binary_test.go
@@ -0,0 +1,64 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+
+ "github.com/apache/arrow/go/arrow"
+ "github.com/apache/arrow/go/arrow/memory"
+)
+
+func TestBinary(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ b := NewBinaryBuilder(mem, arrow.BinaryTypes.Binary)
+
+ values := [][]byte{
+ []byte("AAA"),
+ nil,
+ []byte("BBBB"),
+ }
+ valid := []bool{true, false, true}
+ b.AppendValues(values, valid)
+
+ b.Retain()
+ b.Release()
+
+ a := b.NewBinaryArray()
+ assert.Equal(t, 3, a.Len())
+ assert.Equal(t, 1, a.NullN())
+ assert.Equal(t, []byte("AAA"), a.Value(0))
+ assert.Equal(t, []byte{}, a.Value(1))
+ assert.Equal(t, []byte("BBBB"), a.Value(2))
+ a.Release()
+
+ // Test builder reset and NewArray API.
+ b.AppendValues(values, valid)
+ a = b.NewArray().(*Binary)
+ assert.Equal(t, 3, a.Len())
+ assert.Equal(t, 1, a.NullN())
+ assert.Equal(t, []byte("AAA"), a.Value(0))
+ assert.Equal(t, []byte{}, a.Value(1))
+ assert.Equal(t, []byte("BBBB"), a.Value(2))
+ a.Release()
+
+ b.Release()
+}
diff --git a/go/arrow/array/builder.go b/go/arrow/array/builder.go
index 7dd0990..c00a365 100644
--- a/go/arrow/array/builder.go
+++ b/go/arrow/array/builder.go
@@ -239,11 +239,17 @@ func newBuilder(mem memory.Allocator, dtype arrow.DataType) Builder {
case arrow.BINARY:
return NewBinaryBuilder(mem, arrow.BinaryTypes.Binary)
case arrow.FIXED_SIZE_BINARY:
+ typ := dtype.(*arrow.FixedSizeBinaryType)
+ return NewFixedSizeBinaryBuilder(mem, typ)
case arrow.DATE32:
case arrow.DATE64:
case arrow.TIMESTAMP:
case arrow.TIME32:
+ typ := dtype.(*arrow.Time32Type)
+ return NewTime32Builder(mem, typ)
case arrow.TIME64:
+ typ := dtype.(*arrow.Time64Type)
+ return NewTime64Builder(mem, typ)
case arrow.INTERVAL:
case arrow.DECIMAL:
case arrow.LIST:
diff --git a/go/arrow/array/fixedsize_binary.go b/go/arrow/array/fixedsize_binary.go
new file mode 100644
index 0000000..258f2d2
--- /dev/null
+++ b/go/arrow/array/fixedsize_binary.go
@@ -0,0 +1,63 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array
+
+import "github.com/apache/arrow/go/arrow"
+
+// A type which represents an immutable sequence of fixed-length binary strings.
+type FixedSizeBinary struct {
+ array
+ valueOffsets []int32
+ valueBytes []byte
+}
+
+// NewFixedSizeBinaryData constructs a new fixed-size binary array from data.
+func NewFixedSizeBinaryData(data *Data) *FixedSizeBinary {
+ a := &FixedSizeBinary{}
+ a.refCount = 1
+ a.setData(data)
+ return a
+}
+
+// Value returns the fixed-size slice at index i. This value should not be mutated.
+func (a *FixedSizeBinary) Value(i int) []byte {
+ return a.valueBytes[a.valueOffsets[i]:a.valueOffsets[i+1]]
+}
+
+func (a *FixedSizeBinary) ValueOffset(i int) int { return int(a.valueOffsets[i]) }
+func (a *FixedSizeBinary) ValueLen(i int) int { return int(a.valueOffsets[i+1] - a.valueOffsets[i]) }
+func (a *FixedSizeBinary) ValueOffsets() []int32 { return a.valueOffsets }
+func (a *FixedSizeBinary) ValueBytes() []byte { return a.valueBytes }
+
+func (a *FixedSizeBinary) setData(data *Data) {
+ if len(data.buffers) != 3 {
+ panic("len(data.buffers) != 3")
+ }
+
+ a.array.setData(data)
+
+ if valueBytes := data.buffers[2]; valueBytes != nil {
+ a.valueBytes = valueBytes.Bytes()
+ }
+ if valueOffsets := data.buffers[1]; valueOffsets != nil {
+ a.valueOffsets = arrow.Int32Traits.CastFromBytes(valueOffsets.Bytes())
+ }
+}
+
+var (
+ _ Interface = (*FixedSizeBinary)(nil)
+)
diff --git a/go/arrow/array/fixedsize_binary_test.go b/go/arrow/array/fixedsize_binary_test.go
new file mode 100644
index 0000000..b7f3d3f
--- /dev/null
+++ b/go/arrow/array/fixedsize_binary_test.go
@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+
+ "github.com/apache/arrow/go/arrow"
+ "github.com/apache/arrow/go/arrow/memory"
+)
+
+func TestFixedSizeBinary(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ dtype := arrow.FixedSizeBinaryType{ByteWidth: 7}
+ b := NewFixedSizeBinaryBuilder(mem, &dtype)
+
+ values := [][]byte{
+ []byte("7654321"),
+ nil,
+ []byte("AZERTYU"),
+ }
+ valid := []bool{true, false, true}
+ b.AppendValues(values, valid)
+
+ b.Retain()
+ b.Release()
+
+ a := b.NewFixedSizeBinaryArray()
+ assert.Equal(t, 3, a.Len())
+ assert.Equal(t, 1, a.NullN())
+ assert.Equal(t, []byte("7654321"), a.Value(0))
+ assert.Equal(t, []byte{}, a.Value(1))
+ assert.Equal(t, []byte("AZERTYU"), a.Value(2))
+ a.Release()
+
+ // Test builder reset and NewArray API.
+ b.AppendValues(values, valid)
+ a = b.NewArray().(*FixedSizeBinary)
+ assert.Equal(t, 3, a.Len())
+ assert.Equal(t, 1, a.NullN())
+ assert.Equal(t, []byte("7654321"), a.Value(0))
+ assert.Equal(t, []byte{}, a.Value(1))
+ assert.Equal(t, []byte("AZERTYU"), a.Value(2))
+ a.Release()
+
+ b.Release()
+}
diff --git a/go/arrow/array/fixedsize_binarybuilder.go b/go/arrow/array/fixedsize_binarybuilder.go
new file mode 100644
index 0000000..11a948f
--- /dev/null
+++ b/go/arrow/array/fixedsize_binarybuilder.go
@@ -0,0 +1,174 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array
+
+import (
+ "sync/atomic"
+
+ "github.com/apache/arrow/go/arrow"
+ "github.com/apache/arrow/go/arrow/internal/debug"
+ "github.com/apache/arrow/go/arrow/memory"
+)
+
+// A FixedSizeBinaryBuilder is used to build a FixedSizeBinary array using the Append methods.
+type FixedSizeBinaryBuilder struct {
+ builder
+
+ dtype *arrow.FixedSizeBinaryType
+ offsets *int32BufferBuilder
+ values *byteBufferBuilder
+}
+
+func NewFixedSizeBinaryBuilder(mem memory.Allocator, dtype *arrow.FixedSizeBinaryType) *FixedSizeBinaryBuilder {
+ b := &FixedSizeBinaryBuilder{
+ builder: builder{refCount: 1, mem: mem},
+ dtype: dtype,
+ offsets: newInt32BufferBuilder(mem),
+ values: newByteBufferBuilder(mem),
+ }
+ return b
+}
+
+// Release decreases the reference count by 1.
+// When the reference count goes to zero, the memory is freed.
+// Release may be called simultaneously from multiple goroutines.
+func (b *FixedSizeBinaryBuilder) Release() {
+ debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
+
+ if atomic.AddInt64(&b.refCount, -1) == 0 {
+ if b.nullBitmap != nil {
+ b.nullBitmap.Release()
+ b.nullBitmap = nil
+ }
+ if b.offsets != nil {
+ b.offsets.Release()
+ b.offsets = nil
+ }
+ if b.values != nil {
+ b.values.Release()
+ b.values = nil
+ }
+ }
+}
+
+func (b *FixedSizeBinaryBuilder) Append(v []byte) {
+ if len(v) != b.dtype.ByteWidth {
+ // TODO(alexandre): should we return an error instead?
+ panic("len(v) != b.dtype.ByteWidth")
+ }
+
+ b.Reserve(1)
+ b.appendNextOffset()
+ b.values.Append(v)
+ b.UnsafeAppendBoolToBitmap(true)
+}
+
+func (b *FixedSizeBinaryBuilder) AppendNull() {
+ b.Reserve(1)
+ b.appendNextOffset()
+ b.UnsafeAppendBoolToBitmap(false)
+}
+
+// AppendValues will append the values in the v slice. The valid slice determines which values
+// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty,
+// all values in v are appended and considered valid.
+func (b *FixedSizeBinaryBuilder) AppendValues(v [][]byte, valid []bool) {
+ if len(v) != len(valid) && len(valid) != 0 {
+ panic("len(v) != len(valid) && len(valid) != 0")
+ }
+
+ b.Reserve(len(v))
+ for _, vv := range v {
+ b.appendNextOffset()
+ b.values.Append(vv)
+ }
+
+ b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
+}
+
+func (b *FixedSizeBinaryBuilder) Value(i int) []byte {
+ offsets := b.offsets.Values()
+ start := int(offsets[i])
+ var end int
+ if i == (b.length - 1) {
+ end = b.values.Len()
+ } else {
+ end = int(offsets[i+1])
+ }
+ return b.values.Bytes()[start:end]
+}
+
+func (b *FixedSizeBinaryBuilder) init(capacity int) {
+ b.builder.init(capacity)
+ b.values.resize(capacity * b.dtype.ByteWidth)
+}
+
+// Reserve ensures there is enough space for appending n elements
+// by checking the capacity and calling Resize if necessary.
+func (b *FixedSizeBinaryBuilder) Reserve(n int) {
+ b.builder.reserve(n, b.Resize)
+}
+
+// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
+// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
+func (b *FixedSizeBinaryBuilder) Resize(n int) {
+ b.offsets.resize((n + 1) * arrow.Int32SizeBytes)
+ b.builder.resize(n, b.init)
+}
+
+// NewArray creates a FixedSizeBinary array from the memory buffers used by the
+// builder and resets the FixedSizeBinaryBuilder so it can be used to build a new array.
+func (b *FixedSizeBinaryBuilder) NewArray() Interface {
+ return b.NewFixedSizeBinaryArray()
+}
+
+// NewFixedSizeBinaryArray creates a FixedSizeBinary array from the memory buffers used by the builder and resets the FixedSizeBinaryBuilder
+// so it can be used to build a new array.
+func (b *FixedSizeBinaryBuilder) NewFixedSizeBinaryArray() (a *FixedSizeBinary) {
+ data := b.newData()
+ a = NewFixedSizeBinaryData(data)
+ data.Release()
+ return
+}
+
+func (b *FixedSizeBinaryBuilder) newData() (data *Data) {
+ b.appendNextOffset()
+ values := b.values.Finish()
+ offsets := b.offsets.Finish()
+ data = NewData(b.dtype, b.length, []*memory.Buffer{b.nullBitmap, offsets, values}, nil, b.nulls, 0)
+
+ if values != nil {
+ values.Release()
+ }
+ if offsets != nil {
+ offsets.Release()
+ }
+
+ b.builder.reset()
+
+ return
+}
+
+func (b *FixedSizeBinaryBuilder) appendNextOffset() {
+ numBytes := b.values.Len()
+ // TODO(alexandre): check binaryArrayMaximumCapacity?
+ b.offsets.AppendValue(int32(numBytes))
+}
+
+var (
+ _ Builder = (*FixedSizeBinaryBuilder)(nil)
+)
diff --git a/go/arrow/array/fixedsize_binarybuilder_test.go b/go/arrow/array/fixedsize_binarybuilder_test.go
new file mode 100644
index 0000000..ce86648
--- /dev/null
+++ b/go/arrow/array/fixedsize_binarybuilder_test.go
@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array
+
+import (
+ "testing"
+
+ "github.com/apache/arrow/go/arrow"
+ "github.com/apache/arrow/go/arrow/memory"
+ "github.com/stretchr/testify/assert"
+)
+
+func TestFixedSizeBinaryBuilder(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ dtype := arrow.FixedSizeBinaryType{ByteWidth: 7}
+ b := NewFixedSizeBinaryBuilder(mem, &dtype)
+
+ b.Append([]byte("1234567"))
+ b.AppendNull()
+ b.Append([]byte("ABCDEFG"))
+ b.AppendNull()
+
+ assert.Equal(t, 4, b.Len(), "unexpected Len()")
+ assert.Equal(t, 2, b.NullN(), "unexpected NullN()")
+
+ assert.Equal(t, b.Value(0), []byte("1234567"))
+ assert.Equal(t, b.Value(1), []byte{})
+ assert.Equal(t, b.Value(2), []byte("ABCDEFG"))
+ assert.Equal(t, b.Value(3), []byte{})
+
+ values := [][]byte{
+ []byte("7654321"),
+ nil,
+ []byte("AZERTYU"),
+ }
+ b.AppendValues(values, []bool{true, false, true})
+
+ assert.Equal(t, 7, b.Len(), "unexpected Len()")
+ assert.Equal(t, 3, b.NullN(), "unexpected NullN()")
+
+ assert.Equal(t, []byte("7654321"), b.Value(4))
+ assert.Equal(t, []byte{}, b.Value(5))
+ assert.Equal(t, []byte("AZERTYU"), b.Value(6))
+
+ a := b.NewFixedSizeBinaryArray()
+
+ // check state of builder after NewFixedSizeBinaryArray
+ assert.Zero(t, b.Len(), "unexpected ArrayBuilder.Len(), NewFixedSizeBinaryArray did not reset state")
+ assert.Zero(t, b.Cap(), "unexpected ArrayBuilder.Cap(), NewFixedSizeBinaryArray did not reset state")
+ assert.Zero(t, b.NullN(), "unexpected ArrayBuilder.NullN(), NewFixedSizeBinaryArray did not reset state")
+
+ b.Release()
+ a.Release()
+}
diff --git a/go/arrow/datatype_fixedwidth.go b/go/arrow/datatype_fixedwidth.go
index cbddcef..60cc98a 100644
--- a/go/arrow/datatype_fixedwidth.go
+++ b/go/arrow/datatype_fixedwidth.go
@@ -24,6 +24,14 @@ func (t *BooleanType) Name() string { return "bool" }
// BitWidth returns the number of bits required to store a single element of this data type in memory.
func (t *BooleanType) BitWidth() int { return 1 }
+type FixedSizeBinaryType struct {
+ ByteWidth int
+}
+
+func (*FixedSizeBinaryType) ID() Type { return FIXED_SIZE_BINARY }
+func (*FixedSizeBinaryType) Name() string { return "fixed_size_binary" }
+func (t *FixedSizeBinaryType) BitWidth() int { return 8 * t.ByteWidth }
+
type (
Timestamp int64
Time32 int32
@@ -86,4 +94,6 @@ var (
Time64us: &Time64Type{Unit: Microsecond},
Time64ns: &Time64Type{Unit: Nanosecond},
}
+
+ _ FixedWidthDataType = (*FixedSizeBinaryType)(nil)
)