You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2018/11/15 13:27:29 UTC
[arrow] branch master updated: ARROW-3672 & ARROW-3673: [Go] add
support for time32 and time64 array
This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new c604adb ARROW-3672 & ARROW-3673: [Go] add support for time32 and time64 array
c604adb is described below
commit c604adbb3ba9eefa9f24093a9b652b9bc66fcd72
Author: alexandreyc <al...@gmail.com>
AuthorDate: Thu Nov 15 08:27:21 2018 -0500
ARROW-3672 & ARROW-3673: [Go] add support for time32 and time64 array
Hello everyone,
My attempt at adding support for time32 and time64 array. Need review because I'm not sure I added all that is needed.
Thanks,
Alexandre
Author: alexandreyc <al...@gmail.com>
Closes #2944 from alexandreyc/master and squashes the following commits:
e686b19b5 <alexandreyc> update mu to u for microseconds
dd2a5d0d1 <alexandreyc> add tests for time 64 array and fix bug
ea68cd50f <alexandreyc> add tests for time 32 array
83eb1310f <alexandreyc> add tests for time32 and time64 builders
e79db3348 <alexandreyc> add time32 and time64 array
3d6ddcf64 <alexandreyc> Fix missing import in numeric builder template
---
go/arrow/array/array.go | 4 +-
go/arrow/array/numeric.gen.go | 90 ++++++++++
go/arrow/array/numeric_test.go | 260 ++++++++++++++++++++++++++++
go/arrow/array/numericbuilder.gen.go | 272 ++++++++++++++++++++++++++++++
go/arrow/array/numericbuilder.gen.go.tmpl | 1 +
go/arrow/array/numericbuilder_test.go | 229 +++++++++++++++++++++++++
go/arrow/datatype_fixedwidth.go | 34 +++-
go/arrow/datatype_fixedwidth_test.go | 2 +-
go/arrow/numeric.tmpldata | 24 +++
go/arrow/type_traits_numeric.gen.go | 98 +++++++++++
10 files changed, 1008 insertions(+), 6 deletions(-)
diff --git a/go/arrow/array/array.go b/go/arrow/array/array.go
index d1dd31d..a225693 100644
--- a/go/arrow/array/array.go
+++ b/go/arrow/array/array.go
@@ -183,8 +183,8 @@ func init() {
arrow.DATE32: unsupportedArrayType,
arrow.DATE64: unsupportedArrayType,
arrow.TIMESTAMP: func(data *Data) Interface { return NewTimestampData(data) },
- arrow.TIME32: unsupportedArrayType,
- arrow.TIME64: unsupportedArrayType,
+ arrow.TIME32: func(data *Data) Interface { return NewTime32Data(data) },
+ arrow.TIME64: func(data *Data) Interface { return NewTime64Data(data) },
arrow.INTERVAL: unsupportedArrayType,
arrow.DECIMAL: unsupportedArrayType,
arrow.LIST: func(data *Data) Interface { return NewListData(data) },
diff --git a/go/arrow/array/numeric.gen.go b/go/arrow/array/numeric.gen.go
index 6f633ea..1f734c0 100644
--- a/go/arrow/array/numeric.gen.go
+++ b/go/arrow/array/numeric.gen.go
@@ -519,3 +519,93 @@ func (a *Timestamp) setData(data *Data) {
a.values = a.values[beg:end]
}
}
+
+// A type which represents an immutable sequence of arrow.Time32 values.
+type Time32 struct {
+ array
+ values []arrow.Time32
+}
+
+func NewTime32Data(data *Data) *Time32 {
+ a := &Time32{}
+ a.refCount = 1
+ a.setData(data)
+ return a
+}
+
+func (a *Time32) Value(i int) arrow.Time32 { return a.values[i] }
+func (a *Time32) Time32Values() []arrow.Time32 { return a.values }
+
+func (a *Time32) String() string {
+ o := new(strings.Builder)
+ o.WriteString("[")
+ for i, v := range a.values {
+ if i > 0 {
+ fmt.Fprintf(o, " ")
+ }
+ switch {
+ case a.IsNull(i):
+ o.WriteString("(null)")
+ default:
+ fmt.Fprintf(o, "%v", v)
+ }
+ }
+ o.WriteString("]")
+ return o.String()
+}
+
+func (a *Time32) setData(data *Data) {
+ a.array.setData(data)
+ vals := data.buffers[1]
+ if vals != nil {
+ a.values = arrow.Time32Traits.CastFromBytes(vals.Bytes())
+ beg := a.array.data.offset
+ end := beg + a.array.data.length
+ a.values = a.values[beg:end]
+ }
+}
+
+// A type which represents an immutable sequence of arrow.Time64 values.
+type Time64 struct {
+ array
+ values []arrow.Time64
+}
+
+func NewTime64Data(data *Data) *Time64 {
+ a := &Time64{}
+ a.refCount = 1
+ a.setData(data)
+ return a
+}
+
+func (a *Time64) Value(i int) arrow.Time64 { return a.values[i] }
+func (a *Time64) Time64Values() []arrow.Time64 { return a.values }
+
+func (a *Time64) String() string {
+ o := new(strings.Builder)
+ o.WriteString("[")
+ for i, v := range a.values {
+ if i > 0 {
+ fmt.Fprintf(o, " ")
+ }
+ switch {
+ case a.IsNull(i):
+ o.WriteString("(null)")
+ default:
+ fmt.Fprintf(o, "%v", v)
+ }
+ }
+ o.WriteString("]")
+ return o.String()
+}
+
+func (a *Time64) setData(data *Data) {
+ a.array.setData(data)
+ vals := data.buffers[1]
+ if vals != nil {
+ a.values = arrow.Time64Traits.CastFromBytes(vals.Bytes())
+ beg := a.array.data.offset
+ end := beg + a.array.data.length
+ a.values = a.values[beg:end]
+ }
+}
diff --git a/go/arrow/array/numeric_test.go b/go/arrow/array/numeric_test.go
index 352ccd1..9e8267a 100644
--- a/go/arrow/array/numeric_test.go
+++ b/go/arrow/array/numeric_test.go
@@ -134,3 +134,263 @@ func TestFloat64SliceDataWithNull(t *testing.T) {
t.Fatalf("got=%v, want=%v", got, want)
}
}
+
+func TestNewTime32Data(t *testing.T) {
+ data := []arrow.Time32{
+ arrow.Time32(1),
+ arrow.Time32(2),
+ arrow.Time32(4),
+ arrow.Time32(8),
+ arrow.Time32(16),
+ }
+
+ dtype := arrow.FixedWidthTypes.Time32s
+ ad := array.NewData(dtype, len(data),
+ []*memory.Buffer{nil, memory.NewBufferBytes(arrow.Time32Traits.CastToBytes(data))},
+ nil, 0, 0,
+ )
+ t32a := array.NewTime32Data(ad)
+
+ assert.Equal(t, len(data), t32a.Len(), "unexpected Len()")
+ assert.Equal(t, data, t32a.Time32Values(), "unexpected Float64Values()")
+}
+
+func TestTime32SliceData(t *testing.T) {
+ pool := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer pool.AssertSize(t, 0)
+
+ const (
+ beg = 2
+ end = 4
+ )
+
+ var (
+ vs = []arrow.Time32{
+ arrow.Time32(1),
+ arrow.Time32(2),
+ arrow.Time32(4),
+ arrow.Time32(8),
+ arrow.Time32(16),
+ }
+ sub = vs[beg:end]
+ )
+
+ dtype := arrow.FixedWidthTypes.Time32s
+ b := array.NewTime32Builder(pool, dtype.(*arrow.Time32Type))
+ defer b.Release()
+
+ for _, v := range vs {
+ b.Append(v)
+ }
+
+ arr := b.NewArray().(*array.Time32)
+ defer arr.Release()
+
+ if got, want := arr.Len(), len(vs); got != want {
+ t.Fatalf("got=%d, want=%d", got, want)
+ }
+
+ if got, want := arr.Time32Values(), vs; !reflect.DeepEqual(got, want) {
+ t.Fatalf("got=%v, want=%v", got, want)
+ }
+
+ slice := array.NewSlice(arr, beg, end).(*array.Time32)
+ defer slice.Release()
+
+ if got, want := slice.Len(), len(sub); got != want {
+ t.Fatalf("got=%d, want=%d", got, want)
+ }
+
+ if got, want := slice.Time32Values(), sub; !reflect.DeepEqual(got, want) {
+ t.Fatalf("got=%v, want=%v", got, want)
+ }
+}
+
+func TestTime32SliceDataWithNull(t *testing.T) {
+ pool := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer pool.AssertSize(t, 0)
+
+ const (
+ beg = 2
+ end = 5
+ )
+
+ var (
+ valids = []bool{true, true, true, false, true, true}
+ vs = []arrow.Time32{
+ arrow.Time32(1),
+ arrow.Time32(2),
+ arrow.Time32(3),
+ arrow.Time32(0),
+ arrow.Time32(4),
+ arrow.Time32(5),
+ }
+ sub = vs[beg:end]
+ )
+
+ dtype := arrow.FixedWidthTypes.Time32s
+ b := array.NewTime32Builder(pool, dtype.(*arrow.Time32Type))
+ defer b.Release()
+
+ b.AppendValues(vs, valids)
+
+ arr := b.NewArray().(*array.Time32)
+ defer arr.Release()
+
+ if got, want := arr.Len(), len(valids); got != want {
+ t.Fatalf("got=%d, want=%d", got, want)
+ }
+
+ if got, want := arr.NullN(), 1; got != want {
+ t.Fatalf("got=%d, want=%d", got, want)
+ }
+
+ if got, want := arr.Time32Values(), vs; !reflect.DeepEqual(got, want) {
+ t.Fatalf("got=%v, want=%v", got, want)
+ }
+
+ slice := array.NewSlice(arr, beg, end).(*array.Time32)
+ defer slice.Release()
+
+ if got, want := slice.NullN(), 1; got != want {
+ t.Errorf("got=%d, want=%d", got, want)
+ }
+
+ if got, want := slice.Len(), len(sub); got != want {
+ t.Fatalf("got=%d, want=%d", got, want)
+ }
+
+ if got, want := slice.Time32Values(), sub; !reflect.DeepEqual(got, want) {
+ t.Fatalf("got=%v, want=%v", got, want)
+ }
+}
+
+func TestNewTime64Data(t *testing.T) {
+ data := []arrow.Time64{
+ arrow.Time64(1),
+ arrow.Time64(2),
+ arrow.Time64(4),
+ arrow.Time64(8),
+ arrow.Time64(16),
+ }
+
+ dtype := arrow.FixedWidthTypes.Time64us
+ ad := array.NewData(dtype, len(data),
+ []*memory.Buffer{nil, memory.NewBufferBytes(arrow.Time64Traits.CastToBytes(data))},
+ nil, 0, 0,
+ )
+ t64a := array.NewTime64Data(ad)
+
+ assert.Equal(t, len(data), t64a.Len(), "unexpected Len()")
+ assert.Equal(t, data, t64a.Time64Values(), "unexpected Float64Values()")
+}
+
+func TestTime64SliceData(t *testing.T) {
+ pool := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer pool.AssertSize(t, 0)
+
+ const (
+ beg = 2
+ end = 4
+ )
+
+ var (
+ vs = []arrow.Time64{
+ arrow.Time64(1),
+ arrow.Time64(2),
+ arrow.Time64(4),
+ arrow.Time64(8),
+ arrow.Time64(16),
+ }
+ sub = vs[beg:end]
+ )
+
+ dtype := arrow.FixedWidthTypes.Time64us
+ b := array.NewTime64Builder(pool, dtype.(*arrow.Time64Type))
+ defer b.Release()
+
+ for _, v := range vs {
+ b.Append(v)
+ }
+
+ arr := b.NewArray().(*array.Time64)
+ defer arr.Release()
+
+ if got, want := arr.Len(), len(vs); got != want {
+ t.Fatalf("got=%d, want=%d", got, want)
+ }
+
+ if got, want := arr.Time64Values(), vs; !reflect.DeepEqual(got, want) {
+ t.Fatalf("got=%v, want=%v", got, want)
+ }
+
+ slice := array.NewSlice(arr, beg, end).(*array.Time64)
+ defer slice.Release()
+
+ if got, want := slice.Len(), len(sub); got != want {
+ t.Fatalf("got=%d, want=%d", got, want)
+ }
+
+ if got, want := slice.Time64Values(), sub; !reflect.DeepEqual(got, want) {
+ t.Fatalf("got=%v, want=%v", got, want)
+ }
+}
+
+func TestTime64SliceDataWithNull(t *testing.T) {
+ pool := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer pool.AssertSize(t, 0)
+
+ const (
+ beg = 2
+ end = 5
+ )
+
+ var (
+ valids = []bool{true, true, true, false, true, true}
+ vs = []arrow.Time64{
+ arrow.Time64(1),
+ arrow.Time64(2),
+ arrow.Time64(3),
+ arrow.Time64(0),
+ arrow.Time64(4),
+ arrow.Time64(5),
+ }
+ sub = vs[beg:end]
+ )
+
+ dtype := arrow.FixedWidthTypes.Time64us
+ b := array.NewTime64Builder(pool, dtype.(*arrow.Time64Type))
+ defer b.Release()
+
+ b.AppendValues(vs, valids)
+
+ arr := b.NewArray().(*array.Time64)
+ defer arr.Release()
+
+ if got, want := arr.Len(), len(valids); got != want {
+ t.Fatalf("got=%d, want=%d", got, want)
+ }
+
+ if got, want := arr.NullN(), 1; got != want {
+ t.Fatalf("got=%d, want=%d", got, want)
+ }
+
+ if got, want := arr.Time64Values(), vs; !reflect.DeepEqual(got, want) {
+ t.Fatalf("got=%v, want=%v", got, want)
+ }
+
+ slice := array.NewSlice(arr, beg, end).(*array.Time64)
+ defer slice.Release()
+
+ if got, want := slice.NullN(), 1; got != want {
+ t.Errorf("got=%d, want=%d", got, want)
+ }
+
+ if got, want := slice.Len(), len(sub); got != want {
+ t.Fatalf("got=%d, want=%d", got, want)
+ }
+
+ if got, want := slice.Time64Values(), sub; !reflect.DeepEqual(got, want) {
+ t.Fatalf("got=%v, want=%v", got, want)
+ }
+}
diff --git a/go/arrow/array/numericbuilder.gen.go b/go/arrow/array/numericbuilder.gen.go
index 6585474..3a7dc16 100644
--- a/go/arrow/array/numericbuilder.gen.go
+++ b/go/arrow/array/numericbuilder.gen.go
@@ -1502,6 +1502,276 @@ func (b *TimestampBuilder) newData() (data *Data) {
return
}
+type Time32Builder struct {
+ builder
+
+ dtype *arrow.Time32Type
+ data *memory.Buffer
+ rawData []arrow.Time32
+}
+
+func NewTime32Builder(mem memory.Allocator, dtype *arrow.Time32Type) *Time32Builder {
+ return &Time32Builder{builder: builder{refCount: 1, mem: mem}, dtype: dtype}
+}
+
+// Release decreases the reference count by 1.
+// When the reference count goes to zero, the memory is freed.
+func (b *Time32Builder) Release() {
+ debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
+
+ if atomic.AddInt64(&b.refCount, -1) == 0 {
+ if b.nullBitmap != nil {
+ b.nullBitmap.Release()
+ b.nullBitmap = nil
+ }
+ if b.data != nil {
+ b.data.Release()
+ b.data = nil
+ b.rawData = nil
+ }
+ }
+}
+
+func (b *Time32Builder) Append(v arrow.Time32) {
+ b.Reserve(1)
+ b.UnsafeAppend(v)
+}
+
+func (b *Time32Builder) AppendNull() {
+ b.Reserve(1)
+ b.UnsafeAppendBoolToBitmap(false)
+}
+
+func (b *Time32Builder) UnsafeAppend(v arrow.Time32) {
+ bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
+ b.rawData[b.length] = v
+ b.length++
+}
+
+func (b *Time32Builder) UnsafeAppendBoolToBitmap(isValid bool) {
+ if isValid {
+ bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
+ } else {
+ b.nulls++
+ }
+ b.length++
+}
+
+// AppendValues will append the values in the v slice. The valid slice determines which values
+// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty,
+// all values in v are appended and considered valid.
+func (b *Time32Builder) AppendValues(v []arrow.Time32, valid []bool) {
+ if len(v) != len(valid) && len(valid) != 0 {
+ panic("len(v) != len(valid) && len(valid) != 0")
+ }
+
+ b.Reserve(len(v))
+ if len(v) > 0 {
+ arrow.Time32Traits.Copy(b.rawData[b.length:], v)
+ }
+ b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
+}
+
+func (b *Time32Builder) init(capacity int) {
+ b.builder.init(capacity)
+
+ b.data = memory.NewResizableBuffer(b.mem)
+ bytesN := arrow.Time32Traits.BytesRequired(capacity)
+ b.data.Resize(bytesN)
+ b.rawData = arrow.Time32Traits.CastFromBytes(b.data.Bytes())
+}
+
+// Reserve ensures there is enough space for appending n elements
+// by checking the capacity and calling Resize if necessary.
+func (b *Time32Builder) Reserve(n int) {
+ b.builder.reserve(n, b.Resize)
+}
+
+// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
+// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
+func (b *Time32Builder) Resize(n int) {
+ nBuilder := n
+ if n < minBuilderCapacity {
+ n = minBuilderCapacity
+ }
+
+ if b.capacity == 0 {
+ b.init(n)
+ } else {
+ b.builder.resize(nBuilder, b.init)
+ b.data.Resize(arrow.Time32Traits.BytesRequired(n))
+ b.rawData = arrow.Time32Traits.CastFromBytes(b.data.Bytes())
+ }
+}
+
+// NewArray creates a Time32 array from the memory buffers used by the builder and resets the Time32Builder
+// so it can be used to build a new array.
+func (b *Time32Builder) NewArray() Interface {
+ return b.NewTime32Array()
+}
+
+// NewTime32Array creates a Time32 array from the memory buffers used by the builder and resets the Time32Builder
+// so it can be used to build a new array.
+func (b *Time32Builder) NewTime32Array() (a *Time32) {
+ data := b.newData()
+ a = NewTime32Data(data)
+ data.Release()
+ return
+}
+
+func (b *Time32Builder) newData() (data *Data) {
+ bytesRequired := arrow.Time32Traits.BytesRequired(b.length)
+ if bytesRequired > 0 && bytesRequired < b.data.Len() {
+ // trim buffers
+ b.data.Resize(bytesRequired)
+ }
+ data = NewData(b.dtype, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0)
+ b.reset()
+
+ if b.data != nil {
+ b.data.Release()
+ b.data = nil
+ b.rawData = nil
+ }
+
+ return
+}
+
+type Time64Builder struct {
+ builder
+
+ dtype *arrow.Time64Type
+ data *memory.Buffer
+ rawData []arrow.Time64
+}
+
+func NewTime64Builder(mem memory.Allocator, dtype *arrow.Time64Type) *Time64Builder {
+ return &Time64Builder{builder: builder{refCount: 1, mem: mem}, dtype: dtype}
+}
+
+// Release decreases the reference count by 1.
+// When the reference count goes to zero, the memory is freed.
+func (b *Time64Builder) Release() {
+ debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
+
+ if atomic.AddInt64(&b.refCount, -1) == 0 {
+ if b.nullBitmap != nil {
+ b.nullBitmap.Release()
+ b.nullBitmap = nil
+ }
+ if b.data != nil {
+ b.data.Release()
+ b.data = nil
+ b.rawData = nil
+ }
+ }
+}
+
+func (b *Time64Builder) Append(v arrow.Time64) {
+ b.Reserve(1)
+ b.UnsafeAppend(v)
+}
+
+func (b *Time64Builder) AppendNull() {
+ b.Reserve(1)
+ b.UnsafeAppendBoolToBitmap(false)
+}
+
+func (b *Time64Builder) UnsafeAppend(v arrow.Time64) {
+ bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
+ b.rawData[b.length] = v
+ b.length++
+}
+
+func (b *Time64Builder) UnsafeAppendBoolToBitmap(isValid bool) {
+ if isValid {
+ bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
+ } else {
+ b.nulls++
+ }
+ b.length++
+}
+
+// AppendValues will append the values in the v slice. The valid slice determines which values
+// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty,
+// all values in v are appended and considered valid.
+func (b *Time64Builder) AppendValues(v []arrow.Time64, valid []bool) {
+ if len(v) != len(valid) && len(valid) != 0 {
+ panic("len(v) != len(valid) && len(valid) != 0")
+ }
+
+ b.Reserve(len(v))
+ if len(v) > 0 {
+ arrow.Time64Traits.Copy(b.rawData[b.length:], v)
+ }
+ b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
+}
+
+func (b *Time64Builder) init(capacity int) {
+ b.builder.init(capacity)
+
+ b.data = memory.NewResizableBuffer(b.mem)
+ bytesN := arrow.Time64Traits.BytesRequired(capacity)
+ b.data.Resize(bytesN)
+ b.rawData = arrow.Time64Traits.CastFromBytes(b.data.Bytes())
+}
+
+// Reserve ensures there is enough space for appending n elements
+// by checking the capacity and calling Resize if necessary.
+func (b *Time64Builder) Reserve(n int) {
+ b.builder.reserve(n, b.Resize)
+}
+
+// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
+// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
+func (b *Time64Builder) Resize(n int) {
+ nBuilder := n
+ if n < minBuilderCapacity {
+ n = minBuilderCapacity
+ }
+
+ if b.capacity == 0 {
+ b.init(n)
+ } else {
+ b.builder.resize(nBuilder, b.init)
+ b.data.Resize(arrow.Time64Traits.BytesRequired(n))
+ b.rawData = arrow.Time64Traits.CastFromBytes(b.data.Bytes())
+ }
+}
+
+// NewArray creates a Time64 array from the memory buffers used by the builder and resets the Time64Builder
+// so it can be used to build a new array.
+func (b *Time64Builder) NewArray() Interface {
+ return b.NewTime64Array()
+}
+
+// NewTime64Array creates a Time64 array from the memory buffers used by the builder and resets the Time64Builder
+// so it can be used to build a new array.
+func (b *Time64Builder) NewTime64Array() (a *Time64) {
+ data := b.newData()
+ a = NewTime64Data(data)
+ data.Release()
+ return
+}
+
+func (b *Time64Builder) newData() (data *Data) {
+ bytesRequired := arrow.Time64Traits.BytesRequired(b.length)
+ if bytesRequired > 0 && bytesRequired < b.data.Len() {
+ // trim buffers
+ b.data.Resize(bytesRequired)
+ }
+ data = NewData(b.dtype, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0)
+ b.reset()
+
+ if b.data != nil {
+ b.data.Release()
+ b.data = nil
+ b.rawData = nil
+ }
+
+ return
+}
+
var (
_ Builder = (*Int64Builder)(nil)
_ Builder = (*Uint64Builder)(nil)
@@ -1514,4 +1784,6 @@ var (
_ Builder = (*Int8Builder)(nil)
_ Builder = (*Uint8Builder)(nil)
_ Builder = (*TimestampBuilder)(nil)
+ _ Builder = (*Time32Builder)(nil)
+ _ Builder = (*Time64Builder)(nil)
)
diff --git a/go/arrow/array/numericbuilder.gen.go.tmpl b/go/arrow/array/numericbuilder.gen.go.tmpl
index 7a3a311..5ae3737 100644
--- a/go/arrow/array/numericbuilder.gen.go.tmpl
+++ b/go/arrow/array/numericbuilder.gen.go.tmpl
@@ -18,6 +18,7 @@ package array
import (
"github.com/apache/arrow/go/arrow"
+ "github.com/apache/arrow/go/arrow/internal/bitutil"
"github.com/apache/arrow/go/arrow/internal/debug"
"github.com/apache/arrow/go/arrow/memory"
)
diff --git a/go/arrow/array/numericbuilder_test.go b/go/arrow/array/numericbuilder_test.go
index eb60569..65f3c86 100644
--- a/go/arrow/array/numericbuilder_test.go
+++ b/go/arrow/array/numericbuilder_test.go
@@ -19,6 +19,7 @@ package array_test
import (
"testing"
+ "github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/array"
"github.com/apache/arrow/go/arrow/memory"
"github.com/stretchr/testify/assert"
@@ -133,3 +134,231 @@ func TestFloat64Builder_Resize(t *testing.T) {
ab.Release()
}
+
+func TestNewTime32Builder(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ dtype := &arrow.Time32Type{Unit: arrow.Second}
+ ab := array.NewTime32Builder(mem, dtype)
+
+ ab.Append(1)
+ ab.Append(2)
+ ab.Append(3)
+ ab.AppendNull()
+ ab.Append(5)
+ ab.Append(6)
+ ab.AppendNull()
+ ab.Append(8)
+ ab.Append(9)
+ ab.Append(10)
+
+ // check state of builder before NewTime32Array
+ assert.Equal(t, 10, ab.Len(), "unexpected Len()")
+ assert.Equal(t, 2, ab.NullN(), "unexpected NullN()")
+
+ a := ab.NewTime32Array()
+
+ // check state of builder after NewTime32Array
+ assert.Zero(t, ab.Len(), "unexpected ArrayBuilder.Len(), NewTime32Array did not reset state")
+ assert.Zero(t, ab.Cap(), "unexpected ArrayBuilder.Cap(), NewTime32Array did not reset state")
+ assert.Zero(t, ab.NullN(), "unexpected ArrayBuilder.NullN(), NewTime32Array did not reset state")
+
+ // check state of array
+ assert.Equal(t, 2, a.NullN(), "unexpected null count")
+ assert.Equal(t, []arrow.Time32{1, 2, 3, 0, 5, 6, 0, 8, 9, 10}, a.Time32Values(), "unexpected Time32Values")
+ assert.Equal(t, []byte{0xb7}, a.NullBitmapBytes()[:1]) // 4 bytes due to minBuilderCapacity
+ assert.Len(t, a.Time32Values(), 10, "unexpected length of Time32Values")
+
+ a.Release()
+
+ ab.Append(7)
+ ab.Append(8)
+
+ a = ab.NewTime32Array()
+
+ assert.Equal(t, 0, a.NullN())
+ assert.Equal(t, []arrow.Time32{7, 8}, a.Time32Values())
+ assert.Len(t, a.Time32Values(), 2)
+
+ a.Release()
+}
+
+func TestTime32Builder_AppendValues(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ dtype := &arrow.Time32Type{Unit: arrow.Second}
+ ab := array.NewTime32Builder(mem, dtype)
+
+ exp := []arrow.Time32{0, 1, 2, 3}
+ ab.AppendValues(exp, nil)
+ a := ab.NewTime32Array()
+ assert.Equal(t, exp, a.Time32Values())
+
+ a.Release()
+ ab.Release()
+}
+
+func TestTime32Builder_Empty(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ dtype := &arrow.Time32Type{Unit: arrow.Second}
+ ab := array.NewTime32Builder(mem, dtype)
+
+ exp := []arrow.Time32{0, 1, 2, 3}
+ ab.AppendValues(exp, nil)
+ a := ab.NewTime32Array()
+ assert.Equal(t, exp, a.Time32Values())
+ a.Release()
+
+ a = ab.NewTime32Array()
+ assert.Zero(t, a.Len())
+ a.Release()
+
+ ab.Release()
+}
+
+func TestTime32Builder_Resize(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ dtype := &arrow.Time32Type{Unit: arrow.Second}
+ ab := array.NewTime32Builder(mem, dtype)
+
+ assert.Equal(t, 0, ab.Cap())
+ assert.Equal(t, 0, ab.Len())
+
+ ab.Reserve(63)
+ assert.Equal(t, 64, ab.Cap())
+ assert.Equal(t, 0, ab.Len())
+
+ for i := 0; i < 63; i++ {
+ ab.Append(0)
+ }
+ assert.Equal(t, 64, ab.Cap())
+ assert.Equal(t, 63, ab.Len())
+
+ ab.Resize(5)
+ assert.Equal(t, 5, ab.Len())
+
+ ab.Resize(32)
+ assert.Equal(t, 5, ab.Len())
+
+ ab.Release()
+}
+
+func TestNewTime64Builder(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ dtype := &arrow.Time64Type{Unit: arrow.Second}
+ ab := array.NewTime64Builder(mem, dtype)
+
+ ab.Append(1)
+ ab.Append(2)
+ ab.Append(3)
+ ab.AppendNull()
+ ab.Append(5)
+ ab.Append(6)
+ ab.AppendNull()
+ ab.Append(8)
+ ab.Append(9)
+ ab.Append(10)
+
+ // check state of builder before NewTime64Array
+ assert.Equal(t, 10, ab.Len(), "unexpected Len()")
+ assert.Equal(t, 2, ab.NullN(), "unexpected NullN()")
+
+ a := ab.NewTime64Array()
+
+ // check state of builder after NewTime64Array
+ assert.Zero(t, ab.Len(), "unexpected ArrayBuilder.Len(), NewTime64Array did not reset state")
+ assert.Zero(t, ab.Cap(), "unexpected ArrayBuilder.Cap(), NewTime64Array did not reset state")
+ assert.Zero(t, ab.NullN(), "unexpected ArrayBuilder.NullN(), NewTime64Array did not reset state")
+
+ // check state of array
+ assert.Equal(t, 2, a.NullN(), "unexpected null count")
+ assert.Equal(t, []arrow.Time64{1, 2, 3, 0, 5, 6, 0, 8, 9, 10}, a.Time64Values(), "unexpected Time64Values")
+ assert.Equal(t, []byte{0xb7}, a.NullBitmapBytes()[:1]) // 4 bytes due to minBuilderCapacity
+ assert.Len(t, a.Time64Values(), 10, "unexpected length of Time64Values")
+
+ a.Release()
+
+ ab.Append(7)
+ ab.Append(8)
+
+ a = ab.NewTime64Array()
+
+ assert.Equal(t, 0, a.NullN())
+ assert.Equal(t, []arrow.Time64{7, 8}, a.Time64Values())
+ assert.Len(t, a.Time64Values(), 2)
+
+ a.Release()
+}
+
+func TestTime64Builder_AppendValues(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ dtype := &arrow.Time64Type{Unit: arrow.Second}
+ ab := array.NewTime64Builder(mem, dtype)
+
+ exp := []arrow.Time64{0, 1, 2, 3}
+ ab.AppendValues(exp, nil)
+ a := ab.NewTime64Array()
+ assert.Equal(t, exp, a.Time64Values())
+
+ a.Release()
+ ab.Release()
+}
+
+func TestTime64Builder_Empty(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ dtype := &arrow.Time64Type{Unit: arrow.Second}
+ ab := array.NewTime64Builder(mem, dtype)
+
+ exp := []arrow.Time64{0, 1, 2, 3}
+ ab.AppendValues(exp, nil)
+ a := ab.NewTime64Array()
+ assert.Equal(t, exp, a.Time64Values())
+ a.Release()
+
+ a = ab.NewTime64Array()
+ assert.Zero(t, a.Len())
+ a.Release()
+
+ ab.Release()
+}
+
+func TestTime64Builder_Resize(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ dtype := &arrow.Time64Type{Unit: arrow.Second}
+ ab := array.NewTime64Builder(mem, dtype)
+
+ assert.Equal(t, 0, ab.Cap())
+ assert.Equal(t, 0, ab.Len())
+
+ ab.Reserve(63)
+ assert.Equal(t, 64, ab.Cap())
+ assert.Equal(t, 0, ab.Len())
+
+ for i := 0; i < 63; i++ {
+ ab.Append(0)
+ }
+ assert.Equal(t, 64, ab.Cap())
+ assert.Equal(t, 63, ab.Len())
+
+ ab.Resize(5)
+ assert.Equal(t, 5, ab.Len())
+
+ ab.Resize(32)
+ assert.Equal(t, 5, ab.Len())
+
+ ab.Release()
+}
diff --git a/go/arrow/datatype_fixedwidth.go b/go/arrow/datatype_fixedwidth.go
index de96ccb..cbddcef 100644
--- a/go/arrow/datatype_fixedwidth.go
+++ b/go/arrow/datatype_fixedwidth.go
@@ -26,6 +26,8 @@ func (t *BooleanType) BitWidth() int { return 1 }
type (
Timestamp int64
+ Time32 int32
+ Time64 int64
TimeUnit int
)
@@ -36,7 +38,7 @@ const (
Second
)
-func (u TimeUnit) String() string { return [...]string{"ns", "µs", "ms", "s"}[uint(u)&3] }
+func (u TimeUnit) String() string { return [...]string{"ns", "us", "ms", "s"}[uint(u)&3] }
// TimestampType is encoded as a 64-bit signed integer since the UNIX epoch (2017-01-01T00:00:00Z).
// The zero-value is a nanosecond and time zone neutral. Time zone neutral can be
@@ -52,10 +54,36 @@ func (*TimestampType) Name() string { return "timestamp" }
// BitWidth returns the number of bits required to store a single element of this data type in memory.
func (*TimestampType) BitWidth() int { return 64 }
+// Time32Type is encoded as a 32-bit signed integer, representing either seconds or milliseconds since midnight.
+type Time32Type struct {
+ Unit TimeUnit
+}
+
+func (*Time32Type) ID() Type { return TIME32 }
+func (*Time32Type) Name() string { return "time32" }
+func (*Time32Type) BitWidth() int { return 32 }
+
+// Time64Type is encoded as a 64-bit signed integer, representing either microseconds or nanoseconds since midnight.
+type Time64Type struct {
+ Unit TimeUnit
+}
+
+func (*Time64Type) ID() Type { return TIME64 }
+func (*Time64Type) Name() string { return "time64" }
+func (*Time64Type) BitWidth() int { return 64 }
+
var (
FixedWidthTypes = struct {
- Boolean FixedWidthDataType
+ Boolean FixedWidthDataType
+ Time32s FixedWidthDataType
+ Time32ms FixedWidthDataType
+ Time64us FixedWidthDataType
+ Time64ns FixedWidthDataType
}{
- Boolean: &BooleanType{},
+ Boolean: &BooleanType{},
+ Time32s: &Time32Type{Unit: Second},
+ Time32ms: &Time32Type{Unit: Millisecond},
+ Time64us: &Time64Type{Unit: Microsecond},
+ Time64ns: &Time64Type{Unit: Nanosecond},
}
)
diff --git a/go/arrow/datatype_fixedwidth_test.go b/go/arrow/datatype_fixedwidth_test.go
index 44fbdd1..865f0ae 100644
--- a/go/arrow/datatype_fixedwidth_test.go
+++ b/go/arrow/datatype_fixedwidth_test.go
@@ -30,7 +30,7 @@ func TestTimeUnit_String(t *testing.T) {
exp string
}{
{arrow.Nanosecond, "ns"},
- {arrow.Microsecond, "µs"},
+ {arrow.Microsecond, "us"},
{arrow.Millisecond, "ms"},
{arrow.Second, "s"},
}
diff --git a/go/arrow/numeric.tmpldata b/go/arrow/numeric.tmpldata
index ea08d8b..b9e976e 100644
--- a/go/arrow/numeric.tmpldata
+++ b/go/arrow/numeric.tmpldata
@@ -83,5 +83,29 @@
"Opt": {
"Parametric": true
}
+ },
+ {
+ "Name": "Time32",
+ "name": "time32",
+ "Type": "Time32",
+ "QualifiedType": "arrow.Time32",
+ "InternalType": "int32",
+ "Default": "0",
+ "Size": "4",
+ "Opt": {
+ "Parametric": true
+ }
+ },
+ {
+ "Name": "Time64",
+ "name": "time64",
+ "Type": "Time64",
+ "QualifiedType": "arrow.Time64",
+ "InternalType": "int64",
+ "Default": "0",
+ "Size": "8",
+ "Opt": {
+ "Parametric": true
+ }
}
]
\ No newline at end of file
diff --git a/go/arrow/type_traits_numeric.gen.go b/go/arrow/type_traits_numeric.gen.go
index b51960b..59ed13f 100644
--- a/go/arrow/type_traits_numeric.gen.go
+++ b/go/arrow/type_traits_numeric.gen.go
@@ -36,6 +36,8 @@ var (
Int8Traits int8Traits
Uint8Traits uint8Traits
TimestampTraits timestampTraits
+ Time32Traits time32Traits
+ Time64Traits time64Traits
)
// Int64 traits
@@ -565,3 +567,99 @@ func (timestampTraits) CastToBytes(b []Timestamp) []byte {
// Copy copies src to dst.
func (timestampTraits) Copy(dst, src []Timestamp) { copy(dst, src) }
+
+// Time32 traits
+
+const (
+ // Time32SizeBytes specifies the number of bytes required to store a single Time32 in memory
+ Time32SizeBytes = int(unsafe.Sizeof(Time32(0)))
+)
+
+type time32Traits struct{}
+
+// BytesRequired returns the number of bytes required to store n elements in memory.
+func (time32Traits) BytesRequired(n int) int { return Time32SizeBytes * n }
+
+// PutValue
+func (time32Traits) PutValue(b []byte, v Time32) {
+ binary.LittleEndian.PutUint32(b, uint32(v))
+}
+
+// CastFromBytes reinterprets the slice b to a slice of type Time32.
+//
+// NOTE: len(b) must be a multiple of Time32SizeBytes.
+func (time32Traits) CastFromBytes(b []byte) []Time32 {
+ h := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+
+ var res []Time32
+ s := (*reflect.SliceHeader)(unsafe.Pointer(&res))
+ s.Data = h.Data
+ s.Len = h.Len / Time32SizeBytes
+ s.Cap = h.Cap / Time32SizeBytes
+
+ return res
+}
+
+// CastToBytes reinterprets the slice b to a slice of bytes.
+func (time32Traits) CastToBytes(b []Time32) []byte {
+ h := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+
+ var res []byte
+ s := (*reflect.SliceHeader)(unsafe.Pointer(&res))
+ s.Data = h.Data
+ s.Len = h.Len * Time32SizeBytes
+ s.Cap = h.Cap * Time32SizeBytes
+
+ return res
+}
+
+// Copy copies src to dst.
+func (time32Traits) Copy(dst, src []Time32) { copy(dst, src) }
+
+// Time64 traits
+
+const (
+ // Time64SizeBytes specifies the number of bytes required to store a single Time64 in memory
+ Time64SizeBytes = int(unsafe.Sizeof(Time64(0)))
+)
+
+type time64Traits struct{}
+
+// BytesRequired returns the number of bytes required to store n elements in memory.
+func (time64Traits) BytesRequired(n int) int { return Time64SizeBytes * n }
+
+// PutValue
+func (time64Traits) PutValue(b []byte, v Time64) {
+ binary.LittleEndian.PutUint64(b, uint64(v))
+}
+
+// CastFromBytes reinterprets the slice b to a slice of type Time64.
+//
+// NOTE: len(b) must be a multiple of Time64SizeBytes.
+func (time64Traits) CastFromBytes(b []byte) []Time64 {
+ h := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+
+ var res []Time64
+ s := (*reflect.SliceHeader)(unsafe.Pointer(&res))
+ s.Data = h.Data
+ s.Len = h.Len / Time64SizeBytes
+ s.Cap = h.Cap / Time64SizeBytes
+
+ return res
+}
+
+// CastToBytes reinterprets the slice b to a slice of bytes.
+func (time64Traits) CastToBytes(b []Time64) []byte {
+ h := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+
+ var res []byte
+ s := (*reflect.SliceHeader)(unsafe.Pointer(&res))
+ s.Data = h.Data
+ s.Len = h.Len * Time64SizeBytes
+ s.Cap = h.Cap * Time64SizeBytes
+
+ return res
+}
+
+// Copy copies src to dst.
+func (time64Traits) Copy(dst, src []Time64) { copy(dst, src) }