You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ze...@apache.org on 2021/11/19 15:17:03 UTC
[arrow] branch master updated: ARROW-9630: [Go] Implement public JSON reader/writer
This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new a2b7ba3 ARROW-9630: [Go] Implement public JSON reader/writer
a2b7ba3 is described below
commit a2b7ba367733ee8686f26680985bed3a530a5722
Author: Matthew Topol <mt...@factset.com>
AuthorDate: Fri Nov 19 10:16:02 2021 -0500
ARROW-9630: [Go] Implement public JSON reader/writer
This JSON reader/writer is intended to support the standard Go encoding/json package for Go and writes out arrays and records as data, not as the integration format. This implements `MarshalJSON` for the array types, and `UnmarshalJSON` for the array builders (in order to unmarshal json data to add to an existing builder) along with providing a few helper functions for single array from json or single recordbatch from json logic.
Closes #11359 from zeroshade/arrow-9630-json-rw
Authored-by: Matthew Topol <mt...@factset.com>
Signed-off-by: Matthew Topol <mt...@factset.com>
---
go/arrow/array/array.go | 5 +
go/arrow/array/binary.go | 18 +
go/arrow/array/binarybuilder.go | 55 ++
go/arrow/array/boolean.go | 20 +
go/arrow/array/booleanbuilder.go | 55 ++
go/arrow/array/builder.go | 7 +
go/arrow/array/decimal128.go | 92 +++
go/arrow/array/extension.go | 14 +
go/arrow/array/fixed_size_list.go | 93 +++
go/arrow/array/fixedsize_binary.go | 21 +
go/arrow/array/fixedsize_binarybuilder.go | 67 ++
go/arrow/array/float16.go | 20 +
go/arrow/array/float16_builder.go | 65 ++
go/arrow/array/interval.go | 195 ++++++
go/arrow/array/json_reader.go | 204 ++++++
go/arrow/array/json_reader_test.go | 117 ++++
go/arrow/array/list.go | 83 +++
go/arrow/array/map.go | 26 +
go/arrow/array/null.go | 54 ++
go/arrow/array/numeric.gen.go | 308 +++++++++
go/arrow/array/numeric.gen.go.tmpl | 43 +-
go/arrow/array/numericbuilder.gen.go | 1030 +++++++++++++++++++++++++++++
go/arrow/array/numericbuilder.gen.go.tmpl | 141 +++-
go/arrow/array/record.go | 57 ++
go/arrow/array/string.go | 65 ++
go/arrow/array/struct.go | 105 +++
go/arrow/array/util.go | 196 ++++++
go/arrow/array/util_test.go | 434 ++++++++++++
go/arrow/datatype_fixedwidth.go | 232 +++++++
go/arrow/datatype_fixedwidth_test.go | 60 ++
go/arrow/scalar/parse.go | 58 +-
go/arrow/scalar/temporal.go | 63 +-
go/arrow/{array/util.go => tools.go} | 14 +-
go/go.mod | 3 +
go/go.sum | 39 +-
35 files changed, 3941 insertions(+), 118 deletions(-)
diff --git a/go/arrow/array/array.go b/go/arrow/array/array.go
index 3ac8c62..2d75b51 100644
--- a/go/arrow/array/array.go
+++ b/go/arrow/array/array.go
@@ -17,6 +17,7 @@
package array
import (
+ "encoding/json"
"sync/atomic"
"github.com/apache/arrow/go/v7/arrow"
@@ -26,6 +27,8 @@ import (
// A type which satisfies array.Interface represents an immutable sequence of values.
type Interface interface {
+ json.Marshaler
+
// DataType returns the type metadata for this instance.
DataType() arrow.DataType
@@ -56,6 +59,8 @@ type Interface interface {
// Release may be called simultaneously from multiple goroutines.
// When the reference count goes to zero, the memory is freed.
Release()
+
+ getOneForMarshal(i int) interface{}
}
const (
diff --git a/go/arrow/array/binary.go b/go/arrow/array/binary.go
index db932a1..66934e2 100644
--- a/go/arrow/array/binary.go
+++ b/go/arrow/array/binary.go
@@ -23,6 +23,7 @@ import (
"unsafe"
"github.com/apache/arrow/go/v7/arrow"
+ "github.com/goccy/go-json"
)
// A type which represents an immutable sequence of variable-length binary strings.
@@ -117,6 +118,23 @@ func (a *Binary) setData(data *Data) {
}
}
+func (a *Binary) getOneForMarshal(i int) interface{} {
+ if a.IsNull(i) {
+ return nil
+ }
+ return a.Value(i)
+}
+
+func (a *Binary) MarshalJSON() ([]byte, error) {
+ vals := make([]interface{}, a.Len())
+ for i := 0; i < a.Len(); i++ {
+ vals[i] = a.getOneForMarshal(i)
+ }
+ // golang marshal standard says that []byte will be marshalled
+ // as a base64-encoded string
+ return json.Marshal(vals)
+}
+
func arrayEqualBinary(left, right *Binary) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
diff --git a/go/arrow/array/binarybuilder.go b/go/arrow/array/binarybuilder.go
index 24ac914..758bea9 100644
--- a/go/arrow/array/binarybuilder.go
+++ b/go/arrow/array/binarybuilder.go
@@ -17,12 +17,17 @@
package array
import (
+ "bytes"
+ "encoding/base64"
+ "fmt"
"math"
+ "reflect"
"sync/atomic"
"github.com/apache/arrow/go/v7/arrow"
"github.com/apache/arrow/go/v7/arrow/internal/debug"
"github.com/apache/arrow/go/v7/arrow/memory"
+ "github.com/goccy/go-json"
)
const (
@@ -212,6 +217,56 @@ func (b *BinaryBuilder) appendNextOffset() {
b.offsets.AppendValue(int32(numBytes))
}
+func (b *BinaryBuilder) unmarshalOne(dec *json.Decoder) error {
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ switch v := t.(type) {
+ case string:
+ data, err := base64.StdEncoding.DecodeString(v)
+ if err != nil {
+ return err
+ }
+ b.Append(data)
+ case []byte:
+ b.Append(v)
+ case nil:
+ b.AppendNull()
+ default:
+ return &json.UnmarshalTypeError{
+ Value: fmt.Sprint(t),
+ Type: reflect.TypeOf([]byte{}),
+ Offset: dec.InputOffset(),
+ }
+ }
+ return nil
+}
+
+func (b *BinaryBuilder) unmarshal(dec *json.Decoder) error {
+ for dec.More() {
+ if err := b.unmarshalOne(dec); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (b *BinaryBuilder) UnmarshalJSON(data []byte) error {
+ dec := json.NewDecoder(bytes.NewReader(data))
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ if delim, ok := t.(json.Delim); !ok || delim != '[' {
+ return fmt.Errorf("binary builder must unpack from json array, found %s", delim)
+ }
+
+ return b.unmarshal(dec)
+}
+
var (
_ Builder = (*BinaryBuilder)(nil)
)
diff --git a/go/arrow/array/boolean.go b/go/arrow/array/boolean.go
index e4c8a77..36e6b12 100644
--- a/go/arrow/array/boolean.go
+++ b/go/arrow/array/boolean.go
@@ -23,6 +23,7 @@ import (
"github.com/apache/arrow/go/v7/arrow"
"github.com/apache/arrow/go/v7/arrow/bitutil"
"github.com/apache/arrow/go/v7/arrow/memory"
+ "github.com/goccy/go-json"
)
// A type which represents an immutable sequence of boolean values.
@@ -78,6 +79,25 @@ func (a *Boolean) setData(data *Data) {
}
}
+func (a *Boolean) getOneForMarshal(i int) interface{} {
+ if a.IsValid(i) {
+ return a.Value(i)
+ }
+ return nil
+}
+
+func (a *Boolean) MarshalJSON() ([]byte, error) {
+ vals := make([]interface{}, a.Len())
+ for i := 0; i < a.Len(); i++ {
+ if a.IsValid(i) {
+ vals[i] = a.Value(i)
+ } else {
+ vals[i] = nil
+ }
+ }
+ return json.Marshal(vals)
+}
+
func arrayEqualBoolean(left, right *Boolean) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
diff --git a/go/arrow/array/booleanbuilder.go b/go/arrow/array/booleanbuilder.go
index 7c81c2e..5cf24d4 100644
--- a/go/arrow/array/booleanbuilder.go
+++ b/go/arrow/array/booleanbuilder.go
@@ -17,12 +17,17 @@
package array
import (
+ "bytes"
+ "fmt"
+ "reflect"
+ "strconv"
"sync/atomic"
"github.com/apache/arrow/go/v7/arrow"
"github.com/apache/arrow/go/v7/arrow/bitutil"
"github.com/apache/arrow/go/v7/arrow/internal/debug"
"github.com/apache/arrow/go/v7/arrow/memory"
+ "github.com/goccy/go-json"
)
type BooleanBuilder struct {
@@ -160,6 +165,56 @@ func (b *BooleanBuilder) newData() *Data {
return res
}
+func (b *BooleanBuilder) unmarshalOne(dec *json.Decoder) error {
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ switch v := t.(type) {
+ case bool:
+ b.Append(v)
+ case string:
+ val, err := strconv.ParseBool(v)
+ if err != nil {
+ return err
+ }
+ b.Append(val)
+ case nil:
+ b.AppendNull()
+ default:
+ return &json.UnmarshalTypeError{
+ Value: fmt.Sprint(t),
+ Type: reflect.TypeOf(true),
+ Offset: dec.InputOffset(),
+ }
+ }
+ return nil
+}
+
+func (b *BooleanBuilder) unmarshal(dec *json.Decoder) error {
+ for dec.More() {
+ if err := b.unmarshalOne(dec); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (b *BooleanBuilder) UnmarshalJSON(data []byte) error {
+ dec := json.NewDecoder(bytes.NewReader(data))
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ if delim, ok := t.(json.Delim); !ok || delim != '[' {
+ return fmt.Errorf("boolean builder must unpack from json array, found %s", delim)
+ }
+
+ return b.unmarshal(dec)
+}
+
var (
_ Builder = (*BooleanBuilder)(nil)
)
diff --git a/go/arrow/array/builder.go b/go/arrow/array/builder.go
index ad56d81..bfd8354 100644
--- a/go/arrow/array/builder.go
+++ b/go/arrow/array/builder.go
@@ -23,6 +23,7 @@ import (
"github.com/apache/arrow/go/v7/arrow"
"github.com/apache/arrow/go/v7/arrow/bitutil"
"github.com/apache/arrow/go/v7/arrow/memory"
+ "github.com/goccy/go-json"
)
const (
@@ -31,6 +32,9 @@ const (
// Builder provides an interface to build arrow arrays.
type Builder interface {
+ // you can unmarshal a json array to add the values to a builder
+ json.Unmarshaler
+
// Retain increases the reference count by 1.
// Retain may be called simultaneously from multiple goroutines.
Retain()
@@ -66,6 +70,9 @@ type Builder interface {
init(capacity int)
resize(newBits int, init func(int))
+
+ unmarshalOne(*json.Decoder) error
+ unmarshal(*json.Decoder) error
}
// builder provides common functionality for managing the validity bitmap (nulls) when building arrays.
diff --git a/go/arrow/array/decimal128.go b/go/arrow/array/decimal128.go
index 1055966..3d7c95e 100644
--- a/go/arrow/array/decimal128.go
+++ b/go/arrow/array/decimal128.go
@@ -17,7 +17,11 @@
package array
import (
+ "bytes"
"fmt"
+ "math"
+ "math/big"
+ "reflect"
"strings"
"sync/atomic"
@@ -26,6 +30,7 @@ import (
"github.com/apache/arrow/go/v7/arrow/decimal128"
"github.com/apache/arrow/go/v7/arrow/internal/debug"
"github.com/apache/arrow/go/v7/arrow/memory"
+ "github.com/goccy/go-json"
)
// A type which represents an immutable sequence of 128-bit decimal values.
@@ -75,6 +80,25 @@ func (a *Decimal128) setData(data *Data) {
}
}
+func (a *Decimal128) getOneForMarshal(i int) interface{} {
+ if a.IsNull(i) {
+ return nil
+ }
+
+ typ := a.DataType().(*arrow.Decimal128Type)
+ f := (&big.Float{}).SetInt(a.Value(i).BigInt())
+ f.Quo(f, big.NewFloat(math.Pow10(int(typ.Scale))))
+ return f.Text('g', int(typ.Precision))
+}
+
+func (a *Decimal128) MarshalJSON() ([]byte, error) {
+ vals := make([]interface{}, a.Len())
+ for i := 0; i < a.Len(); i++ {
+ vals[i] = a.getOneForMarshal(i)
+ }
+ return json.Marshal(vals)
+}
+
func arrayEqualDecimal128(left, right *Decimal128) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
@@ -229,6 +253,74 @@ func (b *Decimal128Builder) newData() (data *Data) {
return
}
+func (b *Decimal128Builder) unmarshalOne(dec *json.Decoder) error {
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ var out *big.Float
+
+ switch v := t.(type) {
+ case float64:
+ out = big.NewFloat(v)
+ case string:
+ // there's no strong rationale for using ToNearestAway, it's just
+ // what got me the closest equivalent values with the values
+ // that I tested with, and there isn't a good way to push
+ // an option all the way down here to control it.
+ out, _, err = big.ParseFloat(v, 10, 128, big.ToNearestAway)
+ if err != nil {
+ return err
+ }
+ case json.Number:
+ out, _, err = big.ParseFloat(v.String(), 10, 128, big.ToNearestAway)
+ if err != nil {
+ return err
+ }
+ case nil:
+ b.AppendNull()
+ return nil
+ default:
+ return &json.UnmarshalTypeError{
+ Value: fmt.Sprint(t),
+ Type: reflect.TypeOf(decimal128.Num{}),
+ Offset: dec.InputOffset(),
+ }
+ }
+
+ val, _ := out.Mul(out, big.NewFloat(math.Pow10(int(b.dtype.Scale)))).Int(nil)
+ b.Append(decimal128.FromBigInt(val))
+ return nil
+}
+
+func (b *Decimal128Builder) unmarshal(dec *json.Decoder) error {
+ for dec.More() {
+ if err := b.unmarshalOne(dec); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// UnmarshalJSON will add the unmarshalled values to this builder.
+//
+// If the values are strings, they will get parsed with big.ParseFloat using
+// a rounding mode of big.ToNearestAway currently.
+func (b *Decimal128Builder) UnmarshalJSON(data []byte) error {
+ dec := json.NewDecoder(bytes.NewReader(data))
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ if delim, ok := t.(json.Delim); !ok || delim != '[' {
+ return fmt.Errorf("decimal128 builder must unpack from json array, found %s", delim)
+ }
+
+ return b.unmarshal(dec)
+}
+
var (
_ Interface = (*Decimal128)(nil)
_ Builder = (*Decimal128Builder)(nil)
diff --git a/go/arrow/array/extension.go b/go/arrow/array/extension.go
index 5172158..db87b48 100644
--- a/go/arrow/array/extension.go
+++ b/go/arrow/array/extension.go
@@ -21,6 +21,7 @@ import (
"github.com/apache/arrow/go/v7/arrow"
"github.com/apache/arrow/go/v7/arrow/memory"
+ "github.com/goccy/go-json"
"golang.org/x/xerrors"
)
@@ -129,6 +130,14 @@ type ExtensionArrayBase struct {
storage Interface
}
+func (e *ExtensionArrayBase) getOneForMarshal(i int) interface{} {
+ return e.storage.getOneForMarshal(i)
+}
+
+func (e *ExtensionArrayBase) MarshalJSON() ([]byte, error) {
+ return json.Marshal(e.storage)
+}
+
// Retain increases the reference count by 1.
// Retain may be called simultaneously from multiple goroutines.
func (e *ExtensionArrayBase) Retain() {
@@ -234,3 +243,8 @@ func (b *ExtensionBuilder) NewExtensionArray() ExtensionArray {
defer data.Release()
return NewExtensionData(data)
}
+
+var (
+ _ Interface = (ExtensionArray)(nil)
+ _ Builder = (*ExtensionBuilder)(nil)
+)
diff --git a/go/arrow/array/fixed_size_list.go b/go/arrow/array/fixed_size_list.go
index 1bf07d6..acef33c 100644
--- a/go/arrow/array/fixed_size_list.go
+++ b/go/arrow/array/fixed_size_list.go
@@ -17,6 +17,7 @@
package array
import (
+ "bytes"
"fmt"
"strings"
"sync/atomic"
@@ -25,6 +26,7 @@ import (
"github.com/apache/arrow/go/v7/arrow/bitutil"
"github.com/apache/arrow/go/v7/arrow/internal/debug"
"github.com/apache/arrow/go/v7/arrow/memory"
+ "github.com/goccy/go-json"
)
// FixedSizeList represents an immutable sequence of N array values.
@@ -110,6 +112,44 @@ func (a *FixedSizeList) Release() {
a.values.Release()
}
+func (a *FixedSizeList) getOneForMarshal(i int) interface{} {
+ if a.IsNull(i) {
+ return nil
+ }
+ slice := a.newListValue(i)
+ defer slice.Release()
+ v, err := json.Marshal(slice)
+ if err != nil {
+ panic(err)
+ }
+
+ return json.RawMessage(v)
+}
+
+func (a *FixedSizeList) MarshalJSON() ([]byte, error) {
+ var buf bytes.Buffer
+ enc := json.NewEncoder(&buf)
+
+ buf.WriteByte('[')
+ for i := 0; i < a.Len(); i++ {
+ if i != 0 {
+ buf.WriteByte(',')
+ }
+ if a.IsNull(i) {
+ enc.Encode(nil)
+ continue
+ }
+
+ slice := a.newListValue(i)
+ if err := enc.Encode(slice); err != nil {
+ return nil, err
+ }
+ slice.Release()
+ }
+ buf.WriteByte(']')
+ return buf.Bytes(), nil
+}
+
type FixedSizeListBuilder struct {
builder
@@ -234,6 +274,59 @@ func (b *FixedSizeListBuilder) newData() (data *Data) {
return
}
+func (b *FixedSizeListBuilder) unmarshalOne(dec *json.Decoder) error {
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ switch t {
+ case json.Delim('['):
+ b.Append(true)
+ if err := b.values.unmarshal(dec); err != nil {
+ return err
+ }
+ // consume ']'
+ _, err := dec.Token()
+ return err
+ case nil:
+ b.AppendNull()
+ for i := int32(0); i < b.n; i++ {
+ b.values.AppendNull()
+ }
+ default:
+ return &json.UnmarshalTypeError{
+ Value: fmt.Sprint(t),
+ Struct: arrow.FixedSizeListOf(b.n, b.etype).String(),
+ }
+ }
+
+ return nil
+}
+
+func (b *FixedSizeListBuilder) unmarshal(dec *json.Decoder) error {
+ for dec.More() {
+ if err := b.unmarshalOne(dec); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (b *FixedSizeListBuilder) UnmarshalJSON(data []byte) error {
+ dec := json.NewDecoder(bytes.NewReader(data))
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ if delim, ok := t.(json.Delim); !ok || delim != '[' {
+ return fmt.Errorf("fixed size list builder must unpack from json array, found %s", delim)
+ }
+
+ return b.unmarshal(dec)
+}
+
var (
_ Interface = (*FixedSizeList)(nil)
_ Builder = (*FixedSizeListBuilder)(nil)
diff --git a/go/arrow/array/fixedsize_binary.go b/go/arrow/array/fixedsize_binary.go
index 6da2d5d..867ea78 100644
--- a/go/arrow/array/fixedsize_binary.go
+++ b/go/arrow/array/fixedsize_binary.go
@@ -22,6 +22,7 @@ import (
"strings"
"github.com/apache/arrow/go/v7/arrow"
+ "github.com/goccy/go-json"
)
// A type which represents an immutable sequence of fixed-length binary strings.
@@ -78,6 +79,26 @@ func (a *FixedSizeBinary) setData(data *Data) {
}
+func (a *FixedSizeBinary) getOneForMarshal(i int) interface{} {
+ if a.IsNull(i) {
+ return nil
+ }
+
+ return a.Value(i)
+}
+
+func (a *FixedSizeBinary) MarshalJSON() ([]byte, error) {
+ vals := make([]interface{}, a.Len())
+ for i := 0; i < a.Len(); i++ {
+ if a.IsValid(i) {
+ vals[i] = a.Value(i)
+ } else {
+ vals[i] = nil
+ }
+ }
+ return json.Marshal(vals)
+}
+
func arrayEqualFixedSizeBinary(left, right *FixedSizeBinary) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
diff --git a/go/arrow/array/fixedsize_binarybuilder.go b/go/arrow/array/fixedsize_binarybuilder.go
index 8aaa93c..fee0a70 100644
--- a/go/arrow/array/fixedsize_binarybuilder.go
+++ b/go/arrow/array/fixedsize_binarybuilder.go
@@ -17,12 +17,16 @@
package array
import (
+ "bytes"
+ "encoding/base64"
"fmt"
+ "reflect"
"sync/atomic"
"github.com/apache/arrow/go/v7/arrow"
"github.com/apache/arrow/go/v7/arrow/internal/debug"
"github.com/apache/arrow/go/v7/arrow/memory"
+ "github.com/goccy/go-json"
)
// A FixedSizeBinaryBuilder is used to build a FixedSizeBinary array using the Append methods.
@@ -149,6 +153,69 @@ func (b *FixedSizeBinaryBuilder) newData() (data *Data) {
return
}
+func (b *FixedSizeBinaryBuilder) unmarshalOne(dec *json.Decoder) error {
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ var val []byte
+ switch v := t.(type) {
+ case string:
+ data, err := base64.RawStdEncoding.DecodeString(v)
+ if err != nil {
+ return err
+ }
+ val = data
+ case []byte:
+ val = v
+ case nil:
+ b.AppendNull()
+ return nil
+ default:
+ return &json.UnmarshalTypeError{
+ Value: fmt.Sprint(t),
+ Type: reflect.TypeOf([]byte{}),
+ Offset: dec.InputOffset(),
+ Struct: fmt.Sprintf("FixedSizeBinary[%d]", b.dtype.ByteWidth),
+ }
+ }
+
+ if len(val) != b.dtype.ByteWidth {
+ return &json.UnmarshalTypeError{
+ Value: fmt.Sprint(val),
+ Type: reflect.TypeOf([]byte{}),
+ Offset: dec.InputOffset(),
+ Struct: fmt.Sprintf("FixedSizeBinary[%d]", b.dtype.ByteWidth),
+ }
+ }
+ b.Append(val)
+ return nil
+}
+
+func (b *FixedSizeBinaryBuilder) unmarshal(dec *json.Decoder) error {
+ for dec.More() {
+ if err := b.unmarshalOne(dec); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (b *FixedSizeBinaryBuilder) UnmarshalJSON(data []byte) error {
+ dec := json.NewDecoder(bytes.NewReader(data))
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ if delim, ok := t.(json.Delim); !ok || delim != '[' {
+ return fmt.Errorf("fixed size binary builder must unpack from json array, found %s", delim)
+ }
+
+ return b.unmarshal(dec)
+}
+
var (
_ Builder = (*FixedSizeBinaryBuilder)(nil)
)
diff --git a/go/arrow/array/float16.go b/go/arrow/array/float16.go
index a7fd4eb..74a5f8b 100644
--- a/go/arrow/array/float16.go
+++ b/go/arrow/array/float16.go
@@ -22,6 +22,7 @@ import (
"github.com/apache/arrow/go/v7/arrow"
"github.com/apache/arrow/go/v7/arrow/float16"
+ "github.com/goccy/go-json"
)
// A type which represents an immutable sequence of Float16 values.
@@ -70,6 +71,25 @@ func (a *Float16) setData(data *Data) {
}
}
+func (a *Float16) getOneForMarshal(i int) interface{} {
+ if a.IsValid(i) {
+ return a.values[i].Float32()
+ }
+ return nil
+}
+
+func (a *Float16) MarshalJSON() ([]byte, error) {
+ vals := make([]interface{}, a.Len())
+ for i, v := range a.values {
+ if a.IsValid(i) {
+ vals[i] = v.Float32()
+ } else {
+ vals[i] = nil
+ }
+ }
+ return json.Marshal(vals)
+}
+
func arrayEqualFloat16(left, right *Float16) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
diff --git a/go/arrow/array/float16_builder.go b/go/arrow/array/float16_builder.go
index 1cf538a..ebd9fc6 100644
--- a/go/arrow/array/float16_builder.go
+++ b/go/arrow/array/float16_builder.go
@@ -17,6 +17,10 @@
package array
import (
+ "bytes"
+ "fmt"
+ "reflect"
+ "strconv"
"sync/atomic"
"github.com/apache/arrow/go/v7/arrow"
@@ -24,6 +28,7 @@ import (
"github.com/apache/arrow/go/v7/arrow/float16"
"github.com/apache/arrow/go/v7/arrow/internal/debug"
"github.com/apache/arrow/go/v7/arrow/memory"
+ "github.com/goccy/go-json"
)
type Float16Builder struct {
@@ -163,3 +168,63 @@ func (b *Float16Builder) newData() (data *Data) {
return
}
+
+func (b *Float16Builder) unmarshalOne(dec *json.Decoder) error {
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ switch v := t.(type) {
+ case float64:
+ b.Append(float16.New(float32(v)))
+ case string:
+ f, err := strconv.ParseFloat(v, 32)
+ if err != nil {
+ return err
+ }
+ // this will currently silently truncate if it is too large
+ b.Append(float16.New(float32(f)))
+ case json.Number:
+ f, err := v.Float64()
+ if err != nil {
+ return err
+ }
+ b.Append(float16.New(float32(f)))
+ case nil:
+ b.AppendNull()
+ default:
+ return &json.UnmarshalTypeError{
+ Value: fmt.Sprint(t),
+ Type: reflect.TypeOf(float16.Num{}),
+ Offset: dec.InputOffset(),
+ }
+ }
+ return nil
+}
+
+func (b *Float16Builder) unmarshal(dec *json.Decoder) error {
+ for dec.More() {
+ if err := b.unmarshalOne(dec); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// UnmarshalJSON will add values to this builder from unmarshalling the
+// array of values. Currently values that are larger than a float16 will
+// be silently truncated.
+func (b *Float16Builder) UnmarshalJSON(data []byte) error {
+ dec := json.NewDecoder(bytes.NewReader(data))
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ if delim, ok := t.(json.Delim); !ok || delim != '[' {
+ return fmt.Errorf("float16 builder must unpack from json array, found %s", delim)
+ }
+
+ return b.unmarshal(dec)
+}
diff --git a/go/arrow/array/interval.go b/go/arrow/array/interval.go
index 2e0c1ee..bf6651c 100644
--- a/go/arrow/array/interval.go
+++ b/go/arrow/array/interval.go
@@ -17,6 +17,7 @@
package array
import (
+ "bytes"
"fmt"
"strings"
"sync/atomic"
@@ -25,6 +26,7 @@ import (
"github.com/apache/arrow/go/v7/arrow/bitutil"
"github.com/apache/arrow/go/v7/arrow/internal/debug"
"github.com/apache/arrow/go/v7/arrow/memory"
+ "github.com/goccy/go-json"
"golang.org/x/xerrors"
)
@@ -86,6 +88,32 @@ func (a *MonthInterval) setData(data *Data) {
}
}
+func (a *MonthInterval) getOneForMarshal(i int) interface{} {
+ if a.IsValid(i) {
+ return a.values[i]
+ }
+ return nil
+}
+
+// MarshalJSON will create a json array out of a MonthInterval array,
+// each value will be an object of the form {"months": #} where
+// # is the numeric value of that index
+func (a *MonthInterval) MarshalJSON() ([]byte, error) {
+ if a.NullN() == 0 {
+ return json.Marshal(a.values)
+ }
+ vals := make([]interface{}, a.Len())
+ for i := 0; i < a.Len(); i++ {
+ if a.IsValid(i) {
+ vals[i] = a.values[i]
+ } else {
+ vals[i] = nil
+ }
+ }
+
+ return json.Marshal(vals)
+}
+
func arrayEqualMonthInterval(left, right *MonthInterval) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
@@ -234,6 +262,46 @@ func (b *MonthIntervalBuilder) newData() (data *Data) {
return
}
+func (b *MonthIntervalBuilder) unmarshalOne(dec *json.Decoder) error {
+ var v *arrow.MonthInterval
+ if err := dec.Decode(&v); err != nil {
+ return err
+ }
+
+ if v == nil {
+ b.AppendNull()
+ } else {
+ b.Append(*v)
+ }
+ return nil
+}
+
+func (b *MonthIntervalBuilder) unmarshal(dec *json.Decoder) error {
+ for dec.More() {
+ if err := b.unmarshalOne(dec); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// UnmarshalJSON will add the unmarshalled values of an array to the builder,
+// values are expected to be strings of the form "#months" where # is the int32
+// value that will be added to the builder.
+func (b *MonthIntervalBuilder) UnmarshalJSON(data []byte) error {
+ dec := json.NewDecoder(bytes.NewReader(data))
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ if delim, ok := t.(json.Delim); !ok || delim != '[' {
+ return fmt.Errorf("month interval builder must unpack from json array, found %s", delim)
+ }
+
+ return b.unmarshal(dec)
+}
+
// A type which represents an immutable sequence of arrow.DayTimeInterval values.
type DayTimeInterval struct {
array
@@ -279,6 +347,30 @@ func (a *DayTimeInterval) setData(data *Data) {
}
}
+func (a *DayTimeInterval) getOneForMarshal(i int) interface{} {
+ if a.IsValid(i) {
+ return a.values[i]
+ }
+ return nil
+}
+
+// MarshalJSON will marshal this array to JSON as an array of objects,
+// consisting of the form {"days": #, "milliseconds": #} for each element.
+func (a *DayTimeInterval) MarshalJSON() ([]byte, error) {
+ if a.NullN() == 0 {
+ return json.Marshal(a.values)
+ }
+ vals := make([]interface{}, a.Len())
+ for i, v := range a.values {
+ if a.IsValid(i) {
+ vals[i] = v
+ } else {
+ vals[i] = nil
+ }
+ }
+ return json.Marshal(vals)
+}
+
func arrayEqualDayTimeInterval(left, right *DayTimeInterval) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
@@ -427,6 +519,45 @@ func (b *DayTimeIntervalBuilder) newData() (data *Data) {
return
}
+func (b *DayTimeIntervalBuilder) unmarshalOne(dec *json.Decoder) error {
+ var v *arrow.DayTimeInterval
+ if err := dec.Decode(&v); err != nil {
+ return err
+ }
+
+ if v == nil {
+ b.AppendNull()
+ } else {
+ b.Append(*v)
+ }
+ return nil
+}
+
+func (b *DayTimeIntervalBuilder) unmarshal(dec *json.Decoder) error {
+ for dec.More() {
+ if err := b.unmarshalOne(dec); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// UnmarshalJSON will add the values unmarshalled from an array to the builder,
+// with the values expected to be objects of the form {"days": #, "milliseconds": #}
+func (b *DayTimeIntervalBuilder) UnmarshalJSON(data []byte) error {
+ dec := json.NewDecoder(bytes.NewReader(data))
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ if delim, ok := t.(json.Delim); !ok || delim != '[' {
+ return fmt.Errorf("day_time interval builder must unpack from json array, found %s", delim)
+ }
+
+ return b.unmarshal(dec)
+}
+
// A type which represents an immutable sequence of arrow.DayTimeInterval values.
type MonthDayNanoInterval struct {
array
@@ -474,6 +605,30 @@ func (a *MonthDayNanoInterval) setData(data *Data) {
}
}
+func (a *MonthDayNanoInterval) getOneForMarshal(i int) interface{} {
+ if a.IsValid(i) {
+ return a.values[i]
+ }
+ return nil
+}
+
+// MarshalJSON will marshal this array to a JSON array with elements
+// marshalled to the form {"months": #, "days": #, "nanoseconds": #}
+func (a *MonthDayNanoInterval) MarshalJSON() ([]byte, error) {
+ if a.NullN() == 0 {
+ return json.Marshal(a.values)
+ }
+ vals := make([]interface{}, a.Len())
+ for i, v := range a.values {
+ if a.IsValid(i) {
+ vals[i] = v
+ } else {
+ vals[i] = nil
+ }
+ }
+ return json.Marshal(vals)
+}
+
func arrayEqualMonthDayNanoInterval(left, right *MonthDayNanoInterval) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
@@ -622,6 +777,46 @@ func (b *MonthDayNanoIntervalBuilder) newData() (data *Data) {
return
}
+func (b *MonthDayNanoIntervalBuilder) unmarshalOne(dec *json.Decoder) error {
+ var v *arrow.MonthDayNanoInterval
+ if err := dec.Decode(&v); err != nil {
+ return err
+ }
+
+ if v == nil {
+ b.AppendNull()
+ } else {
+ b.Append(*v)
+ }
+ return nil
+}
+
+func (b *MonthDayNanoIntervalBuilder) unmarshal(dec *json.Decoder) error {
+ for dec.More() {
+ if err := b.unmarshalOne(dec); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// UnmarshalJSON unmarshals a JSON array of objects and adds them to this builder,
+// each element of the array is expected to be an object of the form
+// {"months": #, "days": #, "nanoseconds": #}
+func (b *MonthDayNanoIntervalBuilder) UnmarshalJSON(data []byte) error {
+ dec := json.NewDecoder(bytes.NewReader(data))
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ if delim, ok := t.(json.Delim); !ok || delim != '[' {
+ return fmt.Errorf("month_day_nano interval builder must unpack from json array, found %s", delim)
+ }
+
+ return b.unmarshal(dec)
+}
+
var (
_ Interface = (*MonthInterval)(nil)
_ Interface = (*DayTimeInterval)(nil)
diff --git a/go/arrow/array/json_reader.go b/go/arrow/array/json_reader.go
new file mode 100644
index 0000000..f6397c0
--- /dev/null
+++ b/go/arrow/array/json_reader.go
@@ -0,0 +1,204 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array
+
+import (
+ "fmt"
+ "io"
+ "sync/atomic"
+
+ "github.com/apache/arrow/go/v7/arrow"
+ "github.com/apache/arrow/go/v7/arrow/internal/debug"
+ "github.com/apache/arrow/go/v7/arrow/memory"
+ "github.com/goccy/go-json"
+)
+
+type Option func(config)
+type config interface{}
+
+// WithChunk sets the chunk size for reading in json records. The default is to
+// read in one row per record batch as a single object. If chunk size is set to
+// a negative value, then the entire file is read as a single record batch.
+// Otherwise a record batch is read in with chunk size rows per record batch until
+// it reaches EOF.
+func WithChunk(n int) Option {
+ return func(cfg config) {
+ switch cfg := cfg.(type) {
+ case *JSONReader:
+ cfg.chunk = n
+ default:
+ panic(fmt.Errorf("arrow/json): unknown config type %T", cfg))
+ }
+ }
+}
+
+// WithAllocator specifies the allocator to use for creating the record batches,
+// if it is not called, then memory.DefaultAllocator will be used.
+func WithAllocator(mem memory.Allocator) Option {
+ return func(cfg config) {
+ switch cfg := cfg.(type) {
+ case *JSONReader:
+ cfg.mem = mem
+ default:
+ panic(fmt.Errorf("arrow/json): unknown config type %T", cfg))
+ }
+ }
+}
+
+// JSONReader is a json reader that meets the RecordReader interface definition.
+//
+// To read in an array of objects as a record, you can use RecordFromJSON
+// which is equivalent to reading the json as a struct array whose fields are
+// the columns of the record. This primarily exists to fit the RecordReader
+// interface as a matching reader for the csv reader.
+type JSONReader struct {
+ r *json.Decoder
+ schema *arrow.Schema
+
+ bldr *RecordBuilder
+
+ refs int64
+ cur Record
+ err error
+
+ chunk int
+ done bool
+
+ mem memory.Allocator
+ next func() bool
+}
+
+// NewJSONReader returns a json RecordReader which expects to find one json object
+// per row of dataset. Using WithChunk can control how many rows are processed
+// per record, which is how many objects become a single record from the file.
+//
+// If it is desired to write out an array of rows, then simply use RecordToStructArray
+// and json.Marshal the struct array for the same effect.
+func NewJSONReader(r io.Reader, schema *arrow.Schema, opts ...Option) *JSONReader {
+ rr := &JSONReader{
+ r: json.NewDecoder(r),
+ schema: schema,
+ refs: 1,
+ chunk: 1,
+ }
+ for _, o := range opts {
+ o(rr)
+ }
+
+ if rr.mem == nil {
+ rr.mem = memory.DefaultAllocator
+ }
+
+ rr.bldr = NewRecordBuilder(rr.mem, schema)
+ switch {
+ case rr.chunk < 0:
+ rr.next = rr.nextall
+ case rr.chunk > 1:
+ rr.next = rr.nextn
+ default:
+ rr.next = rr.next1
+ }
+ return rr
+}
+
+// Err returns the last encountered error
+func (r *JSONReader) Err() error { return r.err }
+
+func (r *JSONReader) Schema() *arrow.Schema { return r.schema }
+
+// Record returns the last read in record. The returned record is only valid
+// until the next call to Next unless Retain is called on the record itself.
+func (r *JSONReader) Record() Record { return r.cur }
+
+func (r *JSONReader) Retain() {
+ atomic.AddInt64(&r.refs, 1)
+}
+
+func (r *JSONReader) Release() {
+ debug.Assert(atomic.LoadInt64(&r.refs) > 0, "too many releases")
+
+ if atomic.AddInt64(&r.refs, -1) == 0 {
+ if r.cur != nil {
+ r.cur.Release()
+ r.bldr.Release()
+ r.r = nil
+ }
+ }
+}
+
+// Next returns true if it read in a record, which will be available via Record
+// and false if there is either an error or the end of the reader.
+func (r *JSONReader) Next() bool {
+ if r.cur != nil {
+ r.cur.Release()
+ r.cur = nil
+ }
+
+ if r.err != nil || r.done {
+ return false
+ }
+
+ return r.next()
+}
+
+func (r *JSONReader) readNext() bool {
+ r.err = r.r.Decode(r.bldr)
+ if r.err != nil {
+ r.done = true
+ if r.err == io.EOF {
+ r.err = nil
+ }
+ return false
+ }
+ return true
+}
+
+func (r *JSONReader) nextall() bool {
+ for r.readNext() {
+ }
+
+ r.cur = r.bldr.NewRecord()
+ return r.cur.NumRows() > 0
+}
+
+func (r *JSONReader) next1() bool {
+ if !r.readNext() {
+ return false
+ }
+
+ r.cur = r.bldr.NewRecord()
+ return true
+}
+
+func (r *JSONReader) nextn() bool {
+ var n = 0
+
+ for i := 0; i < r.chunk && !r.done; i, n = i+1, n+1 {
+ if !r.readNext() {
+ break
+ }
+ }
+
+ if n > 0 {
+ r.cur = r.bldr.NewRecord()
+ }
+ return n > 0
+}
+
+var (
+ _ RecordReader = (*JSONReader)(nil)
+)
diff --git a/go/arrow/array/json_reader_test.go b/go/arrow/array/json_reader_test.go
new file mode 100644
index 0000000..71d4c0f
--- /dev/null
+++ b/go/arrow/array/json_reader_test.go
@@ -0,0 +1,117 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array_test
+
+import (
+ "strings"
+ "testing"
+
+ "github.com/apache/arrow/go/v7/arrow"
+ "github.com/apache/arrow/go/v7/arrow/array"
+ "github.com/apache/arrow/go/v7/arrow/memory"
+ "github.com/stretchr/testify/assert"
+)
+
+const jsondata = `
+ {"region": "NY", "model": "3", "sales": 742.0}
+ {"region": "NY", "model": "S", "sales": 304.125}
+ {"region": "NY", "model": "X", "sales": 136.25}
+ {"region": "NY", "model": "Y", "sales": 27.5}
+ {"region": "CA", "model": "3", "sales": 512}
+ {"region": "CA", "model": "S", "sales": 978}
+ {"region": "CA", "model": "X", "sales": 1.0}
+ {"region": "CA", "model": "Y", "sales": 69}
+ {"region": "QC", "model": "3", "sales": 273.5}
+ {"region": "QC", "model": "S", "sales": 13}
+ {"region": "QC", "model": "X", "sales": 54}
+ {"region": "QC", "model": "Y", "sales": 21}
+ {"region": "QC", "model": "3", "sales": 152.25}
+ {"region": "QC", "model": "S", "sales": 10}
+ {"region": "QC", "model": "X", "sales": 42}
+ {"region": "QC", "model": "Y", "sales": 37}`
+
+func TestJSONReader(t *testing.T) {
+ schema := arrow.NewSchema([]arrow.Field{
+ {Name: "region", Type: arrow.BinaryTypes.String, Nullable: true},
+ {Name: "model", Type: arrow.BinaryTypes.String},
+ {Name: "sales", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
+ }, nil)
+
+ rdr := array.NewJSONReader(strings.NewReader(jsondata), schema)
+ defer rdr.Release()
+
+ n := 0
+ for rdr.Next() {
+ n++
+ rec := rdr.Record()
+ assert.NotNil(t, rec)
+ assert.EqualValues(t, 1, rec.NumRows())
+ assert.EqualValues(t, 3, rec.NumCols())
+ }
+
+ assert.NoError(t, rdr.Err())
+ assert.Equal(t, 16, n)
+}
+
+func TestJSONReaderAll(t *testing.T) {
+ schema := arrow.NewSchema([]arrow.Field{
+ {Name: "region", Type: arrow.BinaryTypes.String, Nullable: true},
+ {Name: "model", Type: arrow.BinaryTypes.String},
+ {Name: "sales", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
+ }, nil)
+
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ rdr := array.NewJSONReader(strings.NewReader(jsondata), schema, array.WithAllocator(mem), array.WithChunk(-1))
+ defer rdr.Release()
+
+ assert.True(t, rdr.Next())
+ rec := rdr.Record()
+ assert.NotNil(t, rec)
+ assert.NoError(t, rdr.Err())
+
+ assert.EqualValues(t, 16, rec.NumRows())
+ assert.EqualValues(t, 3, rec.NumCols())
+ assert.False(t, rdr.Next())
+}
+
+func TestJSONReaderChunked(t *testing.T) {
+ schema := arrow.NewSchema([]arrow.Field{
+ {Name: "region", Type: arrow.BinaryTypes.String, Nullable: true},
+ {Name: "model", Type: arrow.BinaryTypes.String},
+ {Name: "sales", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
+ }, nil)
+
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ rdr := array.NewJSONReader(strings.NewReader(jsondata), schema, array.WithAllocator(mem), array.WithChunk(4))
+ defer rdr.Release()
+
+ n := 0
+ for rdr.Next() {
+ n++
+ rec := rdr.Record()
+ assert.NotNil(t, rec)
+ assert.NoError(t, rdr.Err())
+ assert.EqualValues(t, 4, rec.NumRows())
+ }
+
+ assert.Equal(t, 4, n)
+ assert.NoError(t, rdr.Err())
+}
diff --git a/go/arrow/array/list.go b/go/arrow/array/list.go
index 1b81da0..9ebc0a4 100644
--- a/go/arrow/array/list.go
+++ b/go/arrow/array/list.go
@@ -17,6 +17,7 @@
package array
import (
+ "bytes"
"fmt"
"strings"
"sync/atomic"
@@ -25,6 +26,7 @@ import (
"github.com/apache/arrow/go/v7/arrow/bitutil"
"github.com/apache/arrow/go/v7/arrow/internal/debug"
"github.com/apache/arrow/go/v7/arrow/memory"
+ "github.com/goccy/go-json"
)
// List represents an immutable sequence of array values.
@@ -79,6 +81,37 @@ func (a *List) setData(data *Data) {
a.values = MakeFromData(data.childData[0])
}
+func (a *List) getOneForMarshal(i int) interface{} {
+ if a.IsNull(i) {
+ return nil
+ }
+
+ slice := a.newListValue(i)
+ defer slice.Release()
+ v, err := json.Marshal(slice)
+ if err != nil {
+ panic(err)
+ }
+ return json.RawMessage(v)
+}
+
+func (a *List) MarshalJSON() ([]byte, error) {
+ var buf bytes.Buffer
+ enc := json.NewEncoder(&buf)
+
+ buf.WriteByte('[')
+ for i := 0; i < a.Len(); i++ {
+ if i != 0 {
+ buf.WriteByte(',')
+ }
+ if err := enc.Encode(a.getOneForMarshal(i)); err != nil {
+ return nil, err
+ }
+ }
+ buf.WriteByte(']')
+ return buf.Bytes(), nil
+}
+
func arrayEqualList(left, right *List) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
@@ -263,6 +296,56 @@ func (b *ListBuilder) newData() (data *Data) {
return
}
+func (b *ListBuilder) unmarshalOne(dec *json.Decoder) error {
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ switch t {
+ case json.Delim('['):
+ b.Append(true)
+ if err := b.values.unmarshal(dec); err != nil {
+ return err
+ }
+ // consume ']'
+ _, err := dec.Token()
+ return err
+ case nil:
+ b.AppendNull()
+ default:
+ return &json.UnmarshalTypeError{
+ Value: fmt.Sprint(t),
+ Struct: arrow.ListOf(b.etype).String(),
+ }
+ }
+
+ return nil
+}
+
+func (b *ListBuilder) unmarshal(dec *json.Decoder) error {
+ for dec.More() {
+ if err := b.unmarshalOne(dec); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (b *ListBuilder) UnmarshalJSON(data []byte) error {
+ dec := json.NewDecoder(bytes.NewReader(data))
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ if delim, ok := t.(json.Delim); !ok || delim != '[' {
+ return fmt.Errorf("list builder must unpack from json array, found %s", delim)
+ }
+
+ return b.unmarshal(dec)
+}
+
var (
_ Interface = (*List)(nil)
_ Builder = (*ListBuilder)(nil)
diff --git a/go/arrow/array/map.go b/go/arrow/array/map.go
index 7728420..cbcc088 100644
--- a/go/arrow/array/map.go
+++ b/go/arrow/array/map.go
@@ -17,8 +17,12 @@
package array
import (
+ "bytes"
+ "fmt"
+
"github.com/apache/arrow/go/v7/arrow"
"github.com/apache/arrow/go/v7/arrow/memory"
+ "github.com/goccy/go-json"
)
// Map represents an immutable sequence of Key/Value structs. It is a
@@ -266,6 +270,28 @@ func (b *MapBuilder) ValueBuilder() *StructBuilder {
return b.listBuilder.ValueBuilder().(*StructBuilder)
}
+func (b *MapBuilder) unmarshalOne(dec *json.Decoder) error {
+ return b.listBuilder.unmarshalOne(dec)
+}
+
+func (b *MapBuilder) unmarshal(dec *json.Decoder) error {
+ return b.listBuilder.unmarshal(dec)
+}
+
+func (b *MapBuilder) UnmarshalJSON(data []byte) error {
+ dec := json.NewDecoder(bytes.NewReader(data))
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ if delim, ok := t.(json.Delim); !ok || delim != '[' {
+ return fmt.Errorf("map builder must unpack from json array, found %s", delim)
+ }
+
+ return b.unmarshal(dec)
+}
+
var (
_ Interface = (*Map)(nil)
_ Builder = (*MapBuilder)(nil)
diff --git a/go/arrow/array/null.go b/go/arrow/array/null.go
index 945232e..288ccb4 100644
--- a/go/arrow/array/null.go
+++ b/go/arrow/array/null.go
@@ -17,12 +17,16 @@
package array
import (
+ "bytes"
+ "fmt"
+ "reflect"
"strings"
"sync/atomic"
"github.com/apache/arrow/go/v7/arrow"
"github.com/apache/arrow/go/v7/arrow/internal/debug"
"github.com/apache/arrow/go/v7/arrow/memory"
+ "github.com/goccy/go-json"
)
// Null represents an immutable, degenerate array with no physical storage.
@@ -73,6 +77,14 @@ func (a *Null) setData(data *Data) {
a.array.data.nulls = a.array.data.length
}
+func (a *Null) getOneForMarshal(i int) interface{} {
+ return nil
+}
+
+func (a *Null) MarshalJSON() ([]byte, error) {
+ return json.Marshal(make([]interface{}, a.Len()))
+}
+
type NullBuilder struct {
builder
}
@@ -134,6 +146,48 @@ func (b *NullBuilder) newData() (data *Data) {
return
}
+func (b *NullBuilder) unmarshalOne(dec *json.Decoder) error {
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ switch t.(type) {
+ case nil:
+ b.AppendNull()
+ default:
+ return &json.UnmarshalTypeError{
+ Value: fmt.Sprint(t),
+ Type: reflect.TypeOf(nil),
+ Offset: dec.InputOffset(),
+ }
+ }
+ return nil
+}
+
+func (b *NullBuilder) unmarshal(dec *json.Decoder) error {
+ for dec.More() {
+ if err := b.unmarshalOne(dec); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (b *NullBuilder) UnmarshalJSON(data []byte) error {
+ dec := json.NewDecoder(bytes.NewReader(data))
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ if delim, ok := t.(json.Delim); !ok || delim != '[' {
+ return fmt.Errorf("null builder must unpack from json array, found %s", delim)
+ }
+
+ return b.unmarshal(dec)
+}
+
var (
_ Interface = (*Null)(nil)
_ Builder = (*NullBuilder)(nil)
diff --git a/go/arrow/array/numeric.gen.go b/go/arrow/array/numeric.gen.go
index f958c71..5a9cc91 100644
--- a/go/arrow/array/numeric.gen.go
+++ b/go/arrow/array/numeric.gen.go
@@ -23,6 +23,7 @@ import (
"strings"
"github.com/apache/arrow/go/v7/arrow"
+ "github.com/goccy/go-json"
)
// A type which represents an immutable sequence of int64 values.
@@ -80,6 +81,27 @@ func (a *Int64) setData(data *Data) {
}
}
+func (a *Int64) getOneForMarshal(i int) interface{} {
+ if a.IsNull(i) {
+ return nil
+ }
+
+ return a.values[i]
+}
+
+func (a *Int64) MarshalJSON() ([]byte, error) {
+ vals := make([]interface{}, a.Len())
+ for i := 0; i < a.Len(); i++ {
+ if a.IsValid(i) {
+ vals[i] = float64(a.values[i]) // prevent uint8 from being seen as binary data
+ } else {
+ vals[i] = nil
+ }
+ }
+
+ return json.Marshal(vals)
+}
+
func arrayEqualInt64(left, right *Int64) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
@@ -147,6 +169,27 @@ func (a *Uint64) setData(data *Data) {
}
}
+func (a *Uint64) getOneForMarshal(i int) interface{} {
+ if a.IsNull(i) {
+ return nil
+ }
+
+ return a.values[i]
+}
+
+func (a *Uint64) MarshalJSON() ([]byte, error) {
+ vals := make([]interface{}, a.Len())
+ for i := 0; i < a.Len(); i++ {
+ if a.IsValid(i) {
+ vals[i] = float64(a.values[i]) // prevent uint8 from being seen as binary data
+ } else {
+ vals[i] = nil
+ }
+ }
+
+ return json.Marshal(vals)
+}
+
func arrayEqualUint64(left, right *Uint64) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
@@ -214,6 +257,27 @@ func (a *Float64) setData(data *Data) {
}
}
+func (a *Float64) getOneForMarshal(i int) interface{} {
+ if a.IsNull(i) {
+ return nil
+ }
+
+ return a.values[i]
+}
+
+func (a *Float64) MarshalJSON() ([]byte, error) {
+ vals := make([]interface{}, a.Len())
+ for i := 0; i < a.Len(); i++ {
+ if a.IsValid(i) {
+ vals[i] = float64(a.values[i]) // prevent uint8 from being seen as binary data
+ } else {
+ vals[i] = nil
+ }
+ }
+
+ return json.Marshal(vals)
+}
+
func arrayEqualFloat64(left, right *Float64) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
@@ -281,6 +345,27 @@ func (a *Int32) setData(data *Data) {
}
}
+func (a *Int32) getOneForMarshal(i int) interface{} {
+ if a.IsNull(i) {
+ return nil
+ }
+
+ return a.values[i]
+}
+
+func (a *Int32) MarshalJSON() ([]byte, error) {
+ vals := make([]interface{}, a.Len())
+ for i := 0; i < a.Len(); i++ {
+ if a.IsValid(i) {
+ vals[i] = float64(a.values[i]) // prevent uint8 from being seen as binary data
+ } else {
+ vals[i] = nil
+ }
+ }
+
+ return json.Marshal(vals)
+}
+
func arrayEqualInt32(left, right *Int32) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
@@ -348,6 +433,27 @@ func (a *Uint32) setData(data *Data) {
}
}
+func (a *Uint32) getOneForMarshal(i int) interface{} {
+ if a.IsNull(i) {
+ return nil
+ }
+
+ return a.values[i]
+}
+
+func (a *Uint32) MarshalJSON() ([]byte, error) {
+ vals := make([]interface{}, a.Len())
+ for i := 0; i < a.Len(); i++ {
+ if a.IsValid(i) {
+ vals[i] = float64(a.values[i]) // prevent uint8 from being seen as binary data
+ } else {
+ vals[i] = nil
+ }
+ }
+
+ return json.Marshal(vals)
+}
+
func arrayEqualUint32(left, right *Uint32) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
@@ -415,6 +521,27 @@ func (a *Float32) setData(data *Data) {
}
}
+func (a *Float32) getOneForMarshal(i int) interface{} {
+ if a.IsNull(i) {
+ return nil
+ }
+
+ return a.values[i]
+}
+
+func (a *Float32) MarshalJSON() ([]byte, error) {
+ vals := make([]interface{}, a.Len())
+ for i := 0; i < a.Len(); i++ {
+ if a.IsValid(i) {
+ vals[i] = float64(a.values[i]) // prevent uint8 from being seen as binary data
+ } else {
+ vals[i] = nil
+ }
+ }
+
+ return json.Marshal(vals)
+}
+
func arrayEqualFloat32(left, right *Float32) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
@@ -482,6 +609,27 @@ func (a *Int16) setData(data *Data) {
}
}
+func (a *Int16) getOneForMarshal(i int) interface{} {
+ if a.IsNull(i) {
+ return nil
+ }
+
+ return a.values[i]
+}
+
+func (a *Int16) MarshalJSON() ([]byte, error) {
+ vals := make([]interface{}, a.Len())
+ for i := 0; i < a.Len(); i++ {
+ if a.IsValid(i) {
+ vals[i] = float64(a.values[i]) // prevent uint8 from being seen as binary data
+ } else {
+ vals[i] = nil
+ }
+ }
+
+ return json.Marshal(vals)
+}
+
func arrayEqualInt16(left, right *Int16) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
@@ -549,6 +697,27 @@ func (a *Uint16) setData(data *Data) {
}
}
+func (a *Uint16) getOneForMarshal(i int) interface{} {
+ if a.IsNull(i) {
+ return nil
+ }
+
+ return a.values[i]
+}
+
+func (a *Uint16) MarshalJSON() ([]byte, error) {
+ vals := make([]interface{}, a.Len())
+ for i := 0; i < a.Len(); i++ {
+ if a.IsValid(i) {
+ vals[i] = float64(a.values[i]) // prevent uint8 from being seen as binary data
+ } else {
+ vals[i] = nil
+ }
+ }
+
+ return json.Marshal(vals)
+}
+
func arrayEqualUint16(left, right *Uint16) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
@@ -616,6 +785,27 @@ func (a *Int8) setData(data *Data) {
}
}
+func (a *Int8) getOneForMarshal(i int) interface{} {
+ if a.IsNull(i) {
+ return nil
+ }
+
+ return float64(a.values[i]) // prevent uint8 from being seen as binary data
+}
+
+func (a *Int8) MarshalJSON() ([]byte, error) {
+ vals := make([]interface{}, a.Len())
+ for i := 0; i < a.Len(); i++ {
+ if a.IsValid(i) {
+ vals[i] = float64(a.values[i]) // prevent uint8 from being seen as binary data
+ } else {
+ vals[i] = nil
+ }
+ }
+
+ return json.Marshal(vals)
+}
+
func arrayEqualInt8(left, right *Int8) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
@@ -683,6 +873,27 @@ func (a *Uint8) setData(data *Data) {
}
}
+func (a *Uint8) getOneForMarshal(i int) interface{} {
+ if a.IsNull(i) {
+ return nil
+ }
+
+ return float64(a.values[i]) // prevent uint8 from being seen as binary data
+}
+
+func (a *Uint8) MarshalJSON() ([]byte, error) {
+ vals := make([]interface{}, a.Len())
+ for i := 0; i < a.Len(); i++ {
+ if a.IsValid(i) {
+ vals[i] = float64(a.values[i]) // prevent uint8 from being seen as binary data
+ } else {
+ vals[i] = nil
+ }
+ }
+
+ return json.Marshal(vals)
+}
+
func arrayEqualUint8(left, right *Uint8) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
@@ -750,6 +961,22 @@ func (a *Timestamp) setData(data *Data) {
}
}
+func (a *Timestamp) getOneForMarshal(i int) interface{} {
+ if a.IsNull(i) {
+ return nil
+ }
+ return a.values[i].ToTime(a.DataType().(*arrow.TimestampType).Unit).Format("2006-01-02 15:04:05.999999999")
+}
+
+func (a *Timestamp) MarshalJSON() ([]byte, error) {
+ vals := make([]interface{}, a.Len())
+ for i := range a.values {
+ vals[i] = a.getOneForMarshal(i)
+ }
+
+ return json.Marshal(vals)
+}
+
func arrayEqualTimestamp(left, right *Timestamp) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
@@ -817,6 +1044,22 @@ func (a *Time32) setData(data *Data) {
}
}
+func (a *Time32) getOneForMarshal(i int) interface{} {
+ if a.IsNull(i) {
+ return nil
+ }
+ return a.values[i].ToTime(a.DataType().(*arrow.Time32Type).Unit).Format("15:04:05.999999999")
+}
+
+func (a *Time32) MarshalJSON() ([]byte, error) {
+ vals := make([]interface{}, a.Len())
+ for i := range a.values {
+ vals[i] = a.getOneForMarshal(i)
+ }
+
+ return json.Marshal(vals)
+}
+
func arrayEqualTime32(left, right *Time32) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
@@ -884,6 +1127,22 @@ func (a *Time64) setData(data *Data) {
}
}
+func (a *Time64) getOneForMarshal(i int) interface{} {
+ if a.IsNull(i) {
+ return nil
+ }
+ return a.values[i].ToTime(a.DataType().(*arrow.Time64Type).Unit).Format("15:04:05.999999999")
+}
+
+func (a *Time64) MarshalJSON() ([]byte, error) {
+ vals := make([]interface{}, a.Len())
+ for i := range a.values {
+ vals[i] = a.getOneForMarshal(i)
+ }
+
+ return json.Marshal(vals)
+}
+
func arrayEqualTime64(left, right *Time64) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
@@ -951,6 +1210,22 @@ func (a *Date32) setData(data *Data) {
}
}
+func (a *Date32) getOneForMarshal(i int) interface{} {
+ if a.IsNull(i) {
+ return nil
+ }
+ return a.values[i].ToTime().Format("2006-01-02")
+}
+
+func (a *Date32) MarshalJSON() ([]byte, error) {
+ vals := make([]interface{}, a.Len())
+ for i := range a.values {
+ vals[i] = a.getOneForMarshal(i)
+ }
+
+ return json.Marshal(vals)
+}
+
func arrayEqualDate32(left, right *Date32) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
@@ -1018,6 +1293,22 @@ func (a *Date64) setData(data *Data) {
}
}
+func (a *Date64) getOneForMarshal(i int) interface{} {
+ if a.IsNull(i) {
+ return nil
+ }
+ return a.values[i].ToTime().Format("2006-01-02")
+}
+
+func (a *Date64) MarshalJSON() ([]byte, error) {
+ vals := make([]interface{}, a.Len())
+ for i := range a.values {
+ vals[i] = a.getOneForMarshal(i)
+ }
+
+ return json.Marshal(vals)
+}
+
func arrayEqualDate64(left, right *Date64) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
@@ -1085,6 +1376,23 @@ func (a *Duration) setData(data *Data) {
}
}
+func (a *Duration) getOneForMarshal(i int) interface{} {
+ if a.IsNull(i) {
+ return nil
+ }
+ // return value and suffix as a string such as "12345ms"
+ return fmt.Sprintf("%d%s", a.values[i], a.DataType().(*arrow.DurationType).Unit.String())
+}
+
+func (a *Duration) MarshalJSON() ([]byte, error) {
+ vals := make([]interface{}, a.Len())
+ for i := range a.values {
+ vals[i] = a.getOneForMarshal(i)
+ }
+
+ return json.Marshal(vals)
+}
+
func arrayEqualDuration(left, right *Duration) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
diff --git a/go/arrow/array/numeric.gen.go.tmpl b/go/arrow/array/numeric.gen.go.tmpl
index 70cf40f..5a6b46a 100644
--- a/go/arrow/array/numeric.gen.go.tmpl
+++ b/go/arrow/array/numeric.gen.go.tmpl
@@ -19,8 +19,10 @@ package array
import (
"fmt"
"strings"
+ "time"
- "github.com/apache/arrow/go/v7/arrow"
+ "github.com/apache/arrow/go/v7/arrow"
+ "github.com/goccy/go-json"
)
{{range .In}}
@@ -80,6 +82,45 @@ func (a *{{.Name}}) setData(data *Data) {
}
}
+func (a *{{.Name}}) getOneForMarshal(i int) interface{} {
+ if a.IsNull(i) {
+ return nil
+ }
+{{if or (eq .Name "Date32") (eq .Name "Date64") -}}
+ return a.values[i].ToTime().Format("2006-01-02")
+{{else if or (eq .Name "Time32") (eq .Name "Time64") -}}
+ return a.values[i].ToTime(a.DataType().(*{{.QualifiedType}}Type).Unit).Format("15:04:05.999999999")
+{{else if or (eq .Name "Timestamp") -}}
+ return a.values[i].ToTime(a.DataType().(*{{.QualifiedType}}Type).Unit).Format("2006-01-02 15:04:05.999999999")
+{{else if (eq .Name "Duration") -}}
+ // return value and suffix as a string such as "12345ms"
+ return fmt.Sprintf("%d%s", a.values[i], a.DataType().(*{{.QualifiedType}}Type).Unit.String())
+{{else if (eq .Size "1")}}
+ return float64(a.values[i]) // prevent uint8 from being seen as binary data
+{{else}}
+ return a.values[i]
+{{end -}}
+}
+
+func (a *{{.Name}}) MarshalJSON() ([]byte, error) {
+{{if .QualifiedType -}}
+ vals := make([]interface{}, a.Len())
+ for i := range a.values {
+ vals[i] = a.getOneForMarshal(i)
+ }
+{{else -}}
+ vals := make([]interface{}, a.Len())
+ for i := 0; i < a.Len(); i++ {
+ if a.IsValid(i) {
+ vals[i] = float64(a.values[i]) // prevent uint8 from being seen as binary data
+ } else {
+ vals[i] = nil
+ }
+ }
+{{end}}
+ return json.Marshal(vals)
+}
+
func arrayEqual{{.Name}}(left, right *{{.Name}}) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
diff --git a/go/arrow/array/numericbuilder.gen.go b/go/arrow/array/numericbuilder.gen.go
index 0802d81..9786caf 100644
--- a/go/arrow/array/numericbuilder.gen.go
+++ b/go/arrow/array/numericbuilder.gen.go
@@ -19,12 +19,19 @@
package array
import (
+ "bytes"
+ "fmt"
+ "reflect"
+ "strconv"
+ "strings"
"sync/atomic"
+ "time"
"github.com/apache/arrow/go/v7/arrow"
"github.com/apache/arrow/go/v7/arrow/bitutil"
"github.com/apache/arrow/go/v7/arrow/internal/debug"
"github.com/apache/arrow/go/v7/arrow/memory"
+ "github.com/goccy/go-json"
)
type Int64Builder struct {
@@ -163,6 +170,73 @@ func (b *Int64Builder) newData() (data *Data) {
return
}
+func (b *Int64Builder) unmarshalOne(dec *json.Decoder) error {
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ switch v := t.(type) {
+ case nil:
+ b.AppendNull()
+
+ case string:
+ f, err := strconv.ParseInt(v, 10, 8*8)
+ if err != nil {
+ return &json.UnmarshalTypeError{
+ Value: v,
+ Type: reflect.TypeOf(int64(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+ b.Append(int64(f))
+ case float64:
+ b.Append(int64(v))
+ case json.Number:
+ f, err := strconv.ParseInt(v.String(), 10, 8*8)
+ if err != nil {
+ return &json.UnmarshalTypeError{
+ Value: v.String(),
+ Type: reflect.TypeOf(int64(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+ b.Append(int64(f))
+
+ default:
+ return &json.UnmarshalTypeError{
+ Value: fmt.Sprint(t),
+ Type: reflect.TypeOf(int64(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+
+ return nil
+}
+
+func (b *Int64Builder) unmarshal(dec *json.Decoder) error {
+ for dec.More() {
+ if err := b.unmarshalOne(dec); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (b *Int64Builder) UnmarshalJSON(data []byte) error {
+ dec := json.NewDecoder(bytes.NewReader(data))
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ if delim, ok := t.(json.Delim); !ok || delim != '[' {
+ return fmt.Errorf("binary builder must unpack from json array, found %s", delim)
+ }
+
+ return b.unmarshal(dec)
+}
+
type Uint64Builder struct {
builder
@@ -299,6 +373,73 @@ func (b *Uint64Builder) newData() (data *Data) {
return
}
+func (b *Uint64Builder) unmarshalOne(dec *json.Decoder) error {
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ switch v := t.(type) {
+ case nil:
+ b.AppendNull()
+
+ case string:
+ f, err := strconv.ParseUint(v, 10, 8*8)
+ if err != nil {
+ return &json.UnmarshalTypeError{
+ Value: v,
+ Type: reflect.TypeOf(uint64(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+ b.Append(uint64(f))
+ case float64:
+ b.Append(uint64(v))
+ case json.Number:
+ f, err := strconv.ParseUint(v.String(), 10, 8*8)
+ if err != nil {
+ return &json.UnmarshalTypeError{
+ Value: v.String(),
+ Type: reflect.TypeOf(uint64(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+ b.Append(uint64(f))
+
+ default:
+ return &json.UnmarshalTypeError{
+ Value: fmt.Sprint(t),
+ Type: reflect.TypeOf(uint64(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+
+ return nil
+}
+
+func (b *Uint64Builder) unmarshal(dec *json.Decoder) error {
+ for dec.More() {
+ if err := b.unmarshalOne(dec); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (b *Uint64Builder) UnmarshalJSON(data []byte) error {
+ dec := json.NewDecoder(bytes.NewReader(data))
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ if delim, ok := t.(json.Delim); !ok || delim != '[' {
+ return fmt.Errorf("binary builder must unpack from json array, found %s", delim)
+ }
+
+ return b.unmarshal(dec)
+}
+
type Float64Builder struct {
builder
@@ -435,6 +576,73 @@ func (b *Float64Builder) newData() (data *Data) {
return
}
+func (b *Float64Builder) unmarshalOne(dec *json.Decoder) error {
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ switch v := t.(type) {
+ case nil:
+ b.AppendNull()
+
+ case string:
+ f, err := strconv.ParseFloat(v, 8*8)
+ if err != nil {
+ return &json.UnmarshalTypeError{
+ Value: v,
+ Type: reflect.TypeOf(float64(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+ b.Append(float64(f))
+ case float64:
+ b.Append(float64(v))
+ case json.Number:
+ f, err := strconv.ParseFloat(v.String(), 8*8)
+ if err != nil {
+ return &json.UnmarshalTypeError{
+ Value: v.String(),
+ Type: reflect.TypeOf(float64(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+ b.Append(float64(f))
+
+ default:
+ return &json.UnmarshalTypeError{
+ Value: fmt.Sprint(t),
+ Type: reflect.TypeOf(float64(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+
+ return nil
+}
+
+func (b *Float64Builder) unmarshal(dec *json.Decoder) error {
+ for dec.More() {
+ if err := b.unmarshalOne(dec); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (b *Float64Builder) UnmarshalJSON(data []byte) error {
+ dec := json.NewDecoder(bytes.NewReader(data))
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ if delim, ok := t.(json.Delim); !ok || delim != '[' {
+ return fmt.Errorf("binary builder must unpack from json array, found %s", delim)
+ }
+
+ return b.unmarshal(dec)
+}
+
type Int32Builder struct {
builder
@@ -571,6 +779,73 @@ func (b *Int32Builder) newData() (data *Data) {
return
}
+func (b *Int32Builder) unmarshalOne(dec *json.Decoder) error {
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ switch v := t.(type) {
+ case nil:
+ b.AppendNull()
+
+ case string:
+ f, err := strconv.ParseInt(v, 10, 4*8)
+ if err != nil {
+ return &json.UnmarshalTypeError{
+ Value: v,
+ Type: reflect.TypeOf(int32(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+ b.Append(int32(f))
+ case float64:
+ b.Append(int32(v))
+ case json.Number:
+ f, err := strconv.ParseInt(v.String(), 10, 4*8)
+ if err != nil {
+ return &json.UnmarshalTypeError{
+ Value: v.String(),
+ Type: reflect.TypeOf(int32(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+ b.Append(int32(f))
+
+ default:
+ return &json.UnmarshalTypeError{
+ Value: fmt.Sprint(t),
+ Type: reflect.TypeOf(int32(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+
+ return nil
+}
+
+func (b *Int32Builder) unmarshal(dec *json.Decoder) error {
+ for dec.More() {
+ if err := b.unmarshalOne(dec); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (b *Int32Builder) UnmarshalJSON(data []byte) error {
+ dec := json.NewDecoder(bytes.NewReader(data))
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ if delim, ok := t.(json.Delim); !ok || delim != '[' {
+ return fmt.Errorf("binary builder must unpack from json array, found %s", delim)
+ }
+
+ return b.unmarshal(dec)
+}
+
type Uint32Builder struct {
builder
@@ -707,6 +982,73 @@ func (b *Uint32Builder) newData() (data *Data) {
return
}
+func (b *Uint32Builder) unmarshalOne(dec *json.Decoder) error {
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ switch v := t.(type) {
+ case nil:
+ b.AppendNull()
+
+ case string:
+ f, err := strconv.ParseUint(v, 10, 4*8)
+ if err != nil {
+ return &json.UnmarshalTypeError{
+ Value: v,
+ Type: reflect.TypeOf(uint32(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+ b.Append(uint32(f))
+ case float64:
+ b.Append(uint32(v))
+ case json.Number:
+ f, err := strconv.ParseUint(v.String(), 10, 4*8)
+ if err != nil {
+ return &json.UnmarshalTypeError{
+ Value: v.String(),
+ Type: reflect.TypeOf(uint32(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+ b.Append(uint32(f))
+
+ default:
+ return &json.UnmarshalTypeError{
+ Value: fmt.Sprint(t),
+ Type: reflect.TypeOf(uint32(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+
+ return nil
+}
+
+func (b *Uint32Builder) unmarshal(dec *json.Decoder) error {
+ for dec.More() {
+ if err := b.unmarshalOne(dec); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (b *Uint32Builder) UnmarshalJSON(data []byte) error {
+ dec := json.NewDecoder(bytes.NewReader(data))
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ if delim, ok := t.(json.Delim); !ok || delim != '[' {
+ return fmt.Errorf("binary builder must unpack from json array, found %s", delim)
+ }
+
+ return b.unmarshal(dec)
+}
+
type Float32Builder struct {
builder
@@ -843,6 +1185,73 @@ func (b *Float32Builder) newData() (data *Data) {
return
}
+func (b *Float32Builder) unmarshalOne(dec *json.Decoder) error {
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ switch v := t.(type) {
+ case nil:
+ b.AppendNull()
+
+ case string:
+ f, err := strconv.ParseFloat(v, 4*8)
+ if err != nil {
+ return &json.UnmarshalTypeError{
+ Value: v,
+ Type: reflect.TypeOf(float32(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+ b.Append(float32(f))
+ case float64:
+ b.Append(float32(v))
+ case json.Number:
+ f, err := strconv.ParseFloat(v.String(), 4*8)
+ if err != nil {
+ return &json.UnmarshalTypeError{
+ Value: v.String(),
+ Type: reflect.TypeOf(float32(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+ b.Append(float32(f))
+
+ default:
+ return &json.UnmarshalTypeError{
+ Value: fmt.Sprint(t),
+ Type: reflect.TypeOf(float32(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+
+ return nil
+}
+
+func (b *Float32Builder) unmarshal(dec *json.Decoder) error {
+ for dec.More() {
+ if err := b.unmarshalOne(dec); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (b *Float32Builder) UnmarshalJSON(data []byte) error {
+ dec := json.NewDecoder(bytes.NewReader(data))
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ if delim, ok := t.(json.Delim); !ok || delim != '[' {
+ return fmt.Errorf("binary builder must unpack from json array, found %s", delim)
+ }
+
+ return b.unmarshal(dec)
+}
+
type Int16Builder struct {
builder
@@ -979,6 +1388,73 @@ func (b *Int16Builder) newData() (data *Data) {
return
}
+func (b *Int16Builder) unmarshalOne(dec *json.Decoder) error {
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ switch v := t.(type) {
+ case nil:
+ b.AppendNull()
+
+ case string:
+ f, err := strconv.ParseInt(v, 10, 2*8)
+ if err != nil {
+ return &json.UnmarshalTypeError{
+ Value: v,
+ Type: reflect.TypeOf(int16(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+ b.Append(int16(f))
+ case float64:
+ b.Append(int16(v))
+ case json.Number:
+ f, err := strconv.ParseInt(v.String(), 10, 2*8)
+ if err != nil {
+ return &json.UnmarshalTypeError{
+ Value: v.String(),
+ Type: reflect.TypeOf(int16(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+ b.Append(int16(f))
+
+ default:
+ return &json.UnmarshalTypeError{
+ Value: fmt.Sprint(t),
+ Type: reflect.TypeOf(int16(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+
+ return nil
+}
+
+func (b *Int16Builder) unmarshal(dec *json.Decoder) error {
+ for dec.More() {
+ if err := b.unmarshalOne(dec); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (b *Int16Builder) UnmarshalJSON(data []byte) error {
+ dec := json.NewDecoder(bytes.NewReader(data))
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ if delim, ok := t.(json.Delim); !ok || delim != '[' {
+ return fmt.Errorf("binary builder must unpack from json array, found %s", delim)
+ }
+
+ return b.unmarshal(dec)
+}
+
type Uint16Builder struct {
builder
@@ -1115,6 +1591,73 @@ func (b *Uint16Builder) newData() (data *Data) {
return
}
+func (b *Uint16Builder) unmarshalOne(dec *json.Decoder) error {
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ switch v := t.(type) {
+ case nil:
+ b.AppendNull()
+
+ case string:
+ f, err := strconv.ParseUint(v, 10, 2*8)
+ if err != nil {
+ return &json.UnmarshalTypeError{
+ Value: v,
+ Type: reflect.TypeOf(uint16(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+ b.Append(uint16(f))
+ case float64:
+ b.Append(uint16(v))
+ case json.Number:
+ f, err := strconv.ParseUint(v.String(), 10, 2*8)
+ if err != nil {
+ return &json.UnmarshalTypeError{
+ Value: v.String(),
+ Type: reflect.TypeOf(uint16(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+ b.Append(uint16(f))
+
+ default:
+ return &json.UnmarshalTypeError{
+ Value: fmt.Sprint(t),
+ Type: reflect.TypeOf(uint16(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+
+ return nil
+}
+
+func (b *Uint16Builder) unmarshal(dec *json.Decoder) error {
+ for dec.More() {
+ if err := b.unmarshalOne(dec); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (b *Uint16Builder) UnmarshalJSON(data []byte) error {
+ dec := json.NewDecoder(bytes.NewReader(data))
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ if delim, ok := t.(json.Delim); !ok || delim != '[' {
+ return fmt.Errorf("binary builder must unpack from json array, found %s", delim)
+ }
+
+ return b.unmarshal(dec)
+}
+
type Int8Builder struct {
builder
@@ -1251,6 +1794,73 @@ func (b *Int8Builder) newData() (data *Data) {
return
}
+func (b *Int8Builder) unmarshalOne(dec *json.Decoder) error {
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ switch v := t.(type) {
+ case nil:
+ b.AppendNull()
+
+ case string:
+ f, err := strconv.ParseInt(v, 10, 1*8)
+ if err != nil {
+ return &json.UnmarshalTypeError{
+ Value: v,
+ Type: reflect.TypeOf(int8(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+ b.Append(int8(f))
+ case float64:
+ b.Append(int8(v))
+ case json.Number:
+ f, err := strconv.ParseInt(v.String(), 10, 1*8)
+ if err != nil {
+ return &json.UnmarshalTypeError{
+ Value: v.String(),
+ Type: reflect.TypeOf(int8(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+ b.Append(int8(f))
+
+ default:
+ return &json.UnmarshalTypeError{
+ Value: fmt.Sprint(t),
+ Type: reflect.TypeOf(int8(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+
+ return nil
+}
+
+func (b *Int8Builder) unmarshal(dec *json.Decoder) error {
+ for dec.More() {
+ if err := b.unmarshalOne(dec); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (b *Int8Builder) UnmarshalJSON(data []byte) error {
+ dec := json.NewDecoder(bytes.NewReader(data))
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ if delim, ok := t.(json.Delim); !ok || delim != '[' {
+ return fmt.Errorf("binary builder must unpack from json array, found %s", delim)
+ }
+
+ return b.unmarshal(dec)
+}
+
type Uint8Builder struct {
builder
@@ -1387,6 +1997,73 @@ func (b *Uint8Builder) newData() (data *Data) {
return
}
+func (b *Uint8Builder) unmarshalOne(dec *json.Decoder) error {
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ switch v := t.(type) {
+ case nil:
+ b.AppendNull()
+
+ case string:
+ f, err := strconv.ParseUint(v, 10, 1*8)
+ if err != nil {
+ return &json.UnmarshalTypeError{
+ Value: v,
+ Type: reflect.TypeOf(uint8(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+ b.Append(uint8(f))
+ case float64:
+ b.Append(uint8(v))
+ case json.Number:
+ f, err := strconv.ParseUint(v.String(), 10, 1*8)
+ if err != nil {
+ return &json.UnmarshalTypeError{
+ Value: v.String(),
+ Type: reflect.TypeOf(uint8(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+ b.Append(uint8(f))
+
+ default:
+ return &json.UnmarshalTypeError{
+ Value: fmt.Sprint(t),
+ Type: reflect.TypeOf(uint8(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+
+ return nil
+}
+
+func (b *Uint8Builder) unmarshal(dec *json.Decoder) error {
+ for dec.More() {
+ if err := b.unmarshalOne(dec); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (b *Uint8Builder) UnmarshalJSON(data []byte) error {
+ dec := json.NewDecoder(bytes.NewReader(data))
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ if delim, ok := t.(json.Delim); !ok || delim != '[' {
+ return fmt.Errorf("binary builder must unpack from json array, found %s", delim)
+ }
+
+ return b.unmarshal(dec)
+}
+
type TimestampBuilder struct {
builder
@@ -1524,6 +2201,61 @@ func (b *TimestampBuilder) newData() (data *Data) {
return
}
+func (b *TimestampBuilder) unmarshalOne(dec *json.Decoder) error {
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ switch v := t.(type) {
+ case nil:
+ b.AppendNull()
+ case string:
+ tm, err := arrow.TimestampFromString(v, b.dtype.Unit)
+ if err != nil {
+ return &json.UnmarshalTypeError{
+ Value: v,
+ Type: reflect.TypeOf(arrow.Timestamp(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+
+ b.Append(tm)
+
+ default:
+ return &json.UnmarshalTypeError{
+ Value: fmt.Sprint(t),
+ Type: reflect.TypeOf(arrow.Timestamp(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+
+ return nil
+}
+
+func (b *TimestampBuilder) unmarshal(dec *json.Decoder) error {
+ for dec.More() {
+ if err := b.unmarshalOne(dec); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (b *TimestampBuilder) UnmarshalJSON(data []byte) error {
+ dec := json.NewDecoder(bytes.NewReader(data))
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ if delim, ok := t.(json.Delim); !ok || delim != '[' {
+ return fmt.Errorf("binary builder must unpack from json array, found %s", delim)
+ }
+
+ return b.unmarshal(dec)
+}
+
type Time32Builder struct {
builder
@@ -1661,6 +2393,61 @@ func (b *Time32Builder) newData() (data *Data) {
return
}
+func (b *Time32Builder) unmarshalOne(dec *json.Decoder) error {
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ switch v := t.(type) {
+ case nil:
+ b.AppendNull()
+ case string:
+ tm, err := arrow.Time32FromString(v, b.dtype.Unit)
+ if err != nil {
+ return &json.UnmarshalTypeError{
+ Value: v,
+ Type: reflect.TypeOf(arrow.Time32(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+
+ b.Append(tm)
+
+ default:
+ return &json.UnmarshalTypeError{
+ Value: fmt.Sprint(t),
+ Type: reflect.TypeOf(arrow.Time32(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+
+ return nil
+}
+
+func (b *Time32Builder) unmarshal(dec *json.Decoder) error {
+ for dec.More() {
+ if err := b.unmarshalOne(dec); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (b *Time32Builder) UnmarshalJSON(data []byte) error {
+ dec := json.NewDecoder(bytes.NewReader(data))
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ if delim, ok := t.(json.Delim); !ok || delim != '[' {
+ return fmt.Errorf("binary builder must unpack from json array, found %s", delim)
+ }
+
+ return b.unmarshal(dec)
+}
+
type Time64Builder struct {
builder
@@ -1798,6 +2585,61 @@ func (b *Time64Builder) newData() (data *Data) {
return
}
+func (b *Time64Builder) unmarshalOne(dec *json.Decoder) error {
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ switch v := t.(type) {
+ case nil:
+ b.AppendNull()
+ case string:
+ tm, err := arrow.Time64FromString(v, b.dtype.Unit)
+ if err != nil {
+ return &json.UnmarshalTypeError{
+ Value: v,
+ Type: reflect.TypeOf(arrow.Time64(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+
+ b.Append(tm)
+
+ default:
+ return &json.UnmarshalTypeError{
+ Value: fmt.Sprint(t),
+ Type: reflect.TypeOf(arrow.Time64(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+
+ return nil
+}
+
+func (b *Time64Builder) unmarshal(dec *json.Decoder) error {
+ for dec.More() {
+ if err := b.unmarshalOne(dec); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (b *Time64Builder) UnmarshalJSON(data []byte) error {
+ dec := json.NewDecoder(bytes.NewReader(data))
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ if delim, ok := t.(json.Delim); !ok || delim != '[' {
+ return fmt.Errorf("binary builder must unpack from json array, found %s", delim)
+ }
+
+ return b.unmarshal(dec)
+}
+
type Date32Builder struct {
builder
@@ -1934,6 +2776,61 @@ func (b *Date32Builder) newData() (data *Data) {
return
}
+func (b *Date32Builder) unmarshalOne(dec *json.Decoder) error {
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ switch v := t.(type) {
+ case nil:
+ b.AppendNull()
+ case string:
+ tm, err := time.Parse("2006-01-02", v)
+ if err != nil {
+ return &json.UnmarshalTypeError{
+ Value: v,
+ Type: reflect.TypeOf(arrow.Date32(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+
+ b.Append(arrow.Date32FromTime(tm))
+
+ default:
+ return &json.UnmarshalTypeError{
+ Value: fmt.Sprint(t),
+ Type: reflect.TypeOf(arrow.Date32(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+
+ return nil
+}
+
+func (b *Date32Builder) unmarshal(dec *json.Decoder) error {
+ for dec.More() {
+ if err := b.unmarshalOne(dec); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (b *Date32Builder) UnmarshalJSON(data []byte) error {
+ dec := json.NewDecoder(bytes.NewReader(data))
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ if delim, ok := t.(json.Delim); !ok || delim != '[' {
+ return fmt.Errorf("binary builder must unpack from json array, found %s", delim)
+ }
+
+ return b.unmarshal(dec)
+}
+
type Date64Builder struct {
builder
@@ -2070,6 +2967,61 @@ func (b *Date64Builder) newData() (data *Data) {
return
}
+func (b *Date64Builder) unmarshalOne(dec *json.Decoder) error {
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ switch v := t.(type) {
+ case nil:
+ b.AppendNull()
+ case string:
+ tm, err := time.Parse("2006-01-02", v)
+ if err != nil {
+ return &json.UnmarshalTypeError{
+ Value: v,
+ Type: reflect.TypeOf(arrow.Date64(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+
+ b.Append(arrow.Date64FromTime(tm))
+
+ default:
+ return &json.UnmarshalTypeError{
+ Value: fmt.Sprint(t),
+ Type: reflect.TypeOf(arrow.Date64(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+
+ return nil
+}
+
+func (b *Date64Builder) unmarshal(dec *json.Decoder) error {
+ for dec.More() {
+ if err := b.unmarshalOne(dec); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (b *Date64Builder) UnmarshalJSON(data []byte) error {
+ dec := json.NewDecoder(bytes.NewReader(data))
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ if delim, ok := t.(json.Delim); !ok || delim != '[' {
+ return fmt.Errorf("binary builder must unpack from json array, found %s", delim)
+ }
+
+ return b.unmarshal(dec)
+}
+
type DurationBuilder struct {
builder
@@ -2207,6 +3159,84 @@ func (b *DurationBuilder) newData() (data *Data) {
return
}
+func (b *DurationBuilder) unmarshalOne(dec *json.Decoder) error {
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ switch v := t.(type) {
+ case nil:
+ b.AppendNull()
+ case string:
+ // be flexible for specifying durations by accepting forms like
+ // 3h2m0.5s regardless of the unit and converting it to the proper
+ // precision.
+ val, err := time.ParseDuration(v)
+ if err != nil {
+ // if we got an error, maybe it was because the attempt to create
+ // a time.Duration (int64) in nanoseconds would overflow. check if
+ // the string is just a large number followed by the unit suffix
+ if strings.HasSuffix(v, b.dtype.Unit.String()) {
+ value, err := strconv.ParseInt(v[:len(v)-len(b.dtype.Unit.String())], 10, 64)
+ if err == nil {
+ b.Append(arrow.Duration(value))
+ break
+ }
+ }
+
+ return &json.UnmarshalTypeError{
+ Value: v,
+ Type: reflect.TypeOf(arrow.Duration(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+
+ switch b.dtype.Unit {
+ case arrow.Nanosecond:
+ b.Append(arrow.Duration(val.Nanoseconds()))
+ case arrow.Microsecond:
+ b.Append(arrow.Duration(val.Microseconds()))
+ case arrow.Millisecond:
+ b.Append(arrow.Duration(val.Milliseconds()))
+ case arrow.Second:
+ b.Append(arrow.Duration(val.Seconds()))
+ }
+
+ default:
+ return &json.UnmarshalTypeError{
+ Value: fmt.Sprint(t),
+ Type: reflect.TypeOf(arrow.Duration(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+
+ return nil
+}
+
+func (b *DurationBuilder) unmarshal(dec *json.Decoder) error {
+ for dec.More() {
+ if err := b.unmarshalOne(dec); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (b *DurationBuilder) UnmarshalJSON(data []byte) error {
+ dec := json.NewDecoder(bytes.NewReader(data))
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ if delim, ok := t.(json.Delim); !ok || delim != '[' {
+ return fmt.Errorf("binary builder must unpack from json array, found %s", delim)
+ }
+
+ return b.unmarshal(dec)
+}
+
var (
_ Builder = (*Int64Builder)(nil)
_ Builder = (*Uint64Builder)(nil)
diff --git a/go/arrow/array/numericbuilder.gen.go.tmpl b/go/arrow/array/numericbuilder.gen.go.tmpl
index e066d4f..52a0664 100644
--- a/go/arrow/array/numericbuilder.gen.go.tmpl
+++ b/go/arrow/array/numericbuilder.gen.go.tmpl
@@ -20,7 +20,8 @@ import (
"github.com/apache/arrow/go/v7/arrow"
"github.com/apache/arrow/go/v7/arrow/bitutil"
"github.com/apache/arrow/go/v7/arrow/internal/debug"
- "github.com/apache/arrow/go/v7/arrow/memory"
+ "github.com/apache/arrow/go/v7/arrow/memory"
+ "github.com/goccy/go-json"
)
{{range .In}}
@@ -173,6 +174,144 @@ func (b *{{.Name}}Builder) newData() (data *Data) {
return
}
+
+func (b *{{.Name}}Builder) unmarshalOne(dec *json.Decoder) error {
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ switch v := t.(type) {
+ case nil:
+ b.AppendNull()
+{{if or (eq .Name "Date32") (eq .Name "Date64") -}}
+ case string:
+ tm, err := time.Parse("2006-01-02", v)
+ if err != nil {
+ return &json.UnmarshalTypeError{
+ Value: v,
+ Type: reflect.TypeOf({{.QualifiedType}}(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+
+ b.Append({{.QualifiedType}}FromTime(tm))
+{{else if or (eq .Name "Time32") (eq .Name "Time64") (eq .Name "Timestamp") -}}
+ case string:
+ tm, err := {{.QualifiedType}}FromString(v, b.dtype.Unit)
+ if err != nil {
+ return &json.UnmarshalTypeError{
+ Value: v,
+ Type: reflect.TypeOf({{.QualifiedType}}(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+
+ b.Append(tm)
+{{else if eq .Name "Duration" -}}
+ case string:
+ // be flexible for specifying durations by accepting forms like
+ // 3h2m0.5s regardless of the unit and converting it to the proper
+ // precision.
+ val, err := time.ParseDuration(v)
+ if err != nil {
+ // if we got an error, maybe it was because the attempt to create
+ // a time.Duration (int64) in nanoseconds would overflow. check if
+ // the string is just a large number followed by the unit suffix
+ if strings.HasSuffix(v, b.dtype.Unit.String()) {
+ value, err := strconv.ParseInt(v[:len(v)-len(b.dtype.Unit.String())], 10, 64)
+ if err == nil {
+ b.Append(arrow.Duration(value))
+ break
+ }
+ }
+
+ return &json.UnmarshalTypeError{
+ Value: v,
+ Type: reflect.TypeOf({{.QualifiedType}}(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+
+ switch b.dtype.Unit {
+ case arrow.Nanosecond:
+ b.Append({{.QualifiedType}}(val.Nanoseconds()))
+ case arrow.Microsecond:
+ b.Append({{.QualifiedType}}(val.Microseconds()))
+ case arrow.Millisecond:
+ b.Append({{.QualifiedType}}(val.Milliseconds()))
+ case arrow.Second:
+ b.Append({{.QualifiedType}}(val.Seconds()))
+ }
+{{else}}
+ case string:
+{{if or (eq .Name "Float32") (eq .Name "Float64") -}}
+ f, err := strconv.ParseFloat(v, {{.Size}}*8)
+{{else if eq (printf "%.1s" .Name) "U" -}}
+ f, err := strconv.ParseUint(v, 10, {{.Size}}*8)
+{{else -}}
+ f, err := strconv.ParseInt(v, 10, {{.Size}}*8)
+{{end -}}
+ if err != nil {
+ return &json.UnmarshalTypeError{
+ Value: v,
+ Type: reflect.TypeOf({{.name}}(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+ b.Append({{.name}}(f))
+ case float64:
+ b.Append({{.name}}(v))
+ case json.Number:
+{{if or (eq .Name "Float32") (eq .Name "Float64") -}}
+ f, err := strconv.ParseFloat(v.String(), {{.Size}}*8)
+{{else if eq (printf "%.1s" .Name) "U" -}}
+ f, err := strconv.ParseUint(v.String(), 10, {{.Size}}*8)
+{{else -}}
+ f, err := strconv.ParseInt(v.String(), 10, {{.Size}}*8)
+{{end -}}
+ if err != nil {
+ return &json.UnmarshalTypeError{
+ Value: v.String(),
+ Type: reflect.TypeOf({{.name}}(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+ b.Append({{.name}}(f))
+{{end}}
+ default:
+ return &json.UnmarshalTypeError{
+ Value: fmt.Sprint(t),
+ Type: reflect.TypeOf({{or .QualifiedType .Type}}(0)),
+ Offset: dec.InputOffset(),
+ }
+ }
+
+ return nil
+}
+
+func (b *{{.Name}}Builder) unmarshal(dec *json.Decoder) error {
+ for dec.More() {
+ if err := b.unmarshalOne(dec); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (b *{{.Name}}Builder) UnmarshalJSON(data []byte) error {
+ dec := json.NewDecoder(bytes.NewReader(data))
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ if delim, ok := t.(json.Delim); !ok || delim != '[' {
+ return fmt.Errorf("binary builder must unpack from json array, found %s", delim)
+ }
+
+ return b.unmarshal(dec)
+}
{{end}}
var (
diff --git a/go/arrow/array/record.go b/go/arrow/array/record.go
index 741bc51..41e3dc4 100644
--- a/go/arrow/array/record.go
+++ b/go/arrow/array/record.go
@@ -17,6 +17,7 @@
package array
import (
+ "bytes"
"fmt"
"strings"
"sync/atomic"
@@ -24,6 +25,7 @@ import (
"github.com/apache/arrow/go/v7/arrow"
"github.com/apache/arrow/go/v7/arrow/internal/debug"
"github.com/apache/arrow/go/v7/arrow/memory"
+ "github.com/goccy/go-json"
)
// RecordReader reads a stream of records.
@@ -109,6 +111,8 @@ func (rs *simpleRecords) Next() bool {
// Record is a collection of equal-length arrays
// matching a particular Schema.
type Record interface {
+ json.Marshaler
+
Release()
Retain()
@@ -254,6 +258,12 @@ func (rec *simpleRecord) String() string {
return o.String()
}
+func (rec *simpleRecord) MarshalJSON() ([]byte, error) {
+ arr := RecordToStructArray(rec)
+ defer arr.Release()
+ return arr.MarshalJSON()
+}
+
// RecordBuilder eases the process of building a Record, iteratively, from
// a known Schema.
type RecordBuilder struct {
@@ -338,6 +348,53 @@ func (b *RecordBuilder) NewRecord() Record {
return NewRecord(b.schema, cols, rows)
}
+// UnmarshalJSON for record builder will read in a single object and add the values
+// to each field in the recordbuilder, missing fields will get a null and unexpected
+// keys will be ignored. If reading in an array of records as a single batch, then use
+// a structbuilder and use RecordFromStruct.
+func (b *RecordBuilder) UnmarshalJSON(data []byte) error {
+ dec := json.NewDecoder(bytes.NewReader(data))
+ // should start with a '{'
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ if delim, ok := t.(json.Delim); !ok || delim != '{' {
+ return fmt.Errorf("record should start with '{', not %s", t)
+ }
+
+ keylist := make(map[string]bool)
+ for dec.More() {
+ keyTok, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ key := keyTok.(string)
+ if keylist[key] {
+ return fmt.Errorf("key %s shows up twice in row to be decoded", key)
+ }
+ keylist[key] = true
+
+ indices := b.schema.FieldIndices(key)
+ if len(indices) == 0 {
+ continue
+ }
+
+ if err := b.fields[indices[0]].unmarshalOne(dec); err != nil {
+ return err
+ }
+ }
+
+ for i, f := range b.schema.Fields() {
+ if !keylist[f.Name] {
+ b.fields[i].AppendNull()
+ }
+ }
+ return nil
+}
+
var (
_ Record = (*simpleRecord)(nil)
_ RecordReader = (*simpleRecords)(nil)
diff --git a/go/arrow/array/string.go b/go/arrow/array/string.go
index b0b291f..96c7150 100644
--- a/go/arrow/array/string.go
+++ b/go/arrow/array/string.go
@@ -17,6 +17,7 @@
package array
import (
+ "bytes"
"fmt"
"math"
"reflect"
@@ -25,6 +26,7 @@ import (
"github.com/apache/arrow/go/v7/arrow"
"github.com/apache/arrow/go/v7/arrow/memory"
+ "github.com/goccy/go-json"
)
const (
@@ -118,6 +120,25 @@ func (a *String) setData(data *Data) {
}
}
+func (a *String) getOneForMarshal(i int) interface{} {
+ if a.IsValid(i) {
+ return a.Value(i)
+ }
+ return nil
+}
+
+func (a *String) MarshalJSON() ([]byte, error) {
+ vals := make([]interface{}, a.Len())
+ for i := 0; i < a.Len(); i++ {
+ if a.IsValid(i) {
+ vals[i] = a.Value(i)
+ } else {
+ vals[i] = nil
+ }
+ }
+ return json.Marshal(vals)
+}
+
func arrayEqualString(left, right *String) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
@@ -223,6 +244,50 @@ func (b *StringBuilder) NewStringArray() (a *String) {
return
}
+func (b *StringBuilder) unmarshalOne(dec *json.Decoder) error {
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ switch v := t.(type) {
+ case nil:
+ b.AppendNull()
+ case string:
+ b.Append(v)
+ default:
+ return &json.UnmarshalTypeError{
+ Value: fmt.Sprint(v),
+ Type: reflect.TypeOf(string("")),
+ Offset: dec.InputOffset(),
+ }
+ }
+ return nil
+}
+
+func (b *StringBuilder) unmarshal(dec *json.Decoder) error {
+ for dec.More() {
+ if err := b.unmarshalOne(dec); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (b *StringBuilder) UnmarshalJSON(data []byte) error {
+ dec := json.NewDecoder(bytes.NewReader(data))
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ if delim, ok := t.(json.Delim); !ok || delim != '[' {
+ return fmt.Errorf("string builder must unpack from json array, found %s", delim)
+ }
+
+ return b.unmarshal(dec)
+}
+
var (
_ Interface = (*String)(nil)
_ Builder = (*StringBuilder)(nil)
diff --git a/go/arrow/array/struct.go b/go/arrow/array/struct.go
index 807eb70..89baf4c 100644
--- a/go/arrow/array/struct.go
+++ b/go/arrow/array/struct.go
@@ -18,6 +18,7 @@ package array
import (
"bytes"
+ "errors"
"fmt"
"strings"
"sync/atomic"
@@ -26,6 +27,7 @@ import (
"github.com/apache/arrow/go/v7/arrow/bitutil"
"github.com/apache/arrow/go/v7/arrow/internal/debug"
"github.com/apache/arrow/go/v7/arrow/memory"
+ "github.com/goccy/go-json"
)
// Struct represents an ordered sequence of relative types.
@@ -105,6 +107,36 @@ func (a *Struct) setData(data *Data) {
}
}
+func (a *Struct) getOneForMarshal(i int) interface{} {
+ if a.IsNull(i) {
+ return nil
+ }
+
+ tmp := make(map[string]interface{})
+ fieldList := a.data.dtype.(*arrow.StructType).Fields()
+ for j, d := range a.fields {
+ tmp[fieldList[j].Name] = d.getOneForMarshal(i)
+ }
+ return tmp
+}
+
+func (a *Struct) MarshalJSON() ([]byte, error) {
+ var buf bytes.Buffer
+ enc := json.NewEncoder(&buf)
+
+ buf.WriteByte('[')
+ for i := 0; i < a.Len(); i++ {
+ if i != 0 {
+ buf.WriteByte(',')
+ }
+ if err := enc.Encode(a.getOneForMarshal(i)); err != nil {
+ return nil, err
+ }
+ }
+ buf.WriteByte(']')
+ return buf.Bytes(), nil
+}
+
func arrayEqualStruct(left, right *Struct) bool {
for i, lf := range left.fields {
rf := right.fields[i]
@@ -272,6 +304,79 @@ func (b *StructBuilder) newData() (data *Data) {
return
}
+func (b *StructBuilder) unmarshalOne(dec *json.Decoder) error {
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ switch t {
+ case json.Delim('{'):
+ b.Append(true)
+ keylist := make(map[string]bool)
+ for dec.More() {
+ keyTok, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ key, ok := keyTok.(string)
+ if !ok {
+ return errors.New("missing key")
+ }
+
+ if keylist[key] {
+ return fmt.Errorf("key %s is specified twice", key)
+ }
+
+ keylist[key] = true
+
+ idx, ok := b.dtype.(*arrow.StructType).FieldIdx(key)
+ if !ok {
+ continue
+ }
+
+ if err := b.fields[idx].unmarshalOne(dec); err != nil {
+ return err
+ }
+ }
+ // consume '}'
+ _, err := dec.Token()
+ return err
+ case nil:
+ b.AppendNull()
+ default:
+ return &json.UnmarshalTypeError{
+ Offset: dec.InputOffset(),
+ Struct: fmt.Sprint(b.dtype),
+ }
+ }
+ return nil
+}
+
+func (b *StructBuilder) unmarshal(dec *json.Decoder) error {
+ for dec.More() {
+ if err := b.unmarshalOne(dec); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (b *StructBuilder) UnmarshalJSON(data []byte) error {
+ dec := json.NewDecoder(bytes.NewReader(data))
+ t, err := dec.Token()
+ if err != nil {
+ return err
+ }
+
+ if delim, ok := t.(json.Delim); !ok || delim != '[' {
+ return fmt.Errorf("struct builder must unpack from json array, found %s", delim)
+ }
+
+ return b.unmarshal(dec)
+}
+
var (
_ Interface = (*Struct)(nil)
_ Builder = (*StructBuilder)(nil)
diff --git a/go/arrow/array/util.go b/go/arrow/array/util.go
index c8d7e17..b4d735e 100644
--- a/go/arrow/array/util.go
+++ b/go/arrow/array/util.go
@@ -16,9 +16,205 @@
package array
+import (
+ "errors"
+ "fmt"
+ "io"
+
+ "github.com/apache/arrow/go/v7/arrow"
+ "github.com/apache/arrow/go/v7/arrow/memory"
+ "github.com/goccy/go-json"
+)
+
func min(a, b int) int {
if a < b {
return a
}
return b
}
+
+type fromJSONCfg struct {
+ multiDocument bool
+ startOffset int64
+}
+
+type FromJSONOption func(*fromJSONCfg)
+
+func WithMultipleDocs() FromJSONOption {
+ return func(c *fromJSONCfg) {
+ c.multiDocument = true
+ }
+}
+
+// WithStartOffset attempts to start decoding from the reader at the offset
+// passed in. If using this option the reader must fulfill the io.ReadSeeker
+// interface, or else an error will be returned.
+//
+// It will call Seek(off, io.SeekStart) on the reader
+func WithStartOffset(off int64) FromJSONOption {
+ return func(c *fromJSONCfg) {
+ c.startOffset = off
+ }
+}
+
+// FromJSON creates an array.Interface from a corresponding JSON stream and defined data type. If the types in the
+// json do not match the type provided, it will return errors. This is *not* the integration test format
+// and should not be used as such. This intended to be used by consumers more similarly to the current exposing of
+// the csv reader/writer. It also returns the input offset in the reader where it finished decoding since buffering
+// by the decoder could leave the reader's cursor past where the parsing finished if attempting to parse multiple json
+// arrays from one stream.
+//
+// All the Array types implement json.Marshaller and thus can be written to json
+// using the json.Marshal function
+//
+// The JSON provided must be formatted in one of two ways:
+// Default: the top level of the json must be a list which matches the type specified exactly
+// Example: `[1, 2, 3, 4, 5]` for any integer type or `[[...], null, [], .....]` for a List type
+// Struct arrays are represented a list of objects: `[{"foo": 1, "bar": "moo"}, {"foo": 5, "bar": "baz"}]`
+//
+// Using WithMultipleDocs:
+// If the JSON provided is multiple newline separated json documents, then use this option
+// and each json document will be treated as a single row of the array. This is most useful for record batches
+// and interacting with other processes that use json. For example:
+// `{"col1": 1, "col2": "row1", "col3": ...}\n{"col1": 2, "col2": "row2", "col3": ...}\n.....`
+//
+// Duration values get formated upon marshalling as a string consisting of their numeric
+// value followed by the unit suffix such as "10s" for a value of 10 and unit of Seconds.
+// with "ms" for millisecond, "us" for microsecond, and "ns" for nanosecond as the suffixes.
+// Unmarshalling duration values is more permissive since it first tries to use Go's
+// time.ParseDuration function which means it allows values in the form 3h25m0.3s in addition
+// to the same values which are output.
+//
+// Interval types are marshalled / unmarshalled as follows:
+// MonthInterval is marshalled as an object with the format:
+// { "months": #}
+// DayTimeInterval is marshalled using Go's regular marshalling of structs:
+// { "days": #, "milliseconds": # }
+// MonthDayNanoInterval values are marshalled the same as DayTime using Go's struct marshalling:
+// { "months": #, "days": #, "nanoseconds": # }
+//
+// Times use a format of HH:MM or HH:MM:SS[.zzz] where the fractions of a second cannot
+// exceed the precision allowed by the time unit, otherwise unmarshalling will error.
+//
+// Dates use YYYY-MM-DD format
+//
+// Timestamps use RFC3339Nano format except without a timezone, all of the following are valid:
+// YYYY-MM-DD
+// YYYY-MM-DD[T]HH
+// YYYY-MM-DD[T]HH:MM
+// YYYY-MM-DD[T]HH:MM:SS[.zzzzzzzzzz]
+//
+// The fractions of a second cannot exceed the precision allowed by the timeunit of the datatype.
+//
+// When processing structs as objects order of keys does not matter, but keys cannot be repeated.
+func FromJSON(mem memory.Allocator, dt arrow.DataType, r io.Reader, opts ...FromJSONOption) (arr Interface, offset int64, err error) {
+ var cfg fromJSONCfg
+ for _, o := range opts {
+ o(&cfg)
+ }
+
+ if cfg.startOffset != 0 {
+ seeker, ok := r.(io.ReadSeeker)
+ if !ok {
+ return nil, 0, errors.New("using StartOffset option requires reader to be a ReadSeeker, cannot seek")
+ }
+
+ seeker.Seek(cfg.startOffset, io.SeekStart)
+ }
+
+ bldr := NewBuilder(mem, dt)
+ defer bldr.Release()
+
+ dec := json.NewDecoder(r)
+ defer func() {
+ if errors.Is(err, io.EOF) {
+ err = fmt.Errorf("failed parsing json: %w", io.ErrUnexpectedEOF)
+ }
+ }()
+
+ if !cfg.multiDocument {
+ t, err := dec.Token()
+ if err != nil {
+ return nil, dec.InputOffset(), err
+ }
+
+ if delim, ok := t.(json.Delim); !ok || delim != '[' {
+ return nil, dec.InputOffset(), fmt.Errorf("json doc must be an array, found %s", delim)
+ }
+ }
+
+ if err = bldr.unmarshal(dec); err != nil {
+ return nil, dec.InputOffset(), err
+ }
+
+ if !cfg.multiDocument {
+ // consume the last ']'
+ if _, err = dec.Token(); err != nil {
+ return nil, dec.InputOffset(), err
+ }
+ }
+
+ return bldr.NewArray(), dec.InputOffset(), nil
+}
+
+// RecordToStructArray constructs a struct array from the columns of the record batch
+// by referencing them, zero-copy.
+func RecordToStructArray(rec Record) *Struct {
+ cols := make([]*Data, rec.NumCols())
+ for i, c := range rec.Columns() {
+ cols[i] = c.Data()
+ }
+
+ data := NewData(arrow.StructOf(rec.Schema().Fields()...), int(rec.NumRows()), []*memory.Buffer{nil}, cols, 0, 0)
+ defer data.Release()
+
+ return NewStructData(data)
+}
+
+// RecordFromStructArray is a convenience function for converting a struct array into
+// a record batch without copying the data. If the passed in schema is nil, the fields
+// of the struct will be used to define the record batch. Otherwise the passed in
+// schema will be used to create the record batch. If passed in, the schema must match
+// the fields of the struct column.
+func RecordFromStructArray(in *Struct, schema *arrow.Schema) Record {
+ if schema == nil {
+ schema = arrow.NewSchema(in.DataType().(*arrow.StructType).Fields(), nil)
+ }
+
+ return NewRecord(schema, in.fields, int64(in.Len()))
+}
+
+// RecordFromJSON creates a record batch from JSON data. See array.FromJSON for the details
+// of formatting and logic.
+//
+// A record batch from JSON is equivalent to reading a struct array in from json and then
+// converting it to a record batch.
+func RecordFromJSON(mem memory.Allocator, schema *arrow.Schema, r io.Reader, opts ...FromJSONOption) (Record, int64, error) {
+ st := arrow.StructOf(schema.Fields()...)
+ arr, off, err := FromJSON(mem, st, r, opts...)
+ if err != nil {
+ return nil, off, err
+ }
+ defer arr.Release()
+
+ return RecordFromStructArray(arr.(*Struct), schema), off, nil
+}
+
+// RecordToJSON writes out the given record following the format of each row is a single object
+// on a single line of the output.
+func RecordToJSON(rec Record, w io.Writer) error {
+ enc := json.NewEncoder(w)
+
+ fields := rec.Schema().Fields()
+
+ cols := make(map[string]interface{})
+ for i := 0; int64(i) < rec.NumRows(); i++ {
+ for j, c := range rec.Columns() {
+ cols[fields[j].Name] = c.getOneForMarshal(i)
+ }
+ if err := enc.Encode(cols); err != nil {
+ return err
+ }
+ }
+ return nil
+}
diff --git a/go/arrow/array/util_test.go b/go/arrow/array/util_test.go
new file mode 100644
index 0000000..f8c5924
--- /dev/null
+++ b/go/arrow/array/util_test.go
@@ -0,0 +1,434 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package array_test
+
+import (
+ "bytes"
+ "fmt"
+ "io"
+ "reflect"
+ "strings"
+ "testing"
+
+ "github.com/apache/arrow/go/v7/arrow"
+ "github.com/apache/arrow/go/v7/arrow/array"
+ "github.com/apache/arrow/go/v7/arrow/decimal128"
+ "github.com/apache/arrow/go/v7/arrow/internal/arrdata"
+ "github.com/apache/arrow/go/v7/arrow/memory"
+ "github.com/goccy/go-json"
+ "github.com/stretchr/testify/assert"
+)
+
+var typemap = map[arrow.DataType]reflect.Type{
+ arrow.PrimitiveTypes.Int8: reflect.TypeOf(int8(0)),
+ arrow.PrimitiveTypes.Uint8: reflect.TypeOf(uint8(0)),
+ arrow.PrimitiveTypes.Int16: reflect.TypeOf(int16(0)),
+ arrow.PrimitiveTypes.Uint16: reflect.TypeOf(uint16(0)),
+ arrow.PrimitiveTypes.Int32: reflect.TypeOf(int32(0)),
+ arrow.PrimitiveTypes.Uint32: reflect.TypeOf(uint32(0)),
+ arrow.PrimitiveTypes.Int64: reflect.TypeOf(int64(0)),
+ arrow.PrimitiveTypes.Uint64: reflect.TypeOf(uint64(0)),
+}
+
+func TestIntegerArrsJSON(t *testing.T) {
+ const N = 10
+ types := []arrow.DataType{
+ arrow.PrimitiveTypes.Int8,
+ arrow.PrimitiveTypes.Uint8,
+ arrow.PrimitiveTypes.Int16,
+ arrow.PrimitiveTypes.Uint16,
+ arrow.PrimitiveTypes.Int32,
+ arrow.PrimitiveTypes.Uint32,
+ arrow.PrimitiveTypes.Int64,
+ arrow.PrimitiveTypes.Uint64,
+ }
+
+ for _, tt := range types {
+ t.Run(fmt.Sprint(tt), func(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ jsontest := make([]int, N)
+ vals := reflect.MakeSlice(reflect.SliceOf(typemap[tt]), N, N)
+ for i := 0; i < N; i++ {
+ vals.Index(i).Set(reflect.ValueOf(i).Convert(typemap[tt]))
+ jsontest[i] = i
+ }
+
+ data, _ := json.Marshal(jsontest)
+ arr, _, err := array.FromJSON(mem, tt, bytes.NewReader(data))
+ assert.NoError(t, err)
+ defer arr.Release()
+
+ assert.EqualValues(t, N, arr.Len())
+ assert.Zero(t, arr.NullN())
+
+ output, err := json.Marshal(arr)
+ assert.NoError(t, err)
+ assert.JSONEq(t, string(data), string(output))
+ })
+ t.Run(fmt.Sprint(tt)+" errors", func(t *testing.T) {
+ _, _, err := array.FromJSON(memory.DefaultAllocator, tt, strings.NewReader(""))
+ assert.Error(t, err)
+
+ _, _, err = array.FromJSON(memory.DefaultAllocator, tt, strings.NewReader("["))
+ assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
+
+ _, _, err = array.FromJSON(memory.DefaultAllocator, tt, strings.NewReader("0"))
+ assert.Error(t, err)
+
+ _, _, err = array.FromJSON(memory.DefaultAllocator, tt, strings.NewReader("{}"))
+ assert.Error(t, err)
+
+ _, _, err = array.FromJSON(memory.DefaultAllocator, tt, strings.NewReader("[[0]]"))
+ assert.EqualError(t, err, "json: cannot unmarshal [ into Go value of type "+tt.Name())
+ })
+ }
+}
+
+func TestStringsJSON(t *testing.T) {
+ tests := []struct {
+ jsonstring string
+ values []string
+ valids []bool
+ }{
+ {"[]", []string{}, []bool{}},
+ {`["", "foo"]`, []string{"", "foo"}, nil},
+ {`["", null]`, []string{"", ""}, []bool{true, false}},
+ // NUL character in string
+ {`["", "some\u0000char"]`, []string{"", "some\x00char"}, nil},
+ // utf8 sequence in string
+ {"[\"\xc3\xa9\"]", []string{"\xc3\xa9"}, nil},
+ // bytes < 0x20 can be represented as JSON unicode escapes
+ {`["\u0000\u001f"]`, []string{"\x00\x1f"}, nil},
+ }
+
+ for _, tt := range tests {
+ t.Run("json "+tt.jsonstring, func(t *testing.T) {
+ bldr := array.NewStringBuilder(memory.DefaultAllocator)
+ defer bldr.Release()
+
+ bldr.AppendValues(tt.values, tt.valids)
+ expected := bldr.NewStringArray()
+ defer expected.Release()
+
+ arr, _, err := array.FromJSON(memory.DefaultAllocator, arrow.BinaryTypes.String, strings.NewReader(tt.jsonstring))
+ assert.NoError(t, err)
+ defer arr.Release()
+
+ assert.Truef(t, array.ArrayEqual(expected, arr), "expected: %s\ngot: %s\n", expected, arr)
+
+ data, err := json.Marshal(arr)
+ assert.NoError(t, err)
+ assert.JSONEq(t, tt.jsonstring, string(data))
+ })
+ }
+
+ t.Run("errors", func(t *testing.T) {
+ _, _, err := array.FromJSON(memory.DefaultAllocator, arrow.BinaryTypes.String, strings.NewReader("[0]"))
+ assert.Error(t, err)
+
+ _, _, err = array.FromJSON(memory.DefaultAllocator, arrow.BinaryTypes.String, strings.NewReader("[[]]"))
+ assert.Error(t, err)
+ })
+}
+
+func TestStructArrayFromJSON(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ jsonStr := `[{"hello": 3.5, "world": true, "yo": "foo"},{"hello": 3.25, "world": false, "yo": "bar"}]`
+
+ arr, _, err := array.FromJSON(mem, arrow.StructOf(
+ arrow.Field{Name: "hello", Type: arrow.PrimitiveTypes.Float64},
+ arrow.Field{Name: "world", Type: arrow.FixedWidthTypes.Boolean},
+ arrow.Field{Name: "yo", Type: arrow.BinaryTypes.String},
+ ), strings.NewReader(jsonStr))
+ assert.NoError(t, err)
+ defer arr.Release()
+
+ output, err := json.Marshal(arr)
+ assert.NoError(t, err)
+ assert.JSONEq(t, jsonStr, string(output))
+}
+
+func TestArrayFromJSONMulti(t *testing.T) {
+ arr, _, err := array.FromJSON(memory.DefaultAllocator, arrow.StructOf(
+ arrow.Field{Name: "hello", Type: arrow.PrimitiveTypes.Float64},
+ arrow.Field{Name: "world", Type: arrow.FixedWidthTypes.Boolean},
+ arrow.Field{Name: "yo", Type: arrow.BinaryTypes.String},
+ ), strings.NewReader("{\"hello\": 3.5, \"world\": true, \"yo\": \"foo\"}\n{\"hello\": 3.25, \"world\": false, \"yo\": \"bar\"}\n"),
+ array.WithMultipleDocs())
+ assert.NoError(t, err)
+ defer arr.Release()
+
+ assert.EqualValues(t, 2, arr.Len())
+ assert.Zero(t, arr.NullN())
+}
+
+func TestNestedJSONArrs(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ jsonStr := `[{"hello": 1.5, "world": [1, 2, 3, 4], "yo": [{"foo": "2005-05-06", "bar": "15:02:04.123"},{"foo": "1956-01-02", "bar": "02:10:00"}]}]`
+
+ arr, _, err := array.FromJSON(mem, arrow.StructOf(
+ arrow.Field{Name: "hello", Type: arrow.PrimitiveTypes.Float64},
+ arrow.Field{Name: "world", Type: arrow.ListOf(arrow.PrimitiveTypes.Int32)},
+ arrow.Field{Name: "yo", Type: arrow.FixedSizeListOf(2, arrow.StructOf(
+ arrow.Field{Name: "foo", Type: arrow.FixedWidthTypes.Date32},
+ arrow.Field{Name: "bar", Type: arrow.FixedWidthTypes.Time32ms},
+ ))},
+ ), strings.NewReader(jsonStr))
+ assert.NoError(t, err)
+ defer arr.Release()
+
+ v, err := json.Marshal(arr)
+ assert.NoError(t, err)
+ assert.JSONEq(t, jsonStr, string(v))
+}
+
+func TestGetNullsFromJSON(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ jsonStr := `[
+ {"yo": "thing", "arr": null, "nuf": {"ps": "今日は"}},
+ {"yo": null, "nuf": {"ps": null}, "arr": []},
+ { "nuf": null, "yo": "今日は", "arr": [1,2,3]}
+ ]`
+
+ rec, _, err := array.RecordFromJSON(mem, arrow.NewSchema([]arrow.Field{
+ {Name: "yo", Type: arrow.BinaryTypes.String, Nullable: true},
+ {Name: "arr", Type: arrow.ListOf(arrow.PrimitiveTypes.Int32), Nullable: true},
+ {Name: "nuf", Type: arrow.StructOf(arrow.Field{Name: "ps", Type: arrow.BinaryTypes.String, Nullable: true}), Nullable: true},
+ }, nil), strings.NewReader(jsonStr))
+ assert.NoError(t, err)
+ defer rec.Release()
+
+ assert.EqualValues(t, 3, rec.NumCols())
+ assert.EqualValues(t, 3, rec.NumRows())
+
+ data, err := json.Marshal(rec)
+ assert.NoError(t, err)
+ assert.JSONEq(t, jsonStr, string(data))
+}
+
+func TestDurationsJSON(t *testing.T) {
+ tests := []struct {
+ unit arrow.TimeUnit
+ jsonstr string
+ values []arrow.Duration
+ }{
+ {arrow.Second, `["1s", "2s", "3s", "4s", "5s"]`, []arrow.Duration{1, 2, 3, 4, 5}},
+ {arrow.Millisecond, `["1ms", "2ms", "3ms", "4ms", "5ms"]`, []arrow.Duration{1, 2, 3, 4, 5}},
+ {arrow.Microsecond, `["1us", "2us", "3us", "4us", "5us"]`, []arrow.Duration{1, 2, 3, 4, 5}},
+ {arrow.Nanosecond, `["1ns", "2ns", "3ns", "4ns", "5ns"]`, []arrow.Duration{1, 2, 3, 4, 5}},
+ }
+ for _, tt := range tests {
+ dtype := &arrow.DurationType{Unit: tt.unit}
+ bldr := array.NewDurationBuilder(memory.DefaultAllocator, dtype)
+ defer bldr.Release()
+
+ bldr.AppendValues(tt.values, nil)
+ expected := bldr.NewArray()
+ defer expected.Release()
+
+ arr, _, err := array.FromJSON(memory.DefaultAllocator, dtype, strings.NewReader(tt.jsonstr))
+ assert.NoError(t, err)
+ defer arr.Release()
+
+ assert.Truef(t, array.ArrayEqual(expected, arr), "expected: %s\ngot: %s\n", expected, arr)
+ }
+}
+
+func TestTimestampsJSON(t *testing.T) {
+ tests := []struct {
+ unit arrow.TimeUnit
+ jsonstr string
+ values []arrow.Timestamp
+ }{
+ {arrow.Second, `["1970-01-01", "2000-02-29", "3989-07-14", "1900-02-28"]`, []arrow.Timestamp{0, 951782400, 63730281600, -2203977600}},
+ {arrow.Nanosecond, `["1970-01-01", "2000-02-29", "1900-02-28"]`, []arrow.Timestamp{0, 951782400000000000, -2203977600000000000}},
+ }
+
+ for _, tt := range tests {
+ dtype := &arrow.TimestampType{Unit: tt.unit}
+ bldr := array.NewTimestampBuilder(memory.DefaultAllocator, dtype)
+ defer bldr.Release()
+
+ bldr.AppendValues(tt.values, nil)
+ expected := bldr.NewArray()
+ defer expected.Release()
+
+ arr, _, err := array.FromJSON(memory.DefaultAllocator, dtype, strings.NewReader(tt.jsonstr))
+ assert.NoError(t, err)
+ defer arr.Release()
+
+ assert.Truef(t, array.ArrayEqual(expected, arr), "expected: %s\ngot: %s\n", expected, arr)
+ }
+}
+
+func TestDateJSON(t *testing.T) {
+ t.Run("date32", func(t *testing.T) {
+ bldr := array.NewDate32Builder(memory.DefaultAllocator)
+ defer bldr.Release()
+
+ jsonstr := `["1970-01-06", null, "1970-02-12"]`
+
+ bldr.AppendValues([]arrow.Date32{5, 0, 42}, []bool{true, false, true})
+ expected := bldr.NewArray()
+ defer expected.Release()
+
+ arr, _, err := array.FromJSON(memory.DefaultAllocator, arrow.FixedWidthTypes.Date32, strings.NewReader(jsonstr))
+ assert.NoError(t, err)
+ defer arr.Release()
+
+ assert.Truef(t, array.ArrayEqual(expected, arr), "expected: %s\ngot: %s\n", expected, arr)
+
+ data, err := json.Marshal(arr)
+ assert.NoError(t, err)
+ assert.JSONEq(t, jsonstr, string(data))
+ })
+ t.Run("date64", func(t *testing.T) {
+ bldr := array.NewDate64Builder(memory.DefaultAllocator)
+ defer bldr.Release()
+
+ jsonstr := `["1970-01-02", null, "2286-11-20"]`
+
+ bldr.AppendValues([]arrow.Date64{86400000, 0, 9999936000000}, []bool{true, false, true})
+ expected := bldr.NewArray()
+ defer expected.Release()
+
+ arr, _, err := array.FromJSON(memory.DefaultAllocator, arrow.FixedWidthTypes.Date64, strings.NewReader(jsonstr))
+ assert.NoError(t, err)
+ defer arr.Release()
+
+ assert.Truef(t, array.ArrayEqual(expected, arr), "expected: %s\ngot: %s\n", expected, arr)
+
+ data, err := json.Marshal(arr)
+ assert.NoError(t, err)
+ assert.JSONEq(t, jsonstr, string(data))
+ })
+}
+
+func TestTimeJSON(t *testing.T) {
+ tententen := 60*(60*(10)+10) + 10
+ tests := []struct {
+ dt arrow.DataType
+ jsonstr string
+ valueadd int
+ }{
+ {arrow.FixedWidthTypes.Time32s, `[null, "10:10:10"]`, 123},
+ {arrow.FixedWidthTypes.Time32ms, `[null, "10:10:10.123"]`, 456},
+ {arrow.FixedWidthTypes.Time64us, `[null, "10:10:10.123456"]`, 789},
+ {arrow.FixedWidthTypes.Time64ns, `[null, "10:10:10.123456789"]`, 0},
+ }
+
+ for _, tt := range tests {
+ t.Run(fmt.Sprint(tt.dt), func(t *testing.T) {
+ defer func() {
+ tententen = 1000*tententen + tt.valueadd
+ }()
+
+ bldr := array.NewBuilder(memory.DefaultAllocator, tt.dt)
+ defer bldr.Release()
+
+ switch tt.dt.ID() {
+ case arrow.TIME32:
+ bldr.(*array.Time32Builder).AppendValues([]arrow.Time32{0, arrow.Time32(tententen)}, []bool{false, true})
+ case arrow.TIME64:
+ bldr.(*array.Time64Builder).AppendValues([]arrow.Time64{0, arrow.Time64(tententen)}, []bool{false, true})
+ }
+
+ expected := bldr.NewArray()
+ defer expected.Release()
+
+ arr, _, err := array.FromJSON(memory.DefaultAllocator, tt.dt, strings.NewReader(tt.jsonstr))
+ assert.NoError(t, err)
+ defer arr.Release()
+
+ assert.Truef(t, array.ArrayEqual(expected, arr), "expected: %s\ngot: %s\n", expected, arr)
+
+ data, err := json.Marshal(arr)
+ assert.NoError(t, err)
+ assert.JSONEq(t, tt.jsonstr, string(data))
+ })
+ }
+}
+
+func TestDecimal128JSON(t *testing.T) {
+ dt := &arrow.Decimal128Type{Precision: 10, Scale: 4}
+ bldr := array.NewDecimal128Builder(memory.DefaultAllocator, dt)
+ defer bldr.Release()
+
+ bldr.AppendValues([]decimal128.Num{decimal128.FromU64(1234567), {}, decimal128.FromI64(-789000)}, []bool{true, false, true})
+ expected := bldr.NewArray()
+ defer expected.Release()
+
+ arr, _, err := array.FromJSON(memory.DefaultAllocator, dt, strings.NewReader(`["123.4567", null, "-78.9000"]`))
+ assert.NoError(t, err)
+ defer arr.Release()
+
+ assert.Truef(t, array.ArrayEqual(expected, arr), "expected: %s\ngot: %s\n", expected, arr)
+
+ data, err := json.Marshal(arr)
+ assert.NoError(t, err)
+ assert.JSONEq(t, `["123.4567", null, "-78.9"]`, string(data))
+}
+
+func TestArrRecordsJSONRoundTrip(t *testing.T) {
+ for k, v := range arrdata.Records {
+ if k == "decimal128" || k == "fixed_width_types" {
+ // test these separately since the sample data in the arrdata
+ // records doesn't lend itself to exactness when going to/from
+ // json. The fixed_width_types one uses negative values for
+ // time32 and time64 which correctly get interpreted into times,
+ // but re-encoding them in json produces the normalized positive
+ // values instead of re-creating negative ones.
+ // the decimal128 values don't get parsed *exactly* due to fun
+ // float weirdness due to their size, so smaller tests will work fine.
+ continue
+ }
+ t.Run(k, func(t *testing.T) {
+ var buf bytes.Buffer
+ assert.NotPanics(t, func() {
+ enc := json.NewEncoder(&buf)
+ for _, r := range v {
+ if err := enc.Encode(r); err != nil {
+ panic(err)
+ }
+ }
+ })
+
+ rdr := bytes.NewReader(buf.Bytes())
+ var cur int64
+
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ for _, r := range v {
+ rec, off, err := array.RecordFromJSON(mem, r.Schema(), rdr, array.WithStartOffset(cur))
+ assert.NoError(t, err)
+ defer rec.Release()
+
+ assert.Truef(t, array.RecordApproxEqual(r, rec), "expected: %s\ngot: %s\n", r, rec)
+ cur += off
+ }
+ })
+ }
+}
diff --git a/go/arrow/datatype_fixedwidth.go b/go/arrow/datatype_fixedwidth.go
index 541f091..3d11884 100644
--- a/go/arrow/datatype_fixedwidth.go
+++ b/go/arrow/datatype_fixedwidth.go
@@ -17,9 +17,12 @@
package arrow
import (
+ "encoding/json"
"fmt"
"strconv"
"time"
+
+ "golang.org/x/xerrors"
)
type BooleanType struct{}
@@ -54,6 +57,168 @@ type (
Duration int64
)
+// Date32FromTime returns a Date32 value from a time object
+func Date32FromTime(t time.Time) Date32 {
+ return Date32(t.Unix() / int64((time.Hour * 24).Seconds()))
+}
+
+func (d Date32) ToTime() time.Time {
+ return time.Unix(0, 0).UTC().AddDate(0, 0, int(d))
+}
+
+// Date64FromTime returns a Date64 value from a time object
+func Date64FromTime(t time.Time) Date64 {
+ return Date64(t.Unix()*1e3 + int64(t.Nanosecond())/1e6)
+}
+
+func (d Date64) ToTime() time.Time {
+ days := int(int64(d) / (time.Hour * 24).Milliseconds())
+ return time.Unix(0, 0).UTC().AddDate(0, 0, days)
+}
+
+// TimestampFromString parses a string and returns a timestamp for the given unit
+// level.
+//
+// The timestamp should be in one of the following forms, [T] can be either T
+// or a space, and [.zzzzzzzzz] can be either left out or up to 9 digits of
+// fractions of a second.
+//
+// YYYY-MM-DD
+// YYYY-MM-DD[T]HH
+// YYYY-MM-DD[T]HH:MM
+// YYYY-MM-DD[T]HH:MM:SS[.zzzzzzzz]
+func TimestampFromString(val string, unit TimeUnit) (Timestamp, error) {
+ format := "2006-01-02"
+ if val[len(val)-1] == 'Z' {
+ val = val[:len(val)-1]
+ }
+
+ switch {
+ case len(val) == 13:
+ format += string(val[10]) + "15"
+ case len(val) == 16:
+ format += string(val[10]) + "15:04"
+ case len(val) >= 19:
+ format += string(val[10]) + "15:04:05.999999999"
+ }
+
+ // error if we're truncating precision
+ // don't need a case for nano as time.Parse will already error if
+ // more than nanosecond precision is provided
+ switch {
+ case unit == Second && len(val) > 19:
+ return 0, xerrors.New("provided more than second precision for timestamp[s]")
+ case unit == Millisecond && len(val) > 23:
+ return 0, xerrors.New("provided more than millisecond precision for timestamp[ms]")
+ case unit == Microsecond && len(val) > 26:
+ return 0, xerrors.New("provided more than microsecond precision for timestamp[us]")
+ }
+
+ out, err := time.ParseInLocation(format, val, time.UTC)
+ if err != nil {
+ return 0, err
+ }
+
+ switch unit {
+ case Second:
+ return Timestamp(out.Unix()), nil
+ case Millisecond:
+ return Timestamp(out.Unix()*1e3 + int64(out.Nanosecond())/1e6), nil
+ case Microsecond:
+ return Timestamp(out.Unix()*1e6 + int64(out.Nanosecond())/1e3), nil
+ case Nanosecond:
+ return Timestamp(out.UnixNano()), nil
+ }
+ return 0, fmt.Errorf("unexpected timestamp unit: %s", unit)
+}
+
+func (t Timestamp) ToTime(unit TimeUnit) time.Time {
+ if unit == Second {
+ return time.Unix(int64(t), 0).UTC()
+ }
+ return time.Unix(0, int64(t)*int64(unit.Multiplier())).UTC()
+}
+
+// Time32FromString parses a string to return a Time32 value in the given unit,
+// unit needs to be only seconds or milliseconds and the string should be in the
+// form of HH:MM or HH:MM:SS[.zzz] where the fractions of a second are optional.
+func Time32FromString(val string, unit TimeUnit) (Time32, error) {
+ switch unit {
+ case Second:
+ if len(val) > 8 {
+ return 0, xerrors.New("cannot convert larger than second precision to time32s")
+ }
+ case Millisecond:
+ if len(val) > 12 {
+ return 0, xerrors.New("cannot convert larger than millisecond precision to time32ms")
+ }
+ case Microsecond, Nanosecond:
+ return 0, xerrors.New("time32 can only be seconds or milliseconds")
+ }
+
+ var (
+ out time.Time
+ err error
+ )
+ switch {
+ case len(val) == 5:
+ out, err = time.ParseInLocation("15:04", val, time.UTC)
+ default:
+ out, err = time.ParseInLocation("15:04:05.999", val, time.UTC)
+ }
+ if err != nil {
+ return 0, err
+ }
+ t := out.Sub(time.Date(0, 1, 1, 0, 0, 0, 0, time.UTC))
+ if unit == Second {
+ return Time32(t.Seconds()), nil
+ }
+ return Time32(t.Milliseconds()), nil
+}
+
+func (t Time32) ToTime(unit TimeUnit) time.Time {
+ return time.Unix(0, int64(t)*int64(unit.Multiplier())).UTC()
+}
+
+// Time64FromString parses a string to return a Time64 value in the given unit,
+// unit needs to be only microseconds or nanoseconds and the string should be in the
+// form of HH:MM or HH:MM:SS[.zzzzzzzzz] where the fractions of a second are optional.
+func Time64FromString(val string, unit TimeUnit) (Time64, error) {
+ // don't need to check length for nanoseconds as Parse will already error
+ // if more than 9 digits are provided for the fractional second
+ switch unit {
+ case Microsecond:
+ if len(val) > 15 {
+ return 0, xerrors.New("cannot convert larger than microsecond precision to time64us")
+ }
+ case Second, Millisecond:
+ return 0, xerrors.New("time64 should only be microseconds or nanoseconds")
+ }
+
+ var (
+ out time.Time
+ err error
+ )
+ switch {
+ case len(val) == 5:
+ out, err = time.ParseInLocation("15:04", val, time.UTC)
+ default:
+ out, err = time.ParseInLocation("15:04:05.999999999", val, time.UTC)
+ }
+ if err != nil {
+ return 0, err
+ }
+ t := out.Sub(time.Date(0, 1, 1, 0, 0, 0, 0, time.UTC))
+ if unit == Microsecond {
+ return Time64(t.Microseconds()), nil
+ }
+ return Time64(t.Nanoseconds()), nil
+}
+
+func (t Time64) ToTime(unit TimeUnit) time.Time {
+ return time.Unix(0, int64(t)*int64(unit.Multiplier())).UTC()
+}
+
const (
Nanosecond TimeUnit = iota
Microsecond
@@ -163,6 +328,24 @@ func (t *Decimal128Type) Fingerprint() string {
// MonthInterval represents a number of months.
type MonthInterval int32
+func (m *MonthInterval) UnmarshalJSON(data []byte) error {
+ var val struct {
+ Months int32 `json:"months"`
+ }
+ if err := json.Unmarshal(data, &val); err != nil {
+ return err
+ }
+
+ *m = MonthInterval(val.Months)
+ return nil
+}
+
+func (m MonthInterval) MarshalJSON() ([]byte, error) {
+ return json.Marshal(struct {
+ Months int32 `json:"months"`
+ }{int32(m)})
+}
+
// MonthIntervalType is encoded as a 32-bit signed integer,
// representing a number of months.
type MonthIntervalType struct{}
@@ -215,6 +398,55 @@ func (*MonthDayNanoIntervalType) Fingerprint() string {
// BitWidth returns the number of bits required to store a single element of this data type in memory.
func (*MonthDayNanoIntervalType) BitWidth() int { return 128 }
+type op int8
+
+const (
+ convDIVIDE = iota
+ convMULTIPLY
+)
+
+var timestampConversion = [...][4]struct {
+ op op
+ factor int64
+}{
+ Nanosecond: {
+ Nanosecond: {convMULTIPLY, int64(time.Nanosecond)},
+ Microsecond: {convDIVIDE, int64(time.Microsecond)},
+ Millisecond: {convDIVIDE, int64(time.Millisecond)},
+ Second: {convDIVIDE, int64(time.Second)},
+ },
+ Microsecond: {
+ Nanosecond: {convMULTIPLY, int64(time.Microsecond)},
+ Microsecond: {convMULTIPLY, 1},
+ Millisecond: {convDIVIDE, int64(time.Millisecond / time.Microsecond)},
+ Second: {convDIVIDE, int64(time.Second / time.Microsecond)},
+ },
+ Millisecond: {
+ Nanosecond: {convMULTIPLY, int64(time.Millisecond)},
+ Microsecond: {convMULTIPLY, int64(time.Millisecond / time.Microsecond)},
+ Millisecond: {convMULTIPLY, 1},
+ Second: {convDIVIDE, int64(time.Second / time.Millisecond)},
+ },
+ Second: {
+ Nanosecond: {convMULTIPLY, int64(time.Second)},
+ Microsecond: {convMULTIPLY, int64(time.Second / time.Microsecond)},
+ Millisecond: {convMULTIPLY, int64(time.Second / time.Millisecond)},
+ Second: {convMULTIPLY, 1},
+ },
+}
+
+func ConvertTimestampValue(in, out TimeUnit, value int64) int64 {
+ conv := timestampConversion[int(in)][int(out)]
+ switch conv.op {
+ case convMULTIPLY:
+ return value * conv.factor
+ case convDIVIDE:
+ return value / conv.factor
+ }
+
+ return 0
+}
+
var (
FixedWidthTypes = struct {
Boolean FixedWidthDataType
diff --git a/go/arrow/datatype_fixedwidth_test.go b/go/arrow/datatype_fixedwidth_test.go
index 238c98e..5a46fb2 100644
--- a/go/arrow/datatype_fixedwidth_test.go
+++ b/go/arrow/datatype_fixedwidth_test.go
@@ -18,6 +18,7 @@ package arrow_test
import (
"testing"
+ "time"
"github.com/apache/arrow/go/v7/arrow"
"github.com/stretchr/testify/assert"
@@ -158,6 +159,32 @@ func TestTime32Type(t *testing.T) {
}
})
}
+
+ for _, tc := range []struct {
+ unit arrow.TimeUnit
+ str string
+ want arrow.Time32
+ wantErr bool
+ }{
+ {arrow.Second, "12:21", arrow.Time32(12*3600 + 21*60), false},
+ {arrow.Second, "02:30:45", arrow.Time32(2*3600 + 30*60 + 45), false},
+ {arrow.Second, "21:21:21.21", arrow.Time32(0), true},
+ {arrow.Millisecond, "21:21:21.21", arrow.Time32(21*3600000 + 21*60000 + 21*1000 + 210), false},
+ {arrow.Millisecond, "15:02:04.123", arrow.Time32(15*3600000 + 2*60000 + 4*1000 + 123), false},
+ {arrow.Millisecond, "12:12:12.1212", arrow.Time32(0), true},
+ {arrow.Microsecond, "10:10:10", arrow.Time32(0), true},
+ {arrow.Nanosecond, "10:10:10", arrow.Time32(0), true},
+ } {
+ t.Run("FromString", func(t *testing.T) {
+ v, e := arrow.Time32FromString(tc.str, tc.unit)
+ assert.Equal(t, tc.want, v)
+ if tc.wantErr {
+ assert.Error(t, e)
+ } else {
+ assert.NoError(t, e)
+ }
+ })
+ }
}
func TestTime64Type(t *testing.T) {
@@ -187,6 +214,39 @@ func TestTime64Type(t *testing.T) {
}
})
}
+
+ const (
+ h = time.Hour
+ m = time.Minute
+ s = time.Second
+ us = time.Microsecond
+ ns = time.Nanosecond
+ )
+
+ for _, tc := range []struct {
+ unit arrow.TimeUnit
+ str string
+ want arrow.Time64
+ wantErr bool
+ }{
+ {arrow.Second, "12:21", arrow.Time64(0), true},
+ {arrow.Millisecond, "21:21:21.21", arrow.Time64(0), true},
+ {arrow.Microsecond, "10:10:10", arrow.Time64((10*h + 10*m + 10*s).Microseconds()), false},
+ {arrow.Microsecond, "22:10:15.123456", arrow.Time64((22*h + 10*m + 15*s + 123456*us).Microseconds()), false},
+ {arrow.Microsecond, "12:34:56.78901234", arrow.Time64(0), true},
+ {arrow.Nanosecond, "12:34:56.78901234", arrow.Time64(12*h + 34*m + 56*s + 789012340), false},
+ {arrow.Nanosecond, "12:34:56.1234567890", arrow.Time64(0), true},
+ } {
+ t.Run("FromString", func(t *testing.T) {
+ v, e := arrow.Time64FromString(tc.str, tc.unit)
+ assert.Equal(t, tc.want, v)
+ if tc.wantErr {
+ assert.Error(t, e)
+ } else {
+ assert.NoError(t, e)
+ }
+ })
+ }
}
func TestDurationType(t *testing.T) {
diff --git a/go/arrow/scalar/parse.go b/go/arrow/scalar/parse.go
index 7f258e5..6a54628 100644
--- a/go/arrow/scalar/parse.go
+++ b/go/arrow/scalar/parse.go
@@ -247,26 +247,10 @@ func ParseScalar(dt arrow.DataType, val string) (Scalar, error) {
return NewFloat64Scalar(float64(val)), nil
}
case arrow.TIMESTAMP:
- format := "2006-01-02"
- if val[len(val)-1] == 'Z' {
- val = val[:len(val)-1]
- }
-
- switch {
- case len(val) == 13:
- format += string(val[10]) + "15"
- case len(val) == 16:
- format += string(val[10]) + "15:04"
- case len(val) >= 19:
- format += string(val[10]) + "15:04:05.999999999"
- }
-
- out, err := time.ParseInLocation(format, val, time.UTC)
+ value, err := arrow.TimestampFromString(val, dt.(*arrow.TimestampType).Unit)
if err != nil {
return nil, err
}
-
- value := arrow.Timestamp(ConvertTimestampValue(arrow.Nanosecond, dt.(*arrow.TimestampType).Unit, out.UnixNano()))
return NewTimestampScalar(value, dt), nil
case arrow.DURATION:
value, err := time.ParseDuration(val)
@@ -292,48 +276,24 @@ func ParseScalar(dt arrow.DataType, val string) (Scalar, error) {
return nil, err
}
if dt.ID() == arrow.DATE32 {
- return NewDate32Scalar(arrow.Date32(out.Unix() / int64((time.Hour * 24).Seconds()))), nil
+ return NewDate32Scalar(arrow.Date32FromTime(out)), nil
} else {
- return NewDate64Scalar(arrow.Date64(out.Unix() * 1000)), nil
+ return NewDate64Scalar(arrow.Date64FromTime(out)), nil
}
case arrow.TIME32:
- var (
- out time.Time
- err error
- )
- switch {
- case len(val) == 5:
- out, err = time.ParseInLocation("15:04", val, time.UTC)
- default:
- out, err = time.ParseInLocation("15:04:05.999", val, time.UTC)
- }
+ tm, err := arrow.Time32FromString(val, dt.(*arrow.Time32Type).Unit)
if err != nil {
return nil, err
}
- t := out.Sub(time.Date(0, 1, 1, 0, 0, 0, 0, time.UTC))
- if dt.(*arrow.Time32Type).Unit == arrow.Second {
- return NewTime32Scalar(arrow.Time32(t.Seconds()), dt), nil
- }
- return NewTime32Scalar(arrow.Time32(t.Milliseconds()), dt), nil
+
+ return NewTime32Scalar(tm, dt), nil
case arrow.TIME64:
- var (
- out time.Time
- err error
- )
- switch {
- case len(val) == 5:
- out, err = time.ParseInLocation("15:04", val, time.UTC)
- default:
- out, err = time.ParseInLocation("15:04:05.999999999", val, time.UTC)
- }
+ tm, err := arrow.Time64FromString(val, dt.(*arrow.Time64Type).Unit)
if err != nil {
return nil, err
}
- t := out.Sub(time.Date(0, 1, 1, 0, 0, 0, 0, time.UTC))
- if dt.(*arrow.Time64Type).Unit == arrow.Microsecond {
- return NewTime64Scalar(arrow.Time64(t.Microseconds()), dt), nil
- }
- return NewTime64Scalar(arrow.Time64(t.Nanoseconds()), dt), nil
+
+ return NewTime64Scalar(tm, dt), nil
}
return nil, xerrors.Errorf("parsing of scalar for type %s not implemented", dt)
diff --git a/go/arrow/scalar/temporal.go b/go/arrow/scalar/temporal.go
index 4d9949f..4ecc46d 100644
--- a/go/arrow/scalar/temporal.go
+++ b/go/arrow/scalar/temporal.go
@@ -26,55 +26,6 @@ import (
"golang.org/x/xerrors"
)
-type op int8
-
-const (
- convDIVIDE = iota
- convMULTIPLY
-)
-
-var timestampConversion = [...][4]struct {
- op op
- factor int64
-}{
- arrow.Nanosecond: {
- arrow.Nanosecond: {convMULTIPLY, int64(time.Nanosecond)},
- arrow.Microsecond: {convDIVIDE, int64(time.Microsecond)},
- arrow.Millisecond: {convDIVIDE, int64(time.Millisecond)},
- arrow.Second: {convDIVIDE, int64(time.Second)},
- },
- arrow.Microsecond: {
- arrow.Nanosecond: {convMULTIPLY, int64(time.Microsecond)},
- arrow.Microsecond: {convMULTIPLY, 1},
- arrow.Millisecond: {convDIVIDE, int64(time.Millisecond / time.Microsecond)},
- arrow.Second: {convDIVIDE, int64(time.Second / time.Microsecond)},
- },
- arrow.Millisecond: {
- arrow.Nanosecond: {convMULTIPLY, int64(time.Millisecond)},
- arrow.Microsecond: {convMULTIPLY, int64(time.Millisecond / time.Microsecond)},
- arrow.Millisecond: {convMULTIPLY, 1},
- arrow.Second: {convDIVIDE, int64(time.Second / time.Millisecond)},
- },
- arrow.Second: {
- arrow.Nanosecond: {convMULTIPLY, int64(time.Second)},
- arrow.Microsecond: {convMULTIPLY, int64(time.Second / time.Microsecond)},
- arrow.Millisecond: {convMULTIPLY, int64(time.Second / time.Millisecond)},
- arrow.Second: {convMULTIPLY, 1},
- },
-}
-
-func ConvertTimestampValue(in, out arrow.TimeUnit, value int64) int64 {
- conv := timestampConversion[int(in)][int(out)]
- switch conv.op {
- case convMULTIPLY:
- return value * conv.factor
- case convDIVIDE:
- return value / conv.factor
- }
-
- return 0
-}
-
func temporalToString(s TemporalScalar) string {
switch s := s.(type) {
case *Date32:
@@ -178,7 +129,7 @@ func castTemporal(from TemporalScalar, to arrow.DataType) (Scalar, error) {
case *Date64:
newValue = int64(s.Value)
}
- return NewTimestampScalar(arrow.Timestamp(ConvertTimestampValue(arrow.Millisecond, to.(*arrow.TimestampType).Unit, newValue)), to), nil
+ return NewTimestampScalar(arrow.Timestamp(arrow.ConvertTimestampValue(arrow.Millisecond, to.(*arrow.TimestampType).Unit, newValue)), to), nil
}
switch s := s.(type) {
@@ -194,20 +145,20 @@ func castTemporal(from TemporalScalar, to arrow.DataType) (Scalar, error) {
case *Timestamp:
switch to := to.(type) {
case *arrow.TimestampType:
- return NewTimestampScalar(arrow.Timestamp(ConvertTimestampValue(s.Unit(), to.Unit, int64(s.Value))), to), nil
+ return NewTimestampScalar(arrow.Timestamp(arrow.ConvertTimestampValue(s.Unit(), to.Unit, int64(s.Value))), to), nil
case *arrow.Date32Type:
- millis := ConvertTimestampValue(s.Unit(), arrow.Millisecond, int64(s.Value))
+ millis := arrow.ConvertTimestampValue(s.Unit(), arrow.Millisecond, int64(s.Value))
return NewDate32Scalar(arrow.Date32(millis / int64(millisecondsInDay))), nil
case *arrow.Date64Type:
- millis := ConvertTimestampValue(s.Unit(), arrow.Millisecond, int64(s.Value))
+ millis := arrow.ConvertTimestampValue(s.Unit(), arrow.Millisecond, int64(s.Value))
return NewDate64Scalar(arrow.Date64(millis - millis%int64(millisecondsInDay))), nil
}
case TimeScalar:
switch to := to.(type) {
case *arrow.Time32Type:
- return NewTime32Scalar(arrow.Time32(ConvertTimestampValue(s.Unit(), to.Unit, int64(s.value().(arrow.Time64)))), to), nil
+ return NewTime32Scalar(arrow.Time32(arrow.ConvertTimestampValue(s.Unit(), to.Unit, int64(s.value().(arrow.Time64)))), to), nil
case *arrow.Time64Type:
- return NewTime64Scalar(arrow.Time64(ConvertTimestampValue(s.Unit(), to.Unit, int64(s.value().(arrow.Time32)))), to), nil
+ return NewTime64Scalar(arrow.Time64(arrow.ConvertTimestampValue(s.Unit(), to.Unit, int64(s.value().(arrow.Time32)))), to), nil
}
case *Duration:
@@ -215,7 +166,7 @@ func castTemporal(from TemporalScalar, to arrow.DataType) (Scalar, error) {
case *arrow.StringType:
case *arrow.DurationType:
- return NewDurationScalar(arrow.Duration(ConvertTimestampValue(s.Unit(), to.Unit, int64(s.Value))), to), nil
+ return NewDurationScalar(arrow.Duration(arrow.ConvertTimestampValue(s.Unit(), to.Unit, int64(s.Value))), to), nil
}
}
diff --git a/go/arrow/array/util.go b/go/arrow/tools.go
similarity index 89%
copy from go/arrow/array/util.go
copy to go/arrow/tools.go
index c8d7e17..2730c36 100644
--- a/go/arrow/array/util.go
+++ b/go/arrow/tools.go
@@ -14,11 +14,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package array
+//go:build tools
+// +build tools
-func min(a, b int) int {
- if a < b {
- return a
- }
- return b
-}
+package tools
+
+import (
+ _ "golang.org/x/tools/cmd/goimports"
+)
diff --git a/go/go.mod b/go/go.mod
index c4fd0fe..6bd4237 100644
--- a/go/go.mod
+++ b/go/go.mod
@@ -21,7 +21,9 @@ go 1.15
require (
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c
github.com/andybalholm/brotli v1.0.3
+ github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40
github.com/apache/thrift v0.15.0
+ github.com/goccy/go-json v0.7.10
github.com/golang/protobuf v1.5.2
github.com/golang/snappy v0.0.4
github.com/google/flatbuffers v2.0.0+incompatible
@@ -34,6 +36,7 @@ require (
github.com/zeebo/xxh3 v0.13.0
golang.org/x/exp v0.0.0-20211028214138-64b4c8e87d1a
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359
+ golang.org/x/tools v0.1.4
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
gonum.org/v1/gonum v0.9.3
google.golang.org/grpc v1.41.0
diff --git a/go/go.sum b/go/go.sum
index f992bbf..6c1e79c 100644
--- a/go/go.sum
+++ b/go/go.sum
@@ -20,6 +20,8 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF
github.com/andybalholm/brotli v1.0.3 h1:fpcw+r1N1h0Poc1F/pHbW40cUm/lMEQslZtCkBQ0UnM=
github.com/andybalholm/brotli v1.0.3/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
+github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 h1:q4dksr6ICHXqG5hm0ZW5IHyeEJXoIJSOZeBLmWPNeIQ=
+github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40/go.mod h1:Q7yQnSMnLvcXlZ8RV+jwz/6y1rQTqbX6C82SndT52Zs=
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/apache/thrift v0.15.0 h1:aGvdaR0v1t9XLgjtBYwxcBvBOTMqClzwE26CHOgjW1Y=
@@ -44,6 +46,7 @@ github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
@@ -67,6 +70,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
@@ -92,6 +97,8 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG
github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/goccy/go-json v0.7.10 h1:ulhbuNe1JqE68nMRXXTJRrUu0uhouf0VevLINxQq4Ec=
+github.com/goccy/go-json v0.7.10/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@@ -118,6 +125,7 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS
github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -129,8 +137,9 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -181,6 +190,7 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/asmfmt v1.3.1 h1:7xZi1N7s9gTLbqiM8KUv8TLyysavbTRGBT5/ly0bRtw=
github.com/klauspost/asmfmt v1.3.1/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
+github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -246,6 +256,7 @@ github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk
github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I=
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
+github.com/pierrec/lz4/v4 v4.1.8/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pierrec/lz4/v4 v4.1.9 h1:xkrjwpOP5xg1k4Nn4GX4a4YFGhscyQL/3EddJ1Xxqm8=
github.com/pierrec/lz4/v4 v4.1.9/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -309,6 +320,7 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/zeebo/xxh3 v0.13.0 h1:Dmwt3ytycfDL+wm9ljWTS3gdtaQHMwJN9tOKwNJBxJ0=
github.com/zeebo/xxh3 v0.13.0/go.mod h1:AQY73TOrhF3jNsdiM9zZOb8MThrYbZONHj7ryDBaLpg=
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
@@ -364,6 +376,7 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
golang.org/x/mobile v0.0.0-20201217150744-e6ae53a27f4f/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4=
@@ -372,6 +385,8 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.5.1-0.20210830214625-1b1db11ec8f4 h1:7Qds88gNaRx0Dz/1wOwXlR7asekh1B1u26wEwN6FcEI=
golang.org/x/mod v0.5.1-0.20210830214625-1b1db11ec8f4/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -390,8 +405,10 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
+golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -402,6 +419,7 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -422,15 +440,22 @@ golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200727154430-2d971f7391a4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210304124612-50617c2ba197/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 h1:2B5p2L5IfGiD7+b9BOoRMC6DgObAVZV+Fsp050NqXik=
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -452,7 +477,10 @@ golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtn
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
+golang.org/x/tools v0.1.4 h1:cVngSRcfgyZCzys3KYOpCFa+4dqX/Oub9tAq00ttGVs=
+golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -476,8 +504,9 @@ google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRn
google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20210630183607-d20f26d13c79 h1:s1jFTXJryg4a1mew7xv03VZD8N9XjxFhk1o4Js4WvPQ=
+google.golang.org/genproto v0.0.0-20210630183607-d20f26d13c79/go.mod h1:yiaVoXHpRzHGyxV3o4DktVWY4mSUErTKaeEOq6C3t3U=
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=
@@ -491,6 +520,8 @@ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
+google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
google.golang.org/grpc v1.41.0 h1:f+PlOh7QV4iIJkPrx5NQ7qaNGFQ3OTse67yaDHfju4E=
google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=