You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by sb...@apache.org on 2019/06/03 08:24:09 UTC
[arrow] branch master updated: ARROW-5266: [Go] implement
read/write IPC for Float16
This is an automated email from the ASF dual-hosted git repository.
sbinet pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 6c6f090 ARROW-5266: [Go] implement read/write IPC for Float16
6c6f090 is described below
commit 6c6f090be97d9f853ba0e41cf63241d2368e2300
Author: Sebastien Binet <bi...@cern.ch>
AuthorDate: Mon Jun 3 10:23:49 2019 +0200
ARROW-5266: [Go] implement read/write IPC for Float16
Author: Sebastien Binet <bi...@cern.ch>
Closes #4436 from sbinet/issue-5266 and squashes the following commits:
6a1980c40 <Sebastien Binet> ARROW-5266: implement read/write IPC for Float16
---
go/arrow/internal/arrdata/arrdata.go | 54 ++++++++++++++++++++++++++++
go/arrow/ipc/cmd/arrow-cat/main_test.go | 64 +++++++++++++++++++++++++++++++++
go/arrow/ipc/cmd/arrow-ls/main_test.go | 25 +++++++++++++
go/arrow/ipc/file_reader.go | 2 +-
go/arrow/ipc/metadata.go | 6 +++-
5 files changed, 149 insertions(+), 2 deletions(-)
diff --git a/go/arrow/internal/arrdata/arrdata.go b/go/arrow/internal/arrdata/arrdata.go
index 0df3348..cc09d5b 100644
--- a/go/arrow/internal/arrdata/arrdata.go
+++ b/go/arrow/internal/arrdata/arrdata.go
@@ -23,6 +23,7 @@ import (
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/array"
+ "github.com/apache/arrow/go/arrow/float16"
"github.com/apache/arrow/go/arrow/memory"
)
@@ -37,6 +38,7 @@ func init() {
Records["lists"] = makeListsRecords()
Records["strings"] = makeStringsRecords()
Records["fixed_size_lists"] = makeFixedSizeListsRecords()
+ Records["fixed_width_types"] = makeFixedWidthTypesRecords()
for k := range Records {
RecordNames = append(RecordNames, k)
@@ -318,6 +320,51 @@ func makeStringsRecords() []array.Record {
return recs
}
+func makeFixedWidthTypesRecords() []array.Record {
+ mem := memory.NewGoAllocator()
+ schema := arrow.NewSchema(
+ []arrow.Field{
+ arrow.Field{Name: "float16s", Type: arrow.FixedWidthTypes.Float16, Nullable: true},
+ }, nil,
+ )
+
+ float16s := func(vs []float32) []float16.Num {
+ o := make([]float16.Num, len(vs))
+ for i, v := range vs {
+ o[i] = float16.New(v)
+ }
+ return o
+ }
+
+ mask := []bool{true, false, false, true, true}
+ chunks := [][]array.Interface{
+ []array.Interface{
+ arrayOf(mem, float16s([]float32{+1, +2, +3, +4, +5}), mask),
+ },
+ []array.Interface{
+ arrayOf(mem, float16s([]float32{+11, +12, +13, +14, +15}), mask),
+ },
+ []array.Interface{
+ arrayOf(mem, float16s([]float32{+21, +22, +23, +24, +25}), mask),
+ },
+ }
+
+ defer func() {
+ for _, chunk := range chunks {
+ for _, col := range chunk {
+ col.Release()
+ }
+ }
+ }()
+
+ recs := make([]array.Record, len(chunks))
+ for i, chunk := range chunks {
+ recs[i] = array.NewRecord(schema, chunk, -1)
+ }
+
+ return recs
+}
+
func arrayOf(mem memory.Allocator, a interface{}, valids []bool) array.Interface {
if mem == nil {
mem = memory.NewGoAllocator()
@@ -387,6 +434,13 @@ func arrayOf(mem memory.Allocator, a interface{}, valids []bool) array.Interface
bldr.AppendValues(a, valids)
return bldr.NewUint64Array()
+ case []float16.Num:
+ bldr := array.NewFloat16Builder(mem)
+ defer bldr.Release()
+
+ bldr.AppendValues(a, valids)
+ return bldr.NewFloat16Array()
+
case []float32:
bldr := array.NewFloat32Builder(mem)
defer bldr.Release()
diff --git a/go/arrow/ipc/cmd/arrow-cat/main_test.go b/go/arrow/ipc/cmd/arrow-cat/main_test.go
index 1fcd90a..e8bdffe 100644
--- a/go/arrow/ipc/cmd/arrow-cat/main_test.go
+++ b/go/arrow/ipc/cmd/arrow-cat/main_test.go
@@ -108,6 +108,26 @@ record 3...
col[1] "bytes": ["111" (null) (null) "444" "555"]
`,
},
+ {
+ name: "fixed_size_lists",
+ want: `record 1...
+ col[0] "fixed_size_list_nullable": [[1 (null) 3] [11 (null) 13] [21 (null) 23]]
+record 2...
+ col[0] "fixed_size_list_nullable": [[-1 (null) -3] [-11 (null) -13] [-21 (null) -23]]
+record 3...
+ col[0] "fixed_size_list_nullable": [[-1 (null) -3] (null) [-21 (null) -23]]
+`,
+ },
+ {
+ name: "fixed_width_types",
+ want: `record 1...
+ col[0] "float16s": [1 (null) (null) 4 5]
+record 2...
+ col[0] "float16s": [11 (null) (null) 14 15]
+record 3...
+ col[0] "float16s": [21 (null) (null) 24 25]
+`,
+ },
} {
t.Run(tc.name, func(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
@@ -323,6 +343,50 @@ record 3/3...
col[1] "bytes": ["111" (null) (null) "444" "555"]
`,
},
+ {
+ stream: true,
+ name: "fixed_size_lists",
+ want: `record 1...
+ col[0] "fixed_size_list_nullable": [[1 (null) 3] [11 (null) 13] [21 (null) 23]]
+record 2...
+ col[0] "fixed_size_list_nullable": [[-1 (null) -3] [-11 (null) -13] [-21 (null) -23]]
+record 3...
+ col[0] "fixed_size_list_nullable": [[-1 (null) -3] (null) [-21 (null) -23]]
+`,
+ },
+ {
+ name: "fixed_size_lists",
+ want: `version: V4
+record 1/3...
+ col[0] "fixed_size_list_nullable": [[1 (null) 3] [11 (null) 13] [21 (null) 23]]
+record 2/3...
+ col[0] "fixed_size_list_nullable": [[-1 (null) -3] [-11 (null) -13] [-21 (null) -23]]
+record 3/3...
+ col[0] "fixed_size_list_nullable": [[-1 (null) -3] (null) [-21 (null) -23]]
+`,
+ },
+ {
+ stream: true,
+ name: "fixed_width_types",
+ want: `record 1...
+ col[0] "float16s": [1 (null) (null) 4 5]
+record 2...
+ col[0] "float16s": [11 (null) (null) 14 15]
+record 3...
+ col[0] "float16s": [21 (null) (null) 24 25]
+`,
+ },
+ {
+ name: "fixed_width_types",
+ want: `version: V4
+record 1/3...
+ col[0] "float16s": [1 (null) (null) 4 5]
+record 2/3...
+ col[0] "float16s": [11 (null) (null) 14 15]
+record 3/3...
+ col[0] "float16s": [21 (null) (null) 24 25]
+`,
+ },
} {
t.Run(fmt.Sprintf("%s-stream=%v", tc.name, tc.stream), func(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
diff --git a/go/arrow/ipc/cmd/arrow-ls/main_test.go b/go/arrow/ipc/cmd/arrow-ls/main_test.go
index 7fcccd8..6aeb1b3 100644
--- a/go/arrow/ipc/cmd/arrow-ls/main_test.go
+++ b/go/arrow/ipc/cmd/arrow-ls/main_test.go
@@ -70,6 +70,31 @@ records: 2
records: 4
`,
},
+ {
+ name: "strings",
+ want: `schema:
+ fields: 2
+ - strings: type=utf8
+ - bytes: type=binary
+records: 3
+`,
+ },
+ {
+ name: "fixed_size_lists",
+ want: `schema:
+ fields: 1
+ - fixed_size_list_nullable: type=fixed_size_list<item: int32>[3], nullable
+records: 3
+`,
+ },
+ {
+ name: "fixed_width_types",
+ want: `schema:
+ fields: 1
+ - float16s: type=float16, nullable
+records: 3
+`,
+ },
} {
t.Run(tc.name, func(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
diff --git a/go/arrow/ipc/file_reader.go b/go/arrow/ipc/file_reader.go
index b51a356..2ab10fe 100644
--- a/go/arrow/ipc/file_reader.go
+++ b/go/arrow/ipc/file_reader.go
@@ -359,7 +359,7 @@ func (ctx *arrayLoaderContext) loadArray(dt arrow.DataType) array.Interface {
case *arrow.BooleanType,
*arrow.Int8Type, *arrow.Int16Type, *arrow.Int32Type, *arrow.Int64Type,
*arrow.Uint8Type, *arrow.Uint16Type, *arrow.Uint32Type, *arrow.Uint64Type,
- *arrow.Float32Type, *arrow.Float64Type:
+ *arrow.Float16Type, *arrow.Float32Type, *arrow.Float64Type:
return ctx.loadPrimitive(dt)
case *arrow.BinaryType, *arrow.StringType:
diff --git a/go/arrow/ipc/metadata.go b/go/arrow/ipc/metadata.go
index 82afedc..7914e2f 100644
--- a/go/arrow/ipc/metadata.go
+++ b/go/arrow/ipc/metadata.go
@@ -245,6 +245,10 @@ func (fv *fieldVisitor) visit(dt arrow.DataType) {
fv.dtype = flatbuf.TypeInt
fv.offset = intToFB(fv.b, int32(dt.BitWidth()), true)
+ case *arrow.Float16Type:
+ fv.dtype = flatbuf.TypeFloatingPoint
+ fv.offset = floatToFB(fv.b, int32(dt.BitWidth()))
+
case *arrow.Float32Type:
fv.dtype = flatbuf.TypeFloatingPoint
fv.offset = floatToFB(fv.b, int32(dt.BitWidth()))
@@ -574,7 +578,7 @@ func intToFB(b *flatbuffers.Builder, bw int32, isSigned bool) flatbuffers.UOffse
func floatFromFB(data flatbuf.FloatingPoint) (arrow.DataType, error) {
switch p := data.Precision(); p {
case flatbuf.PrecisionHALF:
- return nil, errors.Errorf("arrow/ipc: float16 not implemented")
+ return arrow.FixedWidthTypes.Float16, nil
case flatbuf.PrecisionSINGLE:
return arrow.PrimitiveTypes.Float32, nil
case flatbuf.PrecisionDOUBLE: