You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by sb...@apache.org on 2019/05/31 11:45:34 UTC

[arrow] branch master updated: ARROW-5462: [Go] support writing zero-length List arrays

This is an automated email from the ASF dual-hosted git repository.

sbinet pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git


The following commit(s) were added to refs/heads/master by this push:
     new e61bd90  ARROW-5462: [Go] support writing zero-length List arrays
e61bd90 is described below

commit e61bd90153f61a1b627e693a377f06636d2718bf
Author: Sebastien Binet <bi...@cern.ch>
AuthorDate: Fri May 31 13:45:16 2019 +0200

    ARROW-5462: [Go] support writing zero-length List arrays
    
    Author: Sebastien Binet <bi...@cern.ch>
    
    Closes #4433 from sbinet/issue-5462 and squashes the following commits:
    
    9637f35f <Sebastien Binet> ARROW-5462:  support writing zero-length List arrays
---
 go/arrow/array/record.go                |  5 ++---
 go/arrow/array/table.go                 |  2 +-
 go/arrow/internal/arrdata/arrdata.go    |  8 ++++++++
 go/arrow/ipc/cmd/arrow-cat/main_test.go | 12 +++++++++---
 go/arrow/ipc/cmd/arrow-ls/main_test.go  |  6 +++---
 go/arrow/ipc/file_test.go               |  1 -
 go/arrow/ipc/writer.go                  | 15 ++++++++++++---
 7 files changed, 35 insertions(+), 14 deletions(-)

diff --git a/go/arrow/array/record.go b/go/arrow/array/record.go
index ac70b9c..7bbdab7 100644
--- a/go/arrow/array/record.go
+++ b/go/arrow/array/record.go
@@ -18,7 +18,6 @@ package array
 
 import (
 	"fmt"
-	"reflect"
 	"sync/atomic"
 
 	"github.com/apache/arrow/go/arrow"
@@ -187,10 +186,10 @@ func (rec *simpleRecord) validate() error {
 				arr.Len(), rec.rows,
 			)
 		}
-		if !reflect.DeepEqual(f.Type, arr.DataType()) {
+		if !arrow.TypeEquals(f.Type, arr.DataType()) {
 			return fmt.Errorf("arrow/array: column %q type mismatch: got=%v, want=%v",
 				f.Name,
-				arr.DataType().Name(), f.Type.Name(),
+				arr.DataType(), f.Type,
 			)
 		}
 	}
diff --git a/go/arrow/array/table.go b/go/arrow/array/table.go
index ff2c8ba..4d8bb45 100644
--- a/go/arrow/array/table.go
+++ b/go/arrow/array/table.go
@@ -55,7 +55,7 @@ func NewColumn(field arrow.Field, chunks *Chunked) *Column {
 	}
 	col.data.Retain()
 
-	if col.data.DataType() != col.field.Type {
+	if !arrow.TypeEquals(col.data.DataType(), col.field.Type) {
 		col.data.Release()
 		panic("arrow/array: inconsistent data type")
 	}
diff --git a/go/arrow/internal/arrdata/arrdata.go b/go/arrow/internal/arrdata/arrdata.go
index c3ee8f1..0df3348 100644
--- a/go/arrow/internal/arrdata/arrdata.go
+++ b/go/arrow/internal/arrdata/arrdata.go
@@ -203,6 +203,14 @@ func makeListsRecords() []array.Record {
 				arrayOf(mem, []int32{-21, -22, -23, -24, -25}, mask),
 			}, []bool{true, false, true}),
 		},
+		[]array.Interface{
+			func() array.Interface {
+				bldr := array.NewListBuilder(mem, arrow.PrimitiveTypes.Int32)
+				defer bldr.Release()
+
+				return bldr.NewListArray()
+			}(),
+		},
 	}
 
 	defer func() {
diff --git a/go/arrow/ipc/cmd/arrow-cat/main_test.go b/go/arrow/ipc/cmd/arrow-cat/main_test.go
index 0da306f..1fcd90a 100644
--- a/go/arrow/ipc/cmd/arrow-cat/main_test.go
+++ b/go/arrow/ipc/cmd/arrow-cat/main_test.go
@@ -91,6 +91,8 @@ record 2...
   col[0] "list_nullable": [[-1 (null) (null) -4 -5] [-11 (null) (null) -14 -15] [-21 (null) (null) -24 -25]]
 record 3...
   col[0] "list_nullable": [[-1 (null) (null) -4 -5] (null) [-21 (null) (null) -24 -25]]
+record 4...
+  col[0] "list_nullable": []
 `,
 		},
 		{
@@ -276,17 +278,21 @@ record 2...
   col[0] "list_nullable": [[-1 (null) (null) -4 -5] [-11 (null) (null) -14 -15] [-21 (null) (null) -24 -25]]
 record 3...
   col[0] "list_nullable": [[-1 (null) (null) -4 -5] (null) [-21 (null) (null) -24 -25]]
+record 4...
+  col[0] "list_nullable": []
 `,
 		},
 		{
 			name: "lists",
 			want: `version: V4
-record 1/3...
+record 1/4...
   col[0] "list_nullable": [[1 (null) (null) 4 5] [11 (null) (null) 14 15] [21 (null) (null) 24 25]]
-record 2/3...
+record 2/4...
   col[0] "list_nullable": [[-1 (null) (null) -4 -5] [-11 (null) (null) -14 -15] [-21 (null) (null) -24 -25]]
-record 3/3...
+record 3/4...
   col[0] "list_nullable": [[-1 (null) (null) -4 -5] (null) [-21 (null) (null) -24 -25]]
+record 4/4...
+  col[0] "list_nullable": []
 `,
 		},
 		{
diff --git a/go/arrow/ipc/cmd/arrow-ls/main_test.go b/go/arrow/ipc/cmd/arrow-ls/main_test.go
index 7f77304..7fcccd8 100644
--- a/go/arrow/ipc/cmd/arrow-ls/main_test.go
+++ b/go/arrow/ipc/cmd/arrow-ls/main_test.go
@@ -67,7 +67,7 @@ records: 2
 			want: `schema:
   fields: 1
     - list_nullable: type=list<item: int32>, nullable
-records: 3
+records: 4
 `,
 		},
 	} {
@@ -195,7 +195,7 @@ records: 2
 			want: `schema:
   fields: 1
     - list_nullable: type=list<item: int32>, nullable
-records: 3
+records: 4
 `,
 		},
 		{
@@ -204,7 +204,7 @@ records: 3
 schema:
   fields: 1
     - list_nullable: type=list<item: int32>, nullable
-records: 3
+records: 4
 `,
 		},
 	} {
diff --git a/go/arrow/ipc/file_test.go b/go/arrow/ipc/file_test.go
index 1f0d821..136b6a7 100644
--- a/go/arrow/ipc/file_test.go
+++ b/go/arrow/ipc/file_test.go
@@ -98,5 +98,4 @@ func TestFile(t *testing.T) {
 			}
 		})
 	}
-
 }
diff --git a/go/arrow/ipc/writer.go b/go/arrow/ipc/writer.go
index def8f17..e252023 100644
--- a/go/arrow/ipc/writer.go
+++ b/go/arrow/ipc/writer.go
@@ -224,8 +224,15 @@ func (w *recordEncoder) visit(p *payload, arr array.Interface) error {
 		p.body = append(p.body, nil)
 
 	case *arrow.BooleanType:
-		data := arr.Data()
-		p.body = append(p.body, newTruncatedBitmap(w.mem, int64(data.Offset()), int64(data.Len()), data.Buffers()[1]))
+		var (
+			data = arr.Data()
+			bitm *memory.Buffer
+		)
+
+		if data.Len() != 0 {
+			bitm = newTruncatedBitmap(w.mem, int64(data.Offset()), int64(data.Len()), data.Buffers()[1])
+		}
+		p.body = append(p.body, bitm)
 
 	case arrow.FixedWidthDataType:
 		data := arr.Data()
@@ -237,7 +244,9 @@ func (w *recordEncoder) visit(p *payload, arr array.Interface) error {
 		case needTruncate(int64(data.Offset()), values, minLength):
 			panic("not implemented") // FIXME(sbinet) writer.cc:212
 		default:
-			values.Retain()
+			if values != nil {
+				values.Retain()
+			}
 		}
 		p.body = append(p.body, values)